add sample cdc

Author: ericvergnaud

postgres-cdc-example/Dockerfile

@@ -0,0 +1,16 @@
+FROM postgres:12.16
+
+RUN apt-get update
+RUN apt-get install -y postgresql-12-wal2json
+RUN apt-get install -y redis-tools
+
+COPY startup.sh .
+
+ENV CDC_DB_NAME=CDC_DB_NAME
+
+ENV REDIS_HOST=REDIS_HOST
+ENV REDIS_PORT=6379
+ENV REDIS_CHANNEL=REDIS_CHANNEL
+
+USER postgres
+CMD ./startup.sh

postgres-cdc-example/README.md

@@ -0,0 +1,21 @@
+This is a working CDC example for a local Postgres 12 / Redis.
+It relies on wal2json for change notifications, see https://github.com/eulerto/wal2json.
+It uses redis-cli to publish the notifications.
+
+Demo usage:
+ - the CDC'ed database must pre-exist, it should first be created as follows
+ - run `cd cdc`
+ - run `docker run -p 5432:5432 -v ./postgres-data:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:12.16`
+ - using a postgresql client, create the 'test_db' database and a table
+ - Ctrl-C the docker container started above
+ - now run `docker compose up -d`
+ - checking configuration:
+
+   - from docker desktop, open a terminal to redis container
+   - from that terminal, run `redis-cli SUBSCRIBE test_db_changes`
+   - using a postgresql client, perform some inserts, updates or deletes in the 'test_db' database
+
+Real usage:
+ - you need to configure the env variables of the postgres service in the docker-compose.yml file
+ - the database name should be set to the messages repository name configured for the mediator
+

postgres-cdc-example/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3'
+
+services:
+  postgres:
+    build: .
+    image: pgsql-cdc/latest
+    container_name: cdc-pgsql
+    platform: linux/amd64
+    ports:
+      - 5432:5432
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - CDC_DB_NAME=test_db
+      - REDIS_HOST=host.docker.internal
+      - REDIS_PORT=6379
+      - REDIS_CHANNEL=test_db_changes
+
+  redis:
+    image: redis:7.2.3
+    container_name: cdc-redis
+    platform: linux/amd64
+    ports:
+      - 6379:6379
+
+

postgres-cdc-example/startup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# run postgres in the background so we can run pg_recvlogical
+postgres -c wal_level=logical -c max_wal_senders=1 -c shared_preload_libraries=wal2json &
+# ensure postgres is fully initialized
+sleep 10s
+# create replication slot
+pg_recvlogical -d test_db --slot messages_slot --create-slot --if-not-exists -P wal2json
+# need a function for publishing otherwise bash shouts
+publish_to_redis() {
+  redis-cli -h $REDIS_HOST -p $REDIS_PORT PUBLISH $REDIS_CHANNEL $1
+}
+# listen to replication messages and publish them to redis
+pg_recvlogical -d test_db --slot messages_slot --start -o pretty-print=0 -f - | while read message; do publish_to_redis $message ; done
+# uncomment the following if you comment the previous to keep the container running
+# tail -f /dev/null