diff --git a/samples/postgres-cdc/Dockerfile b/samples/postgres-cdc/Dockerfile new file mode 100644 index 0000000000..be29822cb4 --- /dev/null +++ b/samples/postgres-cdc/Dockerfile @@ -0,0 +1,16 @@ +FROM postgres:12.16 + +RUN apt-get update +RUN apt-get install -y postgresql-12-wal2json +RUN apt-get install -y redis-tools + +COPY startup.sh . + +ENV CDC_DB_NAME=CDC_DB_NAME + +ENV REDIS_HOST=REDIS_HOST +ENV REDIS_PORT=6379 +ENV REDIS_CHANNEL=REDIS_CHANNEL + +USER postgres +CMD ./startup.sh \ No newline at end of file diff --git a/samples/postgres-cdc/README.md b/samples/postgres-cdc/README.md new file mode 100644 index 0000000000..7be4fa332d --- /dev/null +++ b/samples/postgres-cdc/README.md @@ -0,0 +1,22 @@ +This is a working CDC example for a local Postgres 12 / Redis. +It relies on wal2json for change notifications, see https://github.com/eulerto/wal2json. +It uses redis-cli to publish the notifications. + +Demo usage: + +- the CDC'ed database must pre-exist, it should first be created as follows +- run `cd cdc` +- run `docker run -p 5432:5432 -v ./postgres-data:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:12.16` +- using a postgresql client, create the 'test_db' database and a table +- Ctrl-C the docker container started above +- now run `docker compose up -d` +- checking configuration: + + - from docker desktop, open a terminal to redis container + - from that terminal, run `redis-cli SUBSCRIBE test_db_changes` + - using a postgresql client, perform some inserts, updates or deletes in the 'test_db' database + +Real usage: + +- you need to configure the env variables of the postgres service in the docker-compose.yml file +- the database name should be set to the messages repository name configured for the mediator diff --git a/samples/postgres-cdc/docker-compose.yml b/samples/postgres-cdc/docker-compose.yml new file mode 100644 index 0000000000..d0fd978920 --- /dev/null +++ b/samples/postgres-cdc/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3' + +services: + postgres: + build: . + image: pgsql-cdc/latest + container_name: cdc-pgsql + platform: linux/amd64 + ports: + - 5432:5432 + volumes: + - ./postgres-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - CDC_DB_NAME=test_db + - REDIS_HOST=host.docker.internal + - REDIS_PORT=6379 + - REDIS_CHANNEL=test_db_changes + + redis: + image: redis:7.2.3 + container_name: cdc-redis + platform: linux/amd64 + ports: + - 6379:6379 diff --git a/samples/postgres-cdc/startup.sh b/samples/postgres-cdc/startup.sh new file mode 100755 index 0000000000..ba89552636 --- /dev/null +++ b/samples/postgres-cdc/startup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# run postgres in the background so we can run pg_recvlogical +postgres -c wal_level=logical -c max_wal_senders=1 -c shared_preload_libraries=wal2json & +# ensure postgres is fully initialized before running pg_recvlogical +sleep 10s +# create replication slot if required +pg_recvlogical -d test_db --slot ${CDC_DB_NAME}_slot --create-slot --if-not-exists -P wal2json +# need a function for publishing otherwise bash shouts +publish_to_redis() { + redis-cli -h $REDIS_HOST -p $REDIS_PORT PUBLISH $REDIS_CHANNEL $1 +} +# listen to replication messages and publish them to redis +pg_recvlogical -d ${CDC_DB_NAME} --slot ${CDC_DB_NAME}_slot --start -o pretty-print=0 -f - | while read message; do publish_to_redis $message ; done +# uncomment the following if you comment the previous to keep the container running +# tail -f /dev/null