diff --git a/docker-compose.yml b/docker-compose.yml index 9530513fb..a677c7920 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,61 @@ version: '3' +x-postgres: + &postgres-common + image: "ankane/pgvector:v0.5.1" + user: postgres + healthcheck: + test: "exit 0" + interval: 2s + timeout: 12s + retries: 3 + + services: postgres: - image: "ankane/pgvector:v0.5.1" + <<: *postgres-common container_name: "omnivore-postgres" - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=omnivore - - PG_POOL_MAX=20 - healthcheck: - test: "exit 0" - interval: 2s - timeout: 12s - retries: 3 expose: - 5432 ports: - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: omnivore + PG_POOL_MAX: 20 + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256\nhost replication all 0.0.0.0/0 md5" + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 + command: | + postgres + -c wal_level=replica + -c hot_standby=on + -c max_wal_senders=10 + -c max_replication_slots=10 + -c hot_standby_feedback=on + + postgres-replica: + <<: *postgres-common + container_name: "omnivore-postgres-replica" + expose: + - 5433 + ports: + - "5433:5432" + environment: + PGUSER: replicator + PGPASSWORD: replicator_password + command: | + bash -c " + until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=replication_slot --host=postgres --port=5432 + do + echo 'Waiting for primary to connect...' + sleep 1s + done + echo 'Backup done, starting replica...' + chmod 0700 /var/lib/postgresql/data + postgres + " + depends_on: + - postgres migrate: build: diff --git a/packages/api/src/data_source.ts b/packages/api/src/data_source.ts index bb6afd714..cb3ab4549 100644 --- a/packages/api/src/data_source.ts +++ b/packages/api/src/data_source.ts @@ -23,27 +23,24 @@ export const appDataSource = new DataSource({ max: env.pg.pool.max, idleTimeoutMillis: 10000, // 10 seconds }, -}) - -if (env.pg.replication) { - appDataSource.setOptions({ - replication: { - master: { - host: env.pg.host, - port: env.pg.port, - username: env.pg.userName, - password: env.pg.password, - database: env.pg.dbName, - }, - slaves: [ - { - host: env.pg.replica.host, - port: env.pg.replica.port, - username: env.pg.replica.userName, - password: env.pg.replica.password, - database: env.pg.replica.dbName, + replication: env.pg.replication + ? { + master: { + host: env.pg.host, + port: env.pg.port, + username: env.pg.userName, + password: env.pg.password, + database: env.pg.dbName, }, - ], - }, - }) -} + slaves: [ + { + host: env.pg.replica.host, + port: env.pg.replica.port, + username: env.pg.replica.userName, + password: env.pg.replica.password, + database: env.pg.replica.dbName, + }, + ], + } + : undefined, +}) diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 50606a069..7e94ea108 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -36,6 +36,7 @@ export class CustomTypeOrmLogger logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { this.logger.info(query, { + isReplicated: queryRunner?.connection?.driver?.isReplicated, parameters, }) } @@ -45,7 +46,12 @@ export class CustomTypeOrmLogger message: any, queryRunner?: QueryRunner ): void { - this.logger.log(level, message) + if (level === 'warn') { + this.logger.log('warning', message) + return + } + + this.logger.info(message) } } diff --git a/packages/db/setup.sh b/packages/db/setup.sh index 217bc86ad..8b673d9d7 100755 --- a/packages/db/setup.sh +++ b/packages/db/setup.sh @@ -6,6 +6,12 @@ echo "create $PG_DB database" psql --host $PG_HOST --username $POSTGRES_USER --command "CREATE USER app_user WITH ENCRYPTED PASSWORD '$PG_PASSWORD';" || true echo "created app_user" +psql --host $PG_HOST --username $POSTGRES_USER --command "CREATE USER replicator WITH REPLICATION ENCRYPTED PASSWORD 'replicator_password';" || true +echo "created replicator" + +psql --host $PG_HOST --username $POSTGRES_USER --command "SELECT pg_create_physical_replication_slot('replication_slot');" || true +echo "created replication_slot" + PG_USER=$POSTGRES_USER PG_PASSWORD=$PGPASSWORD yarn workspace @omnivore/db migrate psql --host $PG_HOST --username $POSTGRES_USER --dbname $PG_DB --command "GRANT omnivore_user TO app_user;" || true @@ -17,4 +23,4 @@ if [ -z "${NO_DEMO_USER}" ]; then PASSWORD='$2a$10$41G6b1BDUdxNjH1QFPJYDOM29EE0C9nTdjD1FoseuQ8vZU1NWtrh6' psql --host $PG_HOST --username $POSTGRES_USER --dbname $PG_DB --command "INSERT INTO omnivore.user (id, source, email, source_user_id, name, password) VALUES ('$USER_ID', 'EMAIL', 'demo@omnivore.app', 'demo@omnivore.app', 'Demo User', '$PASSWORD'); INSERT INTO omnivore.user_profile (user_id, username) VALUES ('$USER_ID', 'demo_user');" echo "created demo user with email: demo@omnivore.app, password: demo_password" -fi \ No newline at end of file +fi