From dece399faf6d527995b24ba2649348e86af9bc59 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 18 Jun 2024 21:24:43 +0800 Subject: [PATCH 1/8] create index concurrently on topic column in library_item table --- packages/db/migrations/0177.do.public_item.sql | 2 -- .../db/migrations/0183.do.library_item_topic_idx.sql | 5 +++++ .../db/migrations/0183.undo.library_item_topic_idx.sql | 9 +++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100755 packages/db/migrations/0183.do.library_item_topic_idx.sql create mode 100755 packages/db/migrations/0183.undo.library_item_topic_idx.sql diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql index 6a8e27f27..a2e2d8f48 100755 --- a/packages/db/migrations/0177.do.public_item.sql +++ b/packages/db/migrations/0177.do.public_item.sql @@ -87,6 +87,4 @@ ALTER TABLE omnivore.library_item ADD COLUMN topic LTREE, ADD COLUMN score FLOAT; -CREATE INDEX library_item_topic_idx ON omnivore.library_item USING GIST (topic); - COMMIT; diff --git a/packages/db/migrations/0183.do.library_item_topic_idx.sql b/packages/db/migrations/0183.do.library_item_topic_idx.sql new file mode 100755 index 000000000..6660ad596 --- /dev/null +++ b/packages/db/migrations/0183.do.library_item_topic_idx.sql @@ -0,0 +1,5 @@ +-- Type: DO +-- Name: library_item_topic_idx +-- Description: Create index on topic column in library_item table + +CREATE INDEX CONCURRENTLY IF NOT EXISTS library_item_topic_idx ON omnivore.library_item USING GIST (topic); diff --git a/packages/db/migrations/0183.undo.library_item_topic_idx.sql b/packages/db/migrations/0183.undo.library_item_topic_idx.sql new file mode 100755 index 000000000..15cf0ac98 --- /dev/null +++ b/packages/db/migrations/0183.undo.library_item_topic_idx.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: library_item_topic_idx +-- Description: Create index on topic column in library_item table + +BEGIN; + +DROP INDEX IF EXISTS library_item_topic_idx; + +COMMIT; From ef33671f56382e4ca208b059ab0b6c74ca6caf01 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 18 Jun 2024 22:08:11 +0800 Subject: [PATCH 2/8] add schema --- packages/db/migrations/0183.undo.library_item_topic_idx.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/migrations/0183.undo.library_item_topic_idx.sql b/packages/db/migrations/0183.undo.library_item_topic_idx.sql index 15cf0ac98..c5a065551 100755 --- a/packages/db/migrations/0183.undo.library_item_topic_idx.sql +++ b/packages/db/migrations/0183.undo.library_item_topic_idx.sql @@ -4,6 +4,6 @@ BEGIN; -DROP INDEX IF EXISTS library_item_topic_idx; +DROP INDEX IF EXISTS omnivore.library_item_topic_idx; COMMIT; From d618fe92af4f59d324b79fce620f20aa9311899b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 18 Jun 2024 22:09:06 +0800 Subject: [PATCH 3/8] remove drop index query --- packages/db/migrations/0177.undo.public_item.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql index cb068a36c..966be690a 100755 --- a/packages/db/migrations/0177.undo.public_item.sql +++ b/packages/db/migrations/0177.undo.public_item.sql @@ -9,8 +9,6 @@ DROP TABLE omnivore.public_item_stats; DROP TABLE omnivore.public_item; DROP TABLE omnivore.public_item_source; -DROP INDEX omnivore.library_item_topic_idx; - ALTER TABLE omnivore.library_item DROP COLUMN seen_at, DROP COLUMN digested_at, From b50a2b3f7b6cfd8c9811d82a6da1c24d2f66612a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 19 Jun 2024 10:49:45 +0800 Subject: [PATCH 4/8] add lint migration github action --- .github/workflows/lint-migrations.yml | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/lint-migrations.yml diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml new file mode 100644 index 000000000..e10abfc49 --- /dev/null +++ b/.github/workflows/lint-migrations.yml @@ -0,0 +1,28 @@ +name: Lint Migrations + +on: + push: + branches: + - main + paths: + - 'packages/db/migrations/**' + pull_request: + branches: + - main + paths: + - 'packages/db/migrations/**' + +jobs: + lint_migrations: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Find modified migrations + run: | + modified_migrations=$(git diff --diff-filter=d --name-only origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF 'migrations/*.do.*.sql') + echo "$modified_migrations" + echo "::set-output name=file_names::$modified_migrations" + id: modified-migrations + - uses: sbdchd/squawk-action@v1 + with: + pattern: ${{ steps.modified-migrations.outputs.file_names }} From b0326b302f7e85ba0c358745139366cadc96a91d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 19 Jun 2024 10:55:42 +0800 Subject: [PATCH 5/8] update migrations sql path --- .github/workflows/lint-migrations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index e10abfc49..927b51022 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 - name: Find modified migrations run: | - modified_migrations=$(git diff --diff-filter=d --name-only origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF 'migrations/*.do.*.sql') + modified_migrations=$(git diff --diff-filter=d --name-only origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF 'packages/db/migrations/*.do.*.sql') echo "$modified_migrations" echo "::set-output name=file_names::$modified_migrations" id: modified-migrations From 6c102b02aca739a82af4fead970db3686e50154d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 19 Jun 2024 11:08:30 +0800 Subject: [PATCH 6/8] skip building docker images in test ci --- .github/workflows/build-docker-images.yml | 2 +- .github/workflows/lint-migrations.yml | 9 +----- .github/workflows/run-tests.yaml | 36 +++++++++++------------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 4da34f1a2..647e2c502 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -1,4 +1,4 @@ -name: Run tests +name: Build Docker Images on: push: branches: diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index 927b51022..c1587f696 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -1,14 +1,7 @@ name: Lint Migrations on: - push: - branches: - - main - paths: - - 'packages/db/migrations/**' pull_request: - branches: - - main paths: - 'packages/db/migrations/**' @@ -16,7 +9,7 @@ jobs: lint_migrations: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v1 - name: Find modified migrations run: | modified_migrations=$(git diff --diff-filter=d --name-only origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF 'packages/db/migrations/*.do.*.sql') diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 88ea58232..a9766271d 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -96,21 +96,21 @@ jobs: PG_LOGGER: debug REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }} MQ_REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }} - build-docker-images: - name: Build docker images - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Build the API docker image - run: 'docker build --file packages/api/Dockerfile .' - - name: Build the content-fetch docker image - run: 'docker build --file packages/content-fetch/Dockerfile .' - - name: Build the inbound-email-handler docker image - run: 'docker build --file packages/inbound-email-handler/Dockerfile .' - - name: Build the content-fetch cloud function docker image - run: 'docker build --file packages/content-fetch/Dockerfile-gcf .' - - name: Build the tts docker image - run: 'docker build --file packages/text-to-speech/Dockerfile .' + # build-docker-images: + # name: Build docker images + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # with: + # fetch-depth: 0 + # - name: Build the API docker image + # run: 'docker build --file packages/api/Dockerfile .' + # - name: Build the content-fetch docker image + # run: 'docker build --file packages/content-fetch/Dockerfile .' + # - name: Build the inbound-email-handler docker image + # run: 'docker build --file packages/inbound-email-handler/Dockerfile .' + # - name: Build the content-fetch cloud function docker image + # run: 'docker build --file packages/content-fetch/Dockerfile-gcf .' + # - name: Build the tts docker image + # run: 'docker build --file packages/text-to-speech/Dockerfile .' From 5f11726bd585929a3b5bcc5affff177116387357 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 19 Jun 2024 11:52:22 +0800 Subject: [PATCH 7/8] set lock_timeout while running migrations --- .github/workflows/run-tests.yaml | 36 ++++++++++++++++---------------- packages/db/migrate.ts | 1 + 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index a9766271d..88ea58232 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -96,21 +96,21 @@ jobs: PG_LOGGER: debug REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }} MQ_REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }} - # build-docker-images: - # name: Build docker images - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v2 - # with: - # fetch-depth: 0 - # - name: Build the API docker image - # run: 'docker build --file packages/api/Dockerfile .' - # - name: Build the content-fetch docker image - # run: 'docker build --file packages/content-fetch/Dockerfile .' - # - name: Build the inbound-email-handler docker image - # run: 'docker build --file packages/inbound-email-handler/Dockerfile .' - # - name: Build the content-fetch cloud function docker image - # run: 'docker build --file packages/content-fetch/Dockerfile-gcf .' - # - name: Build the tts docker image - # run: 'docker build --file packages/text-to-speech/Dockerfile .' + build-docker-images: + name: Build docker images + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Build the API docker image + run: 'docker build --file packages/api/Dockerfile .' + - name: Build the content-fetch docker image + run: 'docker build --file packages/content-fetch/Dockerfile .' + - name: Build the inbound-email-handler docker image + run: 'docker build --file packages/inbound-email-handler/Dockerfile .' + - name: Build the content-fetch cloud function docker image + run: 'docker build --file packages/content-fetch/Dockerfile-gcf .' + - name: Build the tts docker image + run: 'docker build --file packages/text-to-speech/Dockerfile .' diff --git a/packages/db/migrate.ts b/packages/db/migrate.ts index 97d723acd..f4867b3fa 100755 --- a/packages/db/migrate.ts +++ b/packages/db/migrate.ts @@ -39,6 +39,7 @@ const getEnv = (): DBEnv => { database: database || 'omnivore', username, password, + options: process.env.PG_EXTRA_OPTIONS, } return config From ae4e7b4b97c778b71d44689da4ca5f5011da306e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 19 Jun 2024 11:55:49 +0800 Subject: [PATCH 8/8] set lock_timeout while running migrations in github action too --- .github/workflows/run-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 88ea58232..572408202 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -77,6 +77,7 @@ jobs: PG_PASSWORD: postgres PG_DB: omnivore_test PGPASSWORD: postgres # This is required for the psql command to work without a password prompt + PG_EXTRA_OPTIONS: '-c lock_timeout=2000' - name: TypeScript Build and Lint run: | source ~/.nvm/nvm.sh