From be9e8264464389bb49024386a20a14e07e2dda57 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 18 May 2024 21:24:56 +0800 Subject: [PATCH 1/6] use IN query for state --- packages/api/src/services/library_item.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 9ad7f8471..d6e6d2d22 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -305,12 +305,12 @@ export const buildQueryString = ( case InFilter.ALL: return null case InFilter.ARCHIVE: - return "(library_item.state = 'ARCHIVED' or (library_item.state != 'DELETED' and library_item.archived_at is not null))" + return "(library_item.state = 'ARCHIVED' OR library_item.archived_at IS NOT NULL)" case InFilter.TRASH: // return only deleted pages within 14 days - return "(library_item.state = 'DELETED' AND library_item.deleted_at >= now() - interval '14 days')" + return "(library_item.state = 'DELETED' AND library_item.deleted_at >= NOW() - INTERVAL '14 days')" default: { - let sql = 'library_item.archived_at is null' + let sql = 'library_item.archived_at IS NULL' if (useFolders) { const param = `folder_${parameters.length}` const folderSql = escapeQueryWithParameters( @@ -667,7 +667,9 @@ export const createSearchQueryBuilder = ( } if (!args.includeDeleted) { - queryBuilder.andWhere("library_item.state <> 'DELETED'") + queryBuilder.andWhere( + "library_item.state IN ('SUCCEEDED', 'ARCHIVED', 'PROCESSING', 'FAILED', 'CONTENT_NOT_FETCHED')" + ) } if (queryString) { From 62d4b613f4414e2325b864cbed64ad902c6155f4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 18 May 2024 21:29:12 +0800 Subject: [PATCH 2/6] add index for archived_at column --- .../db/migrations/0175.do.add_index_for_archived_at.sql | 7 +++++++ .../migrations/0175.undo.add_index_for_archived_at.sql | 9 +++++++++ 2 files changed, 16 insertions(+) create mode 100755 packages/db/migrations/0175.do.add_index_for_archived_at.sql create mode 100755 packages/db/migrations/0175.undo.add_index_for_archived_at.sql diff --git a/packages/db/migrations/0175.do.add_index_for_archived_at.sql b/packages/db/migrations/0175.do.add_index_for_archived_at.sql new file mode 100755 index 000000000..96b95e055 --- /dev/null +++ b/packages/db/migrations/0175.do.add_index_for_archived_at.sql @@ -0,0 +1,7 @@ +-- Type: DO +-- Name: add_index_for_archived_at +-- Description: Add index for archived_at column in library_item table + +CREATE INDEX CONCURRENTLY + IF NOT EXISTS library_item_user_id_archived_at_idx + ON omnivore.library_item (user_id, archived_at DESC NULLS LAST); diff --git a/packages/db/migrations/0175.undo.add_index_for_archived_at.sql b/packages/db/migrations/0175.undo.add_index_for_archived_at.sql new file mode 100755 index 000000000..0f0b37aaf --- /dev/null +++ b/packages/db/migrations/0175.undo.add_index_for_archived_at.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_index_for_archived_at +-- Description: Add index for archived_at column in library_item table + +BEGIN; + +DROP INDEX IF EXISTS omnivore.library_item_user_id_archived_at_idx; + +COMMIT; From 913215ab09370f3eff233c5e84223d34555f500d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 18 May 2024 22:05:36 +0800 Subject: [PATCH 3/6] fix archived query --- packages/api/src/services/library_item.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index d6e6d2d22..8d0e57837 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -305,7 +305,9 @@ export const buildQueryString = ( case InFilter.ALL: return null case InFilter.ARCHIVE: - return "(library_item.state = 'ARCHIVED' OR library_item.archived_at IS NOT NULL)" + return `(library_item.state = 'ARCHIVED' + OR (library_item.state IN ('SUCCEEDED', 'ARCHIVED', 'PROCESSING', 'FAILED', 'CONTENT_NOT_FETCHED') + AND library_item.archived_at IS NOT NULL))` case InFilter.TRASH: // return only deleted pages within 14 days return "(library_item.state = 'DELETED' AND library_item.deleted_at >= NOW() - INTERVAL '14 days')" From eaf5632d0adf475a68cc647bfc3fdd2433273f86 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sun, 19 May 2024 10:17:50 +0800 Subject: [PATCH 4/6] profile query time --- packages/api/src/resolvers/article/index.ts | 4 +++- packages/api/src/resolvers/function_resolvers.ts | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 026cb7988..2e659101f 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -670,7 +670,7 @@ export const searchResolver = authorized< SearchSuccess, SearchError, QuerySearchArgs ->(async (_obj, params, { uid }) => { +>(async (_obj, params, { uid, log }) => { const startCursor = params.after || '' const first = Math.min(params.first || 10, 100) // limit to 100 items @@ -679,6 +679,7 @@ export const searchResolver = authorized< return { errorCodes: [SearchErrorCode.QueryTooLong] } } + log.profile('searchResolver') const { libraryItems, count } = await searchAndCountLibraryItems( { from: Number(startCursor), @@ -691,6 +692,7 @@ export const searchResolver = authorized< }, uid ) + log.profile('searchResolver') const start = startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index b4ace7f7b..e72108cd5 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -33,6 +33,7 @@ import { wordsCount, } from '../utils/helpers' import { createImageProxyUrl } from '../utils/imageproxy' +import { logger } from '../utils/logger' import { contentConverter } from '../utils/parser' import { generateDownloadSignedUrl, @@ -482,7 +483,9 @@ export const functionResolvers = { ctx.claims && item.uploadFileId ) { + logger.profile('uploadFiles' + item.id) const upload = await ctx.dataLoaders.uploadFiles.load(item.uploadFileId) + logger.profile('uploadFiles' + item.id) if (!upload || !upload.fileName) { return undefined } @@ -515,7 +518,10 @@ export const functionResolvers = { ) { if (item.labels) return item.labels - return ctx.dataLoaders.labels.load(item.id) + logger.profile('labels' + item.id) + const labels = await ctx.dataLoaders.labels.load(item.id) + logger.profile('labels' + item.id) + return labels }, async recommendations( item: { @@ -527,9 +533,11 @@ export const functionResolvers = { ) { if (item.recommendations) return item.recommendations + logger.profile('recommendations' + item.id) const recommendations = await ctx.dataLoaders.recommendations.load( item.id ) + logger.profile('recommendations' + item.id) return recommendations.map(recommandationDataToRecommendation) }, async aiSummary(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { @@ -551,7 +559,9 @@ export const functionResolvers = { ) { if (item.highlights) return item.highlights + logger.profile('highlights' + item.id) const highlights = await ctx.dataLoaders.highlights.load(item.id) + logger.profile('highlights' + item.id) return highlights.map(highlightDataToHighlight) }, ...readingProgressHandlers, From 8b1dc1494ecb45fb63198f4724ea99ba4b1b8ffa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 20 May 2024 11:22:08 +0800 Subject: [PATCH 5/6] Drop multiple-column index for user_id and state columns in library_item table since we create a new index for count covers them --- .../db/migrations/0175.do.add_index_for_archived_at.sql | 7 ------- .../0175.do.add_index_for_library_item_count.sql | 7 +++++++ .../migrations/0175.undo.add_index_for_archived_at.sql | 9 --------- .../0175.undo.add_index_for_library_item_count.sql | 9 +++++++++ .../db/migrations/0176.do.drop_user_id_state_idx.sql | 9 +++++++++ .../db/migrations/0176.undo.drop_user_id_state_idx.sql | 7 +++++++ 6 files changed, 32 insertions(+), 16 deletions(-) delete mode 100755 packages/db/migrations/0175.do.add_index_for_archived_at.sql create mode 100755 packages/db/migrations/0175.do.add_index_for_library_item_count.sql delete mode 100755 packages/db/migrations/0175.undo.add_index_for_archived_at.sql create mode 100755 packages/db/migrations/0175.undo.add_index_for_library_item_count.sql create mode 100755 packages/db/migrations/0176.do.drop_user_id_state_idx.sql create mode 100755 packages/db/migrations/0176.undo.drop_user_id_state_idx.sql diff --git a/packages/db/migrations/0175.do.add_index_for_archived_at.sql b/packages/db/migrations/0175.do.add_index_for_archived_at.sql deleted file mode 100755 index 96b95e055..000000000 --- a/packages/db/migrations/0175.do.add_index_for_archived_at.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Type: DO --- Name: add_index_for_archived_at --- Description: Add index for archived_at column in library_item table - -CREATE INDEX CONCURRENTLY - IF NOT EXISTS library_item_user_id_archived_at_idx - ON omnivore.library_item (user_id, archived_at DESC NULLS LAST); diff --git a/packages/db/migrations/0175.do.add_index_for_library_item_count.sql b/packages/db/migrations/0175.do.add_index_for_library_item_count.sql new file mode 100755 index 000000000..16ed3664f --- /dev/null +++ b/packages/db/migrations/0175.do.add_index_for_library_item_count.sql @@ -0,0 +1,7 @@ +-- Type: DO +-- Name: add_index_for_library_item_count +-- Description: Add index for counting user items in library_item table + +CREATE INDEX CONCURRENTLY + IF NOT EXISTS library_item_count_idx + ON omnivore.library_item (user_id, state, archived_at, folder); diff --git a/packages/db/migrations/0175.undo.add_index_for_archived_at.sql b/packages/db/migrations/0175.undo.add_index_for_archived_at.sql deleted file mode 100755 index 0f0b37aaf..000000000 --- a/packages/db/migrations/0175.undo.add_index_for_archived_at.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Type: UNDO --- Name: add_index_for_archived_at --- Description: Add index for archived_at column in library_item table - -BEGIN; - -DROP INDEX IF EXISTS omnivore.library_item_user_id_archived_at_idx; - -COMMIT; diff --git a/packages/db/migrations/0175.undo.add_index_for_library_item_count.sql b/packages/db/migrations/0175.undo.add_index_for_library_item_count.sql new file mode 100755 index 000000000..f2cf91f4f --- /dev/null +++ b/packages/db/migrations/0175.undo.add_index_for_library_item_count.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_index_for_library_item_count +-- Description: Add index for counting user items in library_item table + +BEGIN; + +DROP INDEX IF EXISTS omnivore.library_item_count_idx; + +COMMIT; diff --git a/packages/db/migrations/0176.do.drop_user_id_state_idx.sql b/packages/db/migrations/0176.do.drop_user_id_state_idx.sql new file mode 100755 index 000000000..0ad10f11d --- /dev/null +++ b/packages/db/migrations/0176.do.drop_user_id_state_idx.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: drop_library_item_user_id_state_idx +-- Description: Drop multiple-column index for user_id and state columns in library_item table + +BEGIN; + +DROP INDEX IF EXISTS omnivore.library_item_user_id_state_idx; + +COMMIT; diff --git a/packages/db/migrations/0176.undo.drop_user_id_state_idx.sql b/packages/db/migrations/0176.undo.drop_user_id_state_idx.sql new file mode 100755 index 000000000..0fd27bb95 --- /dev/null +++ b/packages/db/migrations/0176.undo.drop_user_id_state_idx.sql @@ -0,0 +1,7 @@ +-- Type: UNDO +-- Name: drop_library_item_user_id_state_idx +-- Description: Drop multiple-column index for user_id and state columns in library_item table + +CREATE INDEX CONCURRENTLY + IF NOT EXISTS library_item_user_id_state_idx + ON omnivore.library_item (user_id, state); From a3fe526c2c88671f9044e44a378842dc17c941f1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 20 May 2024 11:26:06 +0800 Subject: [PATCH 6/6] remove debugging timers --- packages/api/src/resolvers/article/index.ts | 2 -- packages/api/src/resolvers/function_resolvers.ts | 8 -------- 2 files changed, 10 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 2e659101f..dc1489c9e 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -679,7 +679,6 @@ export const searchResolver = authorized< return { errorCodes: [SearchErrorCode.QueryTooLong] } } - log.profile('searchResolver') const { libraryItems, count } = await searchAndCountLibraryItems( { from: Number(startCursor), @@ -692,7 +691,6 @@ export const searchResolver = authorized< }, uid ) - log.profile('searchResolver') const start = startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index e72108cd5..aa41b724d 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -483,9 +483,7 @@ export const functionResolvers = { ctx.claims && item.uploadFileId ) { - logger.profile('uploadFiles' + item.id) const upload = await ctx.dataLoaders.uploadFiles.load(item.uploadFileId) - logger.profile('uploadFiles' + item.id) if (!upload || !upload.fileName) { return undefined } @@ -518,9 +516,7 @@ export const functionResolvers = { ) { if (item.labels) return item.labels - logger.profile('labels' + item.id) const labels = await ctx.dataLoaders.labels.load(item.id) - logger.profile('labels' + item.id) return labels }, async recommendations( @@ -533,11 +529,9 @@ export const functionResolvers = { ) { if (item.recommendations) return item.recommendations - logger.profile('recommendations' + item.id) const recommendations = await ctx.dataLoaders.recommendations.load( item.id ) - logger.profile('recommendations' + item.id) return recommendations.map(recommandationDataToRecommendation) }, async aiSummary(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { @@ -559,9 +553,7 @@ export const functionResolvers = { ) { if (item.highlights) return item.highlights - logger.profile('highlights' + item.id) const highlights = await ctx.dataLoaders.highlights.load(item.id) - logger.profile('highlights' + item.id) return highlights.map(highlightDataToHighlight) }, ...readingProgressHandlers,