From e19d6c86d872ab056ccb166fdb2c066d3cf87cda Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 2 Apr 2024 15:51:02 +0800 Subject: [PATCH 1/3] export last 100 items in one job --- .../src/jobs/integration/export_all_items.ts | 51 ++++++++++++++----- packages/api/src/repository/library_item.ts | 4 ++ .../api/src/services/integrations/notion.ts | 2 +- packages/api/src/services/library_item.ts | 16 ++++-- 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/packages/api/src/jobs/integration/export_all_items.ts b/packages/api/src/jobs/integration/export_all_items.ts index ad687d226..f3bf8950d 100644 --- a/packages/api/src/jobs/integration/export_all_items.ts +++ b/packages/api/src/jobs/integration/export_all_items.ts @@ -1,8 +1,11 @@ import { IntegrationType } from '../../entity/integration' -import { findIntegration, updateIntegration } from '../../services/integrations' +import { + findIntegration, + getIntegrationClient, + updateIntegration, +} from '../../services/integrations' import { findRecentLibraryItems } from '../../services/library_item' import { findActiveUser } from '../../services/user' -import { enqueueExportItem } from '../../utils/createTask' import { logger } from '../../utils/logger' export interface ExportAllItemsJobData { @@ -39,17 +42,23 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => { return } + const client = getIntegrationClient( + integration.name, + integration.token, + integration + ) + const maxItems = 100 const limit = 10 let offset = 0 - // get max 1000 most recent items from the database + // get max 100 most recent items from the database while (offset < maxItems) { const libraryItems = await findRecentLibraryItems(userId, limit, offset) if (libraryItems.length === 0) { logger.info('no library items found', { userId, }) - return + break } logger.info('enqueuing export item...', { @@ -58,24 +67,42 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => { integrationId, }) - await enqueueExportItem({ - userId, - libraryItemIds: libraryItems.map((item) => item.id), - integrationId, + const synced = await client.export(libraryItems) + if (!synced) { + logger.error('failed to export item', jobData) + continue + } + + const syncedAt = new Date() + logger.info('updating integration...', { + ...jobData, + syncedAt, + }) + + // update integration syncedAt if successful + const updated = await updateIntegration( + integration.id, + { + syncedAt, + }, + userId + ) + logger.info('integration updated', { + ...jobData, + updated, }) offset += libraryItems.length logger.info('exported items', { - userId, + ...jobData, offset, - integrationId, }) } logger.info('exported all items', { - userId, - integrationId, + ...jobData, + offset, }) // clear task name in integration diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts index fd933faf2..9dd0b315e 100644 --- a/packages/api/src/repository/library_item.ts +++ b/packages/api/src/repository/library_item.ts @@ -105,3 +105,7 @@ export const libraryItemRepository = appDataSource ) as Promise }, }) + +export const metadataColumnsInItem = getColumns(libraryItemRepository).filter( + (column) => column !== 'readableContent' && column !== 'originalContent' +) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 7e891287d..f22f1b16c 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,6 +1,6 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { updateIntegration } from '.' +import { findIntegrationByName, updateIntegration } from '.' import { Integration } from '../../entity/integration' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 9290b8ece..3e47d6d76 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -23,7 +23,10 @@ import { getRepository, queryBuilderToRawSql, } from '../repository' -import { libraryItemRepository } from '../repository/library_item' +import { + libraryItemRepository, + metadataColumnsInItem, +} from '../repository/library_item' import { Merge } from '../util' import { setRecentlySavedItemInRedis } from '../utils/helpers' import { logger } from '../utils/logger' @@ -720,10 +723,13 @@ export const findRecentLibraryItems = async ( async (tx) => tx .createQueryBuilder(LibraryItem, 'library_item') - .where('library_item.user_id = :userId', { userId }) - .andWhere('library_item.state = :state', { - state: LibraryItemState.Succeeded, - }) + .select(metadataColumnsInItem.map((column) => `library_item.${column}`)) + .leftJoinAndSelect('library_item.labels', 'labels') + .leftJoinAndSelect('library_item.highlights', 'highlights') + .where( + 'library_item.user_id = :userId AND library_item.state = :state', + { userId, state: LibraryItemState.Succeeded } + ) .orderBy('library_item.saved_at', 'DESC', 'NULLS LAST') .take(limit) .skip(offset) From d83dc17651456a67e909bd3421e5f76ca9613763 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 2 Apr 2024 17:49:29 +0800 Subject: [PATCH 2/3] fix tests --- packages/api/src/repository/library_item.ts | 4 ---- packages/api/src/services/library_item.ts | 13 ++++++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts index 9dd0b315e..fd933faf2 100644 --- a/packages/api/src/repository/library_item.ts +++ b/packages/api/src/repository/library_item.ts @@ -105,7 +105,3 @@ export const libraryItemRepository = appDataSource ) as Promise }, }) - -export const metadataColumnsInItem = getColumns(libraryItemRepository).filter( - (column) => column !== 'readableContent' && column !== 'originalContent' -) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 3e47d6d76..12e6b7581 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -23,10 +23,7 @@ import { getRepository, queryBuilderToRawSql, } from '../repository' -import { - libraryItemRepository, - metadataColumnsInItem, -} from '../repository/library_item' +import { libraryItemRepository } from '../repository/library_item' import { Merge } from '../util' import { setRecentlySavedItemInRedis } from '../utils/helpers' import { logger } from '../utils/logger' @@ -719,11 +716,17 @@ export const findRecentLibraryItems = async ( limit = 1000, offset?: number ) => { + const selectColumns = getColumns(libraryItemRepository) + .filter( + (column) => column !== 'readableContent' && column !== 'originalContent' + ) + .map((column) => `library_item.${column}`) + return authTrx( async (tx) => tx .createQueryBuilder(LibraryItem, 'library_item') - .select(metadataColumnsInItem.map((column) => `library_item.${column}`)) + .select(selectColumns) .leftJoinAndSelect('library_item.labels', 'labels') .leftJoinAndSelect('library_item.highlights', 'highlights') .where( From cc07d2bfbf8004658fb416e74396d87ac4997fb2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 2 Apr 2024 17:56:46 +0800 Subject: [PATCH 3/3] remove comment --- packages/api/src/services/integrations/notion.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index f22f1b16c..7e891287d 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,6 +1,6 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { findIntegrationByName, updateIntegration } from '.' +import { updateIntegration } from '.' import { Integration } from '../../entity/integration' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env'