From 9d452c099a568c4dcf5dc3b8f977cca0d92cc7d8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Nov 2023 11:14:55 +0800 Subject: [PATCH 1/2] fix pocket importer stops running when there is no unarchived items in the batch by filtering the items after fetching --- packages/integration-handler/src/index.ts | 22 ++++++++++---- .../src/integrations/index.ts | 2 +- .../src/integrations/integration.ts | 4 ++- .../src/integrations/pocket.ts | 30 +++++++------------ 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/packages/integration-handler/src/index.ts b/packages/integration-handler/src/index.ts index 30de57935..735fbe0e7 100644 --- a/packages/integration-handler/src/index.ts +++ b/packages/integration-handler/src/index.ts @@ -187,11 +187,11 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( let offset = 0 let syncedAt = req.body.syncAt const since = syncedAt - const state = req.body.state || State.UNARCHIVED // default to unarchived + const stateToImport = req.body.state || State.UNARCHIVED // default to unarchived console.log('importing pages from integration...', { userId, - state, + stateToImport, since, }) // get pages from integration @@ -199,7 +199,7 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( token: claims.token, since, offset, - state, + state: stateToImport, }) syncedAt = retrieved.since || Date.now() let retrievedData = retrieved.data @@ -231,8 +231,20 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( // paginate api calls to the integration do { + // filter out items that are deleted or archived if the stateToImport is unarchived // write the list of urls, state and labels to the stream - retrievedData.forEach((row) => stringifier.write(row)) + retrievedData + .filter((row) => { + if (row.state === State.DELETED) { + return false + } + + return ( + stateToImport !== State.UNARCHIVED || + row.state !== State.ARCHIVED + ) + }) + .forEach((row) => stringifier.write(row)) // get next pages from the integration offset += retrievedData.length @@ -241,7 +253,7 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( token: claims.token, since, offset, - state, + state: stateToImport, }) syncedAt = retrieved.since || Date.now() retrievedData = retrieved.data diff --git a/packages/integration-handler/src/integrations/index.ts b/packages/integration-handler/src/integrations/index.ts index 80cb3caff..d2dce7929 100644 --- a/packages/integration-handler/src/integrations/index.ts +++ b/packages/integration-handler/src/integrations/index.ts @@ -57,7 +57,7 @@ export const updateIntegration = async ( token: integrationToken, enabled: true, type, - taskName, + // taskName, // TODO: remove this }, }, }) diff --git a/packages/integration-handler/src/integrations/integration.ts b/packages/integration-handler/src/integrations/integration.ts index 40fbd38f7..41aab8ca7 100644 --- a/packages/integration-handler/src/integrations/integration.ts +++ b/packages/integration-handler/src/integrations/integration.ts @@ -1,16 +1,18 @@ import { Item } from '../item' export enum State { + SUCCEEDED = 'SUCCEEDED', ARCHIVED = 'ARCHIVED', UNREAD = 'UNREAD', UNARCHIVED = 'UNARCHIVED', + DELETED = 'DELETED', ALL = 'ALL', } export interface RetrievedData { url: string labels?: string[] - state?: string + state?: State } export interface RetrievedResult { data: RetrievedData[] diff --git a/packages/integration-handler/src/integrations/pocket.ts b/packages/integration-handler/src/integrations/pocket.ts index e2f911717..48f5fac78 100644 --- a/packages/integration-handler/src/integrations/pocket.ts +++ b/packages/integration-handler/src/integrations/pocket.ts @@ -122,26 +122,18 @@ export class PocketClient extends IntegrationClient { } const pocketItems = Object.values(pocketData.list) - const statusToState: Record = { - '0': 'SUCCEEDED', - '1': 'ARCHIVED', - '2': 'DELETED', + const statusToState: Record = { + '0': State.SUCCEEDED, + '1': State.ARCHIVED, + '2': State.DELETED, } - const data = pocketItems - .map((item) => ({ - url: item.given_url, - labels: item.tags - ? Object.values(item.tags).map((tag) => tag.tag) - : undefined, - state: statusToState[item.status], - })) - .filter((item) => { - if (item.state === 'DELETED') { - return false - } - - return state !== State.UNARCHIVED || item.state !== 'ARCHIVED' - }) + const data = pocketItems.map((item) => ({ + url: item.given_url, + labels: item.tags + ? Object.values(item.tags).map((tag) => tag.tag) + : undefined, + state: statusToState[item.status], + })) if (pocketData.error) { throw new Error(`Error retrieving pocket data: ${pocketData.error}`) From eb901a534bb2fb3b2a914252dbeda21f5af09cc3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Nov 2023 11:20:52 +0800 Subject: [PATCH 2/2] update comment --- packages/integration-handler/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/integration-handler/src/index.ts b/packages/integration-handler/src/index.ts index 735fbe0e7..04dc2c909 100644 --- a/packages/integration-handler/src/index.ts +++ b/packages/integration-handler/src/index.ts @@ -231,14 +231,15 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( // paginate api calls to the integration do { - // filter out items that are deleted or archived if the stateToImport is unarchived // write the list of urls, state and labels to the stream retrievedData .filter((row) => { + // filter out items that are deleted if (row.state === State.DELETED) { return false } + // filter out items that archived if the stateToImport is unarchived return ( stateToImport !== State.UNARCHIVED || row.state !== State.ARCHIVED