From a337e0e39d5fb9d7c7d51dacb24316d7ca921c91 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 1 Mar 2024 14:52:49 +0800 Subject: [PATCH 1/3] reduce export-item job priority to 50 --- packages/api/src/utils/createTask.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index e63cd6c7f..5b7bde636 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -66,13 +66,13 @@ export const getJobPriority = (jobName: string): number => { return 1 case TRIGGER_RULE_JOB_NAME: case CALL_WEBHOOK_JOB_NAME: - case EXPORT_ITEM_JOB_NAME: case AI_SUMMARIZE_JOB_NAME: return 5 case BULK_ACTION_JOB_NAME: case `${REFRESH_FEED_JOB_NAME}_high`: return 10 case `${REFRESH_FEED_JOB_NAME}_low`: + case EXPORT_ITEM_JOB_NAME: return 50 case EXPORT_ALL_ITEMS_JOB_NAME: case REFRESH_ALL_FEEDS_JOB_NAME: From e33cf465fe6fce7515ee227c92bad56d1fb0f6f5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 1 Mar 2024 15:00:28 +0800 Subject: [PATCH 2/3] throw error if export-item job fails --- .../api/src/jobs/integration/export_item.ts | 68 +++++++++---------- .../api/src/services/integrations/readwise.ts | 46 ++++--------- 2 files changed, 45 insertions(+), 69 deletions(-) diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts index cd99bba5e..327f16206 100644 --- a/packages/api/src/jobs/integration/export_item.ts +++ b/packages/api/src/jobs/integration/export_item.ts @@ -35,47 +35,41 @@ export const exportItem = async (jobData: ExportItemJobData) => { return } - await Promise.all( - integrations.map(async (integration) => { - const logObject = { - userId, - integrationId: integration.id, - } - logger.info('exporting item...', logObject) + // currently only readwise integration is supported + const integration = integrations[0] - try { - const client = getIntegrationClient(integration.name) + const logObject = { + userId, + integrationId: integration.id, + } + logger.info('exporting item...', logObject) - const synced = await client.export(integration.token, libraryItems) - if (!synced) { - logger.error('failed to export item', logObject) - return Promise.resolve(false) - } + const client = getIntegrationClient(integration.name) - const syncedAt = new Date() - logger.info('updating integration...', { - ...logObject, - syncedAt, - }) + const synced = await client.export(integration.token, libraryItems) + if (!synced) { + logger.error('failed to export item', logObject) + return false + } - // update integration syncedAt if successful - const updated = await updateIntegration( - integration.id, - { - syncedAt, - }, - userId - ) - logger.info('integration updated', { - ...logObject, - updated, - }) + const syncedAt = new Date() + logger.info('updating integration...', { + ...logObject, + syncedAt, + }) - return Promise.resolve(true) - } catch (err) { - logger.error('export with integration failed', err) - return Promise.resolve(false) - } - }) + // update integration syncedAt if successful + const updated = await updateIntegration( + integration.id, + { + syncedAt, + }, + userId ) + logger.info('integration updated', { + ...logObject, + updated, + }) + + return true } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 0b0b6be0c..ae23810bd 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,6 +1,6 @@ import axios from 'axios' import { LibraryItem } from '../../entity/library_item' -import { highlightUrl, wait } from '../../utils/helpers' +import { highlightUrl } from '../../utils/helpers' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' @@ -95,40 +95,22 @@ export class ReadwiseClient implements IntegrationClient { syncWithReadwise = async ( token: string, - highlights: ReadwiseHighlight[], - retryCount = 0 + highlights: ReadwiseHighlight[] ): Promise => { const url = `${this.apiUrl}/highlights` - try { - const response = await axios.post( - url, - { - highlights, + const response = await axios.post( + url, + { + highlights, + }, + { + headers: { + Authorization: `Token ${token}`, + 'Content-Type': 'application/json', }, - { - headers: { - Authorization: `Token ${token}`, - 'Content-Type': 'application/json', - }, - timeout: 5000, // 5 seconds - } - ) - return response.status === 200 - } catch (error) { - console.error(error) - - if (axios.isAxiosError(error)) { - if (error.response?.status === 429 && retryCount < 3) { - console.log('Readwise API rate limit exceeded, retrying...') - // wait for Retry-After seconds in the header if rate limited - // max retry count is 3 - const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds - await wait(parseInt(retryAfter, 10) * 1000) - return this.syncWithReadwise(token, highlights, retryCount + 1) - } + timeout: 5000, // 5 seconds } - - return false - } + ) + return response.status === 200 } } From 08e5caddc27ff366ffec37b041a71f46f5b60093 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 1 Mar 2024 15:03:01 +0800 Subject: [PATCH 3/3] retry export-item job if fails --- packages/api/src/utils/createTask.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 5b7bde636..56e209091 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -781,8 +781,12 @@ export const enqueueExportItem = async (jobData: ExportItemJobData) => { } return queue.add(EXPORT_ITEM_JOB_NAME, jobData, { - attempts: 1, + attempts: 3, priority: getJobPriority(EXPORT_ITEM_JOB_NAME), + backoff: { + type: 'exponential', + delay: 10000, // 10 seconds + }, }) }