From 34db7bb2d8de0ccfbde752b65cd60fa5096a1bf4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 20 Feb 2024 22:42:06 +0800 Subject: [PATCH] create a job for exporting item to integrations --- packages/api/src/jobs/export_item.ts | 72 ++++++++++++ .../src/services/integrations/integration.ts | 4 +- .../api/src/services/integrations/pocket.ts | 4 + .../api/src/services/integrations/readwise.ts | 107 ++++++++++++++++++ packages/api/src/utils/helpers.ts | 4 + 5 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/jobs/export_item.ts diff --git a/packages/api/src/jobs/export_item.ts b/packages/api/src/jobs/export_item.ts new file mode 100644 index 000000000..add1f1061 --- /dev/null +++ b/packages/api/src/jobs/export_item.ts @@ -0,0 +1,72 @@ +import { IntegrationType } from '../entity/integration' +import { LibraryItem } from '../entity/library_item' +import { + findIntegrations, + getIntegrationClient, + updateIntegration, +} from '../services/integrations' +import { logger } from '../utils/logger' + +export interface ExportItemJobData { + userId: string + libraryItem: LibraryItem +} + +export const EXPORT_ITEM_JOB_NAME = 'export-item' + +export const exportItem = async (jobData: ExportItemJobData) => { + const { libraryItem, userId } = jobData + const integrations = await findIntegrations(userId, { + enabled: true, + type: IntegrationType.Export, + }) + + if (integrations.length <= 0) { + return + } + + await Promise.all( + integrations.map(async (integration) => { + const logObject = { + userId, + libraryItemId: libraryItem.id, + integrationId: integration.id, + } + logger.info('exporting item...', logObject) + + try { + const client = getIntegrationClient(integration.name) + + const synced = await client.export(integration.token, [libraryItem]) + if (!synced) { + logger.error('failed to export item', logObject) + return Promise.resolve(false) + } + + const lastItemUpdatedAt = libraryItem.updatedAt + logger.info('updating integration...', { + ...logObject, + syncedAt: lastItemUpdatedAt, + }) + + // update integration syncedAt if successful + const updated = await updateIntegration( + integration.id, + { + syncedAt: lastItemUpdatedAt, + }, + userId + ) + logger.info('integration updated', { + ...logObject, + updated, + }) + + return Promise.resolve(true) + } catch (err) { + logger.error('export with integration failed', err) + return Promise.resolve(false) + } + }) + ) +} diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index 6aa51a49f..e3f1edbc8 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -1,4 +1,4 @@ -import { LibraryItemState } from '../../entity/library_item' +import { LibraryItem, LibraryItemState } from '../../entity/library_item' export interface RetrievedData { url: string @@ -23,4 +23,6 @@ export interface IntegrationClient { apiUrl: string accessToken(token: string): Promise + + export(token: string, items: LibraryItem[]): Promise } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 1ea4d6d87..517d3befa 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -35,4 +35,8 @@ export class PocketClient implements IntegrationClient { return null } } + + export = async (): Promise => { + return Promise.resolve(false) + } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 7764294d2..0b0b6be0c 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,7 +1,36 @@ import axios from 'axios' +import { LibraryItem } from '../../entity/library_item' +import { highlightUrl, wait } from '../../utils/helpers' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' +interface ReadwiseHighlight { + // The highlight text, (technically the only field required in a highlight object) + text: string + // The title of the page the highlight is on + title?: string + // The author of the page the highlight is on + author?: string + // The URL of the page image + image_url?: string + // The URL of the page + source_url?: string + // A meaningful unique identifier for your app + source_type?: string + // One of: books, articles, tweets or podcasts + category?: string + // Annotation note attached to the specific highlight + note?: string + // Highlight's location in the source text. Used to order the highlights + location?: number + // One of: page, order or time_offset + location_type?: string + // A datetime representing when the highlight was taken in the ISO 8601 format + highlighted_at?: string + // Unique url of the specific highlight + highlight_url?: string +} + export class ReadwiseClient implements IntegrationClient { name = 'READWISE' apiUrl = 'https://readwise.io/api/v2' @@ -24,4 +53,82 @@ export class ReadwiseClient implements IntegrationClient { return null } } + + export = async (token: string, items: LibraryItem[]): Promise => { + let result = true + + const highlights = items.flatMap(this.itemToReadwiseHighlight) + + // If there are no highlights, we will skip the sync + if (highlights.length > 0) { + result = await this.syncWithReadwise(token, highlights) + } + + return result + } + + itemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => { + const category = item.siteName === 'Twitter' ? 'tweets' : 'articles' + return item.highlights + ?.map((highlight) => { + // filter out highlights that are not of type highlight or have no quote + if (highlight.highlightType !== 'HIGHLIGHT' || !highlight.quote) { + return undefined + } + + return { + text: highlight.quote, + title: item.title, + author: item.author || undefined, + highlight_url: highlightUrl(item.slug, highlight.id), + highlighted_at: new Date(highlight.createdAt).toISOString(), + category, + image_url: item.thumbnail || undefined, + location_type: 'order', + note: highlight.annotation || undefined, + source_type: 'omnivore', + source_url: item.originalUrl, + } + }) + .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[] + } + + syncWithReadwise = async ( + token: string, + highlights: ReadwiseHighlight[], + retryCount = 0 + ): Promise => { + const url = `${this.apiUrl}/highlights` + try { + const response = await axios.post( + url, + { + highlights, + }, + { + headers: { + Authorization: `Token ${token}`, + 'Content-Type': 'application/json', + }, + timeout: 5000, // 5 seconds + } + ) + return response.status === 200 + } catch (error) { + console.error(error) + + if (axios.isAxiosError(error)) { + if (error.response?.status === 429 && retryCount < 3) { + console.log('Readwise API rate limit exceeded, retrying...') + // wait for Retry-After seconds in the header if rate limited + // max retry count is 3 + const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds + await wait(parseInt(retryAfter, 10) * 1000) + return this.syncWithReadwise(token, highlights, retryCount + 1) + } + } + + return false + } + } } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index c53b1f630..e76a9875a 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -10,6 +10,7 @@ import { Highlight as HighlightData } from '../entity/highlight' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { Recommendation as RecommendationData } from '../entity/recommendation' import { RegistrationType, User } from '../entity/user' +import { env } from '../env' import { Article, ArticleSavingRequest, @@ -400,3 +401,6 @@ export const setRecentlySavedItemInRedis = async ( }) } } + +export const highlightUrl = (slug: string, highlightId: string): string => + `${env.client.url}/me/${slug}#${highlightId}`