create a job for exporting item to integrations

packages/api/src/jobs/export_item.ts

@@ -0,0 +1,72 @@
+import { IntegrationType } from '../entity/integration'
+import { LibraryItem } from '../entity/library_item'
+import {
+  findIntegrations,
+  getIntegrationClient,
+  updateIntegration,
+} from '../services/integrations'
+import { logger } from '../utils/logger'
+
+export interface ExportItemJobData {
+  userId: string
+  libraryItem: LibraryItem
+}
+
+export const EXPORT_ITEM_JOB_NAME = 'export-item'
+
+export const exportItem = async (jobData: ExportItemJobData) => {
+  const { libraryItem, userId } = jobData
+  const integrations = await findIntegrations(userId, {
+    enabled: true,
+    type: IntegrationType.Export,
+  })
+
+  if (integrations.length <= 0) {
+    return
+  }
+
+  await Promise.all(
+    integrations.map(async (integration) => {
+      const logObject = {
+        userId,
+        libraryItemId: libraryItem.id,
+        integrationId: integration.id,
+      }
+      logger.info('exporting item...', logObject)
+
+      try {
+        const client = getIntegrationClient(integration.name)
+
+        const synced = await client.export(integration.token, [libraryItem])
+        if (!synced) {
+          logger.error('failed to export item', logObject)
+          return Promise.resolve(false)
+        }
+
+        const lastItemUpdatedAt = libraryItem.updatedAt
+        logger.info('updating integration...', {
+          ...logObject,
+          syncedAt: lastItemUpdatedAt,
+        })
+
+        // update integration syncedAt if successful
+        const updated = await updateIntegration(
+          integration.id,
+          {
+            syncedAt: lastItemUpdatedAt,
+          },
+          userId
+        )
+        logger.info('integration updated', {
+          ...logObject,
+          updated,
+        })
+
+        return Promise.resolve(true)
+      } catch (err) {
+        logger.error('export with integration failed', err)
+        return Promise.resolve(false)
+      }
+    })
+  )
+}

packages/api/src/services/integrations/integration.ts

@@ -1,4 +1,4 @@
-import { LibraryItemState } from '../../entity/library_item'
+import { LibraryItem, LibraryItemState } from '../../entity/library_item'
 
 export interface RetrievedData {
   url: string
@@ -23,4 +23,6 @@ export interface IntegrationClient {
   apiUrl: string
 
   accessToken(token: string): Promise<string | null>
+
+  export(token: string, items: LibraryItem[]): Promise<boolean>
 }

packages/api/src/services/integrations/pocket.ts

@@ -35,4 +35,8 @@ export class PocketClient implements IntegrationClient {
       return null
     }
   }
+
+  export = async (): Promise<boolean> => {
+    return Promise.resolve(false)
+  }
 }

packages/api/src/services/integrations/readwise.ts

@@ -1,7 +1,36 @@
 import axios from 'axios'
+import { LibraryItem } from '../../entity/library_item'
+import { highlightUrl, wait } from '../../utils/helpers'
 import { logger } from '../../utils/logger'
 import { IntegrationClient } from './integration'
 
+interface ReadwiseHighlight {
+  // The highlight text, (technically the only field required in a highlight object)
+  text: string
+  // The title of the page the highlight is on
+  title?: string
+  // The author of the page the highlight is on
+  author?: string
+  // The URL of the page image
+  image_url?: string
+  // The URL of the page
+  source_url?: string
+  // A meaningful unique identifier for your app
+  source_type?: string
+  // One of: books, articles, tweets or podcasts
+  category?: string
+  // Annotation note attached to the specific highlight
+  note?: string
+  // Highlight's location in the source text. Used to order the highlights
+  location?: number
+  // One of: page, order or time_offset
+  location_type?: string
+  // A datetime representing when the highlight was taken in the ISO 8601 format
+  highlighted_at?: string
+  // Unique url of the specific highlight
+  highlight_url?: string
+}
+
 export class ReadwiseClient implements IntegrationClient {
   name = 'READWISE'
   apiUrl = 'https://readwise.io/api/v2'
@@ -24,4 +53,82 @@ export class ReadwiseClient implements IntegrationClient {
       return null
     }
   }
+
+  export = async (token: string, items: LibraryItem[]): Promise<boolean> => {
+    let result = true
+
+    const highlights = items.flatMap(this.itemToReadwiseHighlight)
+
+    // If there are no highlights, we will skip the sync
+    if (highlights.length > 0) {
+      result = await this.syncWithReadwise(token, highlights)
+    }
+
+    return result
+  }
+
+  itemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => {
+    const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
+    return item.highlights
+      ?.map((highlight) => {
+        // filter out highlights that are not of type highlight or have no quote
+        if (highlight.highlightType !== 'HIGHLIGHT' || !highlight.quote) {
+          return undefined
+        }
+
+        return {
+          text: highlight.quote,
+          title: item.title,
+          author: item.author || undefined,
+          highlight_url: highlightUrl(item.slug, highlight.id),
+          highlighted_at: new Date(highlight.createdAt).toISOString(),
+          category,
+          image_url: item.thumbnail || undefined,
+          location_type: 'order',
+          note: highlight.annotation || undefined,
+          source_type: 'omnivore',
+          source_url: item.originalUrl,
+        }
+      })
+      .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[]
+  }
+
+  syncWithReadwise = async (
+    token: string,
+    highlights: ReadwiseHighlight[],
+    retryCount = 0
+  ): Promise<boolean> => {
+    const url = `${this.apiUrl}/highlights`
+    try {
+      const response = await axios.post(
+        url,
+        {
+          highlights,
+        },
+        {
+          headers: {
+            Authorization: `Token ${token}`,
+            'Content-Type': 'application/json',
+          },
+          timeout: 5000, // 5 seconds
+        }
+      )
+      return response.status === 200
+    } catch (error) {
+      console.error(error)
+
+      if (axios.isAxiosError(error)) {
+        if (error.response?.status === 429 && retryCount < 3) {
+          console.log('Readwise API rate limit exceeded, retrying...')
+          // wait for Retry-After seconds in the header if rate limited
+          // max retry count is 3
+          const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
+          await wait(parseInt(retryAfter, 10) * 1000)
+          return this.syncWithReadwise(token, highlights, retryCount + 1)
+        }
+      }
+
+      return false
+    }
+  }
 }

packages/api/src/utils/helpers.ts

@@ -10,6 +10,7 @@ import { Highlight as HighlightData } from '../entity/highlight'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { Recommendation as RecommendationData } from '../entity/recommendation'
 import { RegistrationType, User } from '../entity/user'
+import { env } from '../env'
 import {
   Article,
   ArticleSavingRequest,
@@ -400,3 +401,6 @@ export const setRecentlySavedItemInRedis = async (
     })
   }
 }
+
+export const highlightUrl = (slug: string, highlightId: string): string =>
+  `${env.client.url}/me/${slug}#${highlightId}`