fix: readwise export not working for PAGE and LABEL events

packages/api/src/entity/integration.ts

@@ -30,6 +30,9 @@ export class Integration {
   @JoinColumn({ name: 'user_id' })
   user!: User
 
+  @Column('uuid', { name: 'user_id' })
+  userId!: string
+
   @Column('varchar', { length: 40 })
   name!: string
 

packages/api/src/services/highlights.ts

@@ -72,10 +72,6 @@ export const createHighlight = async (
       const newHighlight = await repo.createAndSave(highlight)
       return repo.findOneOrFail({
         where: { id: newHighlight.id },
-        relations: {
-          user: true,
-          libraryItem: true,
-        },
       })
     },
     {
@@ -89,11 +85,7 @@ export const createHighlight = async (
     {
       id: libraryItemId,
       highlights: [data],
-      // for Readwise
-      originalUrl: newHighlight.libraryItem.originalUrl,
-      title: newHighlight.libraryItem.title,
-      author: newHighlight.libraryItem.author,
-      thumbnail: newHighlight.libraryItem.thumbnail,
+      updatedAt: new Date(),
     },
     userId
   )
@@ -133,10 +125,6 @@ export const mergeHighlights = async (
 
     return highlightRepo.findOneOrFail({
       where: { id: newHighlight.id },
-      relations: {
-        user: true,
-        libraryItem: true,
-      },
     })
   })
 
@@ -144,11 +132,8 @@ export const mergeHighlights = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      originalUrl: newHighlight.libraryItem.originalUrl,
-      title: newHighlight.libraryItem.title,
-      author: newHighlight.libraryItem.author,
-      thumbnail: newHighlight.libraryItem.thumbnail,
       highlights: [newHighlight],
+      updatedAt: new Date(),
     },
     userId
   )
@@ -173,10 +158,6 @@ export const updateHighlight = async (
 
     return highlightRepo.findOneOrFail({
       where: { id: highlightId },
-      relations: {
-        libraryItem: true,
-        user: true,
-      },
     })
   })
 
@@ -185,10 +166,6 @@ export const updateHighlight = async (
     EntityType.HIGHLIGHT,
     {
       id: libraryItemId,
-      originalUrl: updatedHighlight.libraryItem.originalUrl,
-      title: updatedHighlight.libraryItem.title,
-      author: updatedHighlight.libraryItem.author,
-      thumbnail: updatedHighlight.libraryItem.thumbnail,
       highlights: [
         {
           ...highlight,
@@ -219,9 +196,6 @@ export const deleteHighlightById = async (
       const highlightRepo = tx.withRepository(highlightRepository)
       const highlight = await highlightRepo.findOneOrFail({
         where: { id: highlightId },
-        relations: {
-          user: true,
-        },
       })
 
       await highlightRepo.delete(highlightId)

packages/api/src/services/integrations/index.ts

@@ -13,7 +13,7 @@ export const getIntegrationClient = (
 ): IntegrationClient => {
   switch (name.toLowerCase()) {
     case 'readwise':
-      return new ReadwiseClient(token)
+      return new ReadwiseClient(token, integrationData)
     case 'pocket':
       return new PocketClient(token)
     case 'notion':

packages/api/src/services/integrations/notion.ts

@@ -3,12 +3,11 @@ import { GetDatabaseResponse } from '@notionhq/client/build/src/api-endpoints'
 import axios from 'axios'
 import { HighlightType } from '../../entity/highlight'
 import { Integration } from '../../entity/integration'
-import { User } from '../../entity/user'
 import { env } from '../../env'
 import { Merge } from '../../util'
 import { logger } from '../../utils/logger'
 import { getHighlightUrl } from '../highlights'
-import { findLibraryItemById, getItemUrl, ItemEvent } from '../library_item'
+import { findLibraryItemsByIds, getItemUrl, ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
 type AnnotationColor =
@@ -179,38 +178,11 @@ export class NotionClient implements IntegrationClient {
     return Promise.resolve(env.notion.authUrl)
   }
 
-  private itemToNotionPage = async (
-    userId: string,
+  private itemToNotionPage = (
     item: ItemEvent,
     settings: Settings,
     lastSync?: Date | null
-  ): Promise<NotionPage> => {
-    if (!item.updatedAt) {
-      const libraryItem = await findLibraryItemById(item.id, userId, {
-        select: [
-          'originalUrl',
-          'title',
-          'author',
-          'thumbnail',
-          'updatedAt',
-          'siteIcon',
-          'savedAt',
-        ],
-      })
-
-      if (!libraryItem) {
-        throw new Error('Library item not found')
-      }
-
-      item.originalUrl = libraryItem.originalUrl
-      item.title = libraryItem.title
-      item.author = libraryItem.author
-      item.thumbnail = libraryItem.thumbnail
-      item.updatedAt = libraryItem.updatedAt
-      item.siteIcon = libraryItem.siteIcon
-      item.savedAt = libraryItem.savedAt
-    }
-
+  ): NotionPage => {
     return {
       parent: {
         database_id: settings.parentDatabaseId,
@@ -367,13 +339,45 @@ export class NotionClient implements IntegrationClient {
       return false
     }
 
-    const userId = this.integrationData.user.id
+    const userId = this.integrationData.userId
+
+    // fetch the original url if not found
+    if (!items[0].originalUrl) {
+      const libraryItems = await findLibraryItemsByIds(
+        items.map((item) => item.id),
+        userId,
+        {
+          select: [
+            'id',
+            'originalUrl',
+            'title',
+            'author',
+            'thumbnail',
+            'siteIcon',
+            'savedAt',
+          ],
+        }
+      )
+
+      items.forEach((item) => {
+        const libraryItem = libraryItems.find((li) => li.id === item.id)
+        if (!libraryItem) {
+          return
+        }
+
+        item.originalUrl = libraryItem.originalUrl
+        item.title = libraryItem.title
+        item.author = libraryItem.author
+        item.thumbnail = libraryItem.thumbnail
+        item.siteIcon = libraryItem.siteIcon
+        item.savedAt = libraryItem.savedAt
+      })
+    }
 
     await Promise.all(
       items.map(async (item) => {
         try {
-          const notionPage = await this.itemToNotionPage(
-            userId,
+          const notionPage = this.itemToNotionPage(
             item,
             settings,
             this.integrationData?.syncedAt

packages/api/src/services/integrations/readwise.ts

@@ -1,8 +1,9 @@
 import axios from 'axios'
 import { HighlightType } from '../../entity/highlight'
+import { Integration } from '../../entity/integration'
 import { logger } from '../../utils/logger'
 import { getHighlightUrl } from '../highlights'
-import { getItemUrl, ItemEvent } from '../library_item'
+import { findLibraryItemsByIds, getItemUrl, ItemEvent } from '../library_item'
 import { IntegrationClient } from './integration'
 
 interface ReadwiseHighlight {
@@ -43,9 +44,11 @@ export class ReadwiseClient implements IntegrationClient {
     baseURL: 'https://readwise.io/api/v2',
     timeout: 5000, // 5 seconds
   })
+  private integrationData?: Integration
 
-  constructor(token: string) {
+  constructor(token: string, integration?: Integration) {
     this.token = token
+    this.integrationData = integration
   }
 
   accessToken = async (): Promise<string | null> => {
@@ -68,10 +71,42 @@ export class ReadwiseClient implements IntegrationClient {
   }
 
   export = async (items: ItemEvent[]): Promise<boolean> => {
-    let result = true
+    if (!this.integrationData) {
+      logger.error('Integration data is missing')
+      return false
+    }
+
+    if (
+      items.every((item) => !item.highlights || item.highlights.length === 0)
+    ) {
+      return false
+    }
+
+    const userId = this.integrationData.userId
+    const libraryItems = await findLibraryItemsByIds(
+      items.map((item) => item.id),
+      userId,
+      {
+        select: ['id', 'title', 'author', 'thumbnail', 'siteName'],
+      }
+    )
+    console.log(libraryItems)
+
+    items.forEach((item) => {
+      const libraryItem = libraryItems.find((li) => li.id === item.id)
+      if (!libraryItem) {
+        return
+      }
+
+      item.title = libraryItem.title
+      item.author = libraryItem.author
+      item.thumbnail = libraryItem.thumbnail
+      item.siteName = libraryItem.siteName
+    })
 
     const highlights = items.flatMap(this._itemToReadwiseHighlight)
 
+    let result = true
     // If there are no highlights, we will skip the sync
     if (highlights.length > 0) {
       result = await this._syncWithReadwise(highlights)

packages/api/src/services/library_item.ts

@@ -794,6 +794,7 @@ export const findLibraryItemsByIds = async (
   userId?: string,
   options?: {
     select?: (keyof LibraryItem)[]
+    relations?: Array<'labels' | 'highlights'>
   }
 ) => {
   const selectColumns =
@@ -802,12 +803,20 @@ export const findLibraryItemsByIds = async (
       .filter((column) => column !== 'originalContent')
       .map((column) => `library_item.${column}`)
   return authTrx(
-    async (tx) =>
-      tx
+    async (tx) => {
+      const qb = tx
         .createQueryBuilder(LibraryItem, 'library_item')
         .select(selectColumns)
         .where('library_item.id IN (:...ids)', { ids })
-        .getMany(),
+
+      if (options?.relations) {
+        options.relations.forEach((relation) => {
+          qb.leftJoinAndSelect(`library_item.${relation}`, relation)
+        })
+      }
+
+      return qb.getMany()
+    },
     {
       uid: userId,
       replicationMode: 'replica',