batching highlights

packages/api/src/resolvers/article/index.ts

@@ -64,7 +64,6 @@ import { libraryItemRepository } from '../../repository/library_item'
 import { userRepository } from '../../repository/user'
 import { clearCachedReadingPosition } from '../../services/cached_reading_position'
 import { createPageSaveRequest } from '../../services/create_page_save_request'
-import { findHighlightsByLibraryItemId } from '../../services/highlights'
 import {
   addLabelsToLibraryItem,
   createAndSaveLabelsInLibraryItem,
@@ -104,7 +103,6 @@ import {
   userDataToUser,
 } from '../../utils/helpers'
 import {
-  contentConverter,
   getDistillerResult,
   htmlToMarkdown,
   ParsedContentPuppeteer,
@@ -667,7 +665,7 @@ export const searchResolver = authorized<
   SearchSuccess,
   SearchError,
   QuerySearchArgs
->(async (_obj, params, { log, uid }) => {
+>(async (_obj, params, { uid }) => {
   const startCursor = params.after || ''
   const first = Math.min(params.first || 10, 100) // limit to 100 items
 
@@ -699,38 +697,41 @@ export const searchResolver = authorized<
     libraryItems.pop()
   }
 
-  const edges = await Promise.all(
-    libraryItems.map(async (libraryItem) => {
-      libraryItem.highlights = await findHighlightsByLibraryItemId(
-        libraryItem.id,
-        uid
-      )
+  // const edges = await Promise.all(
+  //   libraryItems.map(async (libraryItem) => {
+  //     libraryItem.highlights = await findHighlightsByLibraryItemId(
+  //       libraryItem.id,
+  //       uid
+  //     )
 
-      if (params.includeContent && libraryItem.readableContent) {
-        // convert html to the requested format
-        const format = params.format || ArticleFormat.Html
-        try {
-          const converter = contentConverter(format)
-          if (converter) {
-            libraryItem.readableContent = converter(
-              libraryItem.readableContent,
-              libraryItem.highlights
-            )
-          }
-        } catch (error) {
-          log.error('Error converting content', error)
-        }
-      }
+  //     if (params.includeContent && libraryItem.readableContent) {
+  //       // convert html to the requested format
+  //       const format = params.format || ArticleFormat.Html
+  //       try {
+  //         const converter = contentConverter(format)
+  //         if (converter) {
+  //           libraryItem.readableContent = converter(
+  //             libraryItem.readableContent,
+  //             libraryItem.highlights
+  //           )
+  //         }
+  //       } catch (error) {
+  //         log.error('Error converting content', error)
+  //       }
+  //     }
 
-      return {
-        node: libraryItemToSearchItem(libraryItem),
-        cursor: endCursor,
-      }
-    })
-  )
+  //     return {
+  //       node: libraryItemToSearchItem(libraryItem),
+  //       cursor: endCursor,
+  //     }
+  //   })
+  // )
 
   return {
-    edges,
+    edges: libraryItems.map((item) => ({
+      node: libraryItemToSearchItem(item),
+      cursor: endCursor,
+    })),
     pageInfo: {
       hasPreviousPage: false,
       startCursor,

packages/api/src/resolvers/function_resolvers.ts

@@ -24,7 +24,10 @@ import {
 } from '../generated/graphql'
 import { getAISummary } from '../services/ai-summaries'
 import { findUserFeatures } from '../services/features'
-import { findHighlightsByLibraryItemId } from '../services/highlights'
+import {
+  findHighlightsByLibraryItemId,
+  highlightsLoader,
+} from '../services/highlights'
 import { labelsLoader } from '../services/labels'
 import { findRecommendationsByLibraryItemId } from '../services/recommendation'
 import { findUploadFileById } from '../services/upload_file'
@@ -433,13 +436,17 @@ export const functionResolvers = {
       return article.content ? wordsCount(article.content) : undefined
     },
     async labels(
-      article: { id: string; labels?: Label[] },
+      article: { id: string; labels?: Label[]; labelNames?: string[] },
       _: unknown,
       ctx: WithDataSourcesContext
     ) {
       if (article.labels) return article.labels
 
-      return labelsLoader.load(article.id)
+      if (article.labelNames?.length) {
+        return labelsLoader.load(article.id)
+      }
+
+      return []
     },
     ...readingProgressHandlers,
   },
@@ -511,7 +518,11 @@ export const functionResolvers = {
     ) {
       if (item.labels) return item.labels
 
-      return labelsLoader.load(item.id)
+      if (item.labelNames?.length) {
+        return labelsLoader.load(item.id)
+      }
+
+      return []
     },
     async recommendations(
       item: {
@@ -547,14 +558,19 @@ export const functionResolvers = {
       item: {
         id: string
         highlights?: Highlight[]
+        highlightAnnotations?: string[]
       },
       _: unknown,
       ctx: WithDataSourcesContext
     ) {
       if (item.highlights) return item.highlights
 
-      const highlights = await findHighlightsByLibraryItemId(item.id, ctx.uid)
-      return highlights.map(highlightDataToHighlight)
+      if (item.highlightAnnotations?.length) {
+        const highlights = await highlightsLoader.load(item.id)
+        return highlights.map(highlightDataToHighlight)
+      }
+
+      return []
     },
     ...readingProgressHandlers,
   },

packages/api/src/services/highlights.ts

@@ -1,9 +1,11 @@
+import DataLoader from 'dataloader'
 import { diff_match_patch } from 'diff-match-patch'
-import { DeepPartial } from 'typeorm'
+import { DeepPartial, In } from 'typeorm'
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
 import { EntityLabel } from '../entity/entity_label'
 import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
+import { LibraryItem } from '../entity/library_item'
 import { homePageURL } from '../env'
 import { createPubSubClient, EntityEvent, EntityType } from '../pubsub'
 import { authTrx } from '../repository'
@@ -20,6 +22,31 @@ export type HighlightEvent = Merge<
   EntityEvent
 >
 
+const batchGetHighlightsFromLibraryItemIds = async (
+  libraryItemIds: readonly string[]
+): Promise<Highlight[][]> => {
+  const libraryItems = await authTrx(async (tx) =>
+    tx.getRepository(LibraryItem).find({
+      where: { id: In(libraryItemIds as string[]) },
+      relations: {
+        highlights: {
+          user: true,
+        },
+      },
+    })
+  )
+
+  return libraryItemIds.map(
+    (libraryItemId) =>
+      libraryItems.find((libraryItem) => libraryItem.id === libraryItemId)
+        ?.highlights || []
+  )
+}
+
+export const highlightsLoader = new DataLoader(
+  batchGetHighlightsFromLibraryItemIds
+)
+
 export const getHighlightLocation = (patch: string): number | undefined => {
   const dmp = new diff_match_patch()
   const patches = dmp.patch_fromText(patch)