From 2904c321e9a5ed2773ce52adb4d067f9bfe3fe8f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Apr 2024 23:03:08 +0800 Subject: [PATCH] batching highlights --- packages/api/src/resolvers/article/index.ts | 63 ++++++++++--------- .../api/src/resolvers/function_resolvers.ts | 28 +++++++-- packages/api/src/services/highlights.ts | 29 ++++++++- 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index f2fbe9c3a..a5c80caa8 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -64,7 +64,6 @@ import { libraryItemRepository } from '../../repository/library_item' import { userRepository } from '../../repository/user' import { clearCachedReadingPosition } from '../../services/cached_reading_position' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, createAndSaveLabelsInLibraryItem, @@ -104,7 +103,6 @@ import { userDataToUser, } from '../../utils/helpers' import { - contentConverter, getDistillerResult, htmlToMarkdown, ParsedContentPuppeteer, @@ -667,7 +665,7 @@ export const searchResolver = authorized< SearchSuccess, SearchError, QuerySearchArgs ->(async (_obj, params, { log, uid }) => { +>(async (_obj, params, { uid }) => { const startCursor = params.after || '' const first = Math.min(params.first || 10, 100) // limit to 100 items @@ -699,38 +697,41 @@ export const searchResolver = authorized< libraryItems.pop() } - const edges = await Promise.all( - libraryItems.map(async (libraryItem) => { - libraryItem.highlights = await findHighlightsByLibraryItemId( - libraryItem.id, - uid - ) + // const edges = await Promise.all( + // libraryItems.map(async (libraryItem) => { + // libraryItem.highlights = await findHighlightsByLibraryItemId( + // libraryItem.id, + // uid + // ) - if (params.includeContent && libraryItem.readableContent) { - // convert html to the requested format - const format = params.format || ArticleFormat.Html - try { - const converter = contentConverter(format) - if (converter) { - libraryItem.readableContent = converter( - libraryItem.readableContent, - libraryItem.highlights - ) - } - } catch (error) { - log.error('Error converting content', error) - } - } + // if (params.includeContent && libraryItem.readableContent) { + // // convert html to the requested format + // const format = params.format || ArticleFormat.Html + // try { + // const converter = contentConverter(format) + // if (converter) { + // libraryItem.readableContent = converter( + // libraryItem.readableContent, + // libraryItem.highlights + // ) + // } + // } catch (error) { + // log.error('Error converting content', error) + // } + // } - return { - node: libraryItemToSearchItem(libraryItem), - cursor: endCursor, - } - }) - ) + // return { + // node: libraryItemToSearchItem(libraryItem), + // cursor: endCursor, + // } + // }) + // ) return { - edges, + edges: libraryItems.map((item) => ({ + node: libraryItemToSearchItem(item), + cursor: endCursor, + })), pageInfo: { hasPreviousPage: false, startCursor, diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index a3a0d7f2a..2a6396474 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -24,7 +24,10 @@ import { } from '../generated/graphql' import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' -import { findHighlightsByLibraryItemId } from '../services/highlights' +import { + findHighlightsByLibraryItemId, + highlightsLoader, +} from '../services/highlights' import { labelsLoader } from '../services/labels' import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' @@ -433,13 +436,17 @@ export const functionResolvers = { return article.content ? wordsCount(article.content) : undefined }, async labels( - article: { id: string; labels?: Label[] }, + article: { id: string; labels?: Label[]; labelNames?: string[] }, _: unknown, ctx: WithDataSourcesContext ) { if (article.labels) return article.labels - return labelsLoader.load(article.id) + if (article.labelNames?.length) { + return labelsLoader.load(article.id) + } + + return [] }, ...readingProgressHandlers, }, @@ -511,7 +518,11 @@ export const functionResolvers = { ) { if (item.labels) return item.labels - return labelsLoader.load(item.id) + if (item.labelNames?.length) { + return labelsLoader.load(item.id) + } + + return [] }, async recommendations( item: { @@ -547,14 +558,19 @@ export const functionResolvers = { item: { id: string highlights?: Highlight[] + highlightAnnotations?: string[] }, _: unknown, ctx: WithDataSourcesContext ) { if (item.highlights) return item.highlights - const highlights = await findHighlightsByLibraryItemId(item.id, ctx.uid) - return highlights.map(highlightDataToHighlight) + if (item.highlightAnnotations?.length) { + const highlights = await highlightsLoader.load(item.id) + return highlights.map(highlightDataToHighlight) + } + + return [] }, ...readingProgressHandlers, }, diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 3893bdbc5..0b5c527d2 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -1,9 +1,11 @@ +import DataLoader from 'dataloader' import { diff_match_patch } from 'diff-match-patch' -import { DeepPartial } from 'typeorm' +import { DeepPartial, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' +import { LibraryItem } from '../entity/library_item' import { homePageURL } from '../env' import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { authTrx } from '../repository' @@ -20,6 +22,31 @@ export type HighlightEvent = Merge< EntityEvent > +const batchGetHighlightsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const libraryItems = await authTrx(async (tx) => + tx.getRepository(LibraryItem).find({ + where: { id: In(libraryItemIds as string[]) }, + relations: { + highlights: { + user: true, + }, + }, + }) + ) + + return libraryItemIds.map( + (libraryItemId) => + libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) + ?.highlights || [] + ) +} + +export const highlightsLoader = new DataLoader( + batchGetHighlightsFromLibraryItemIds +) + export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() const patches = dmp.patch_fromText(patch)