From 1445dee93321b9c0d75d2f4c1ef80437d4bdf886 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Apr 2024 22:20:56 +0800 Subject: [PATCH 01/15] batching labels --- Makefile | 12 ++++++++ packages/api/package.json | 2 +- .../api/src/resolvers/function_resolvers.ts | 8 ++--- packages/api/src/services/labels.ts | 30 +++++++++++-------- yarn.lock | 7 ++++- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 1688ff4af..aec0605b2 100644 --- a/Makefile +++ b/Makefile @@ -20,3 +20,15 @@ webview_gen: yarn workspace @omnivore/appreader build cp packages/appreader/build/bundle.js apple/OmnivoreKit/Sources/Views/Resources/bundle.js cp packages/appreader/build/bundle.js android/Omnivore/app/src/main/assets/bundle.js + +api: + yarn workspace @omnivore/api dev + +web: + yarn workspace @omnivore/web dev + +qp: + yarn workspace @omnivore/api dev_qp + +content: + yarn workspace @omnivore/content_fetch start diff --git a/packages/api/package.json b/packages/api/package.json index cea023808..554e99f45 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -56,7 +56,7 @@ "cookie-parser": "^1.4.5", "cors": "^2.8.5", "csv-stringify": "^6.4.0", - "dataloader": "^2.0.0", + "dataloader": "^2.2.2", "diff-match-patch": "^1.0.5", "dompurify": "^2.0.17", "dot-case": "^3.0.4", diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index d65d054c8..a3a0d7f2a 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -25,7 +25,7 @@ import { import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' import { findHighlightsByLibraryItemId } from '../services/highlights' -import { findLabelsByLibraryItemId } from '../services/labels' +import { labelsLoader } from '../services/labels' import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' import { @@ -439,7 +439,7 @@ export const functionResolvers = { ) { if (article.labels) return article.labels - return findLabelsByLibraryItemId(article.id, ctx.uid) + return labelsLoader.load(article.id) }, ...readingProgressHandlers, }, @@ -505,13 +505,13 @@ export const functionResolvers = { return item.siteIcon }, async labels( - item: { id: string; labels?: Label[] }, + item: { id: string; labels?: Label[]; labelNames?: string[] }, _: unknown, ctx: WithDataSourcesContext ) { if (item.labels) return item.labels - return findLabelsByLibraryItemId(item.id, ctx.uid) + return labelsLoader.load(item.id) }, async recommendations( item: { diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 21b66fa4c..2d0d5c0af 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -1,7 +1,9 @@ +import DataLoader from 'dataloader' import { DeepPartial, FindOptionsWhere, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel, LabelSource } from '../entity/entity_label' import { Label } from '../entity/label' +import { LibraryItem } from '../entity/library_item' import { createPubSubClient, EntityEvent, @@ -22,20 +24,24 @@ export type LabelEvent = Merge< EntityEvent > -// const batchGetLabelsFromLinkIds = async ( -// linkIds: readonly string[] -// ): Promise => { -// const links = await getRepository(Link).find({ -// where: { id: In(linkIds as string[]) }, -// relations: ['labels'], -// }) +const batchGetLabelsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const libraryItems = await authTrx((tx) => + tx.getRepository(LibraryItem).find({ + where: { id: In(libraryItemIds as string[]) }, + relations: ['labels'], + }) + ) -// return linkIds.map( -// (linkId) => links.find((link) => link.id === linkId)?.labels || [] -// ) -// } + return libraryItemIds.map( + (libraryItemId) => + libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) + ?.labels || [] + ) +} -// export const labelsLoader = new DataLoader(batchGetLabelsFromLinkIds) +export const labelsLoader = new DataLoader(batchGetLabelsFromLibraryItemIds) export const findOrCreateLabels = async ( labels: CreateLabelInput[], diff --git a/yarn.lock b/yarn.lock index 9c651f55f..08b76b456 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13348,11 +13348,16 @@ data-urls@^2.0.0: whatwg-mimetype "^2.3.0" whatwg-url "^8.0.0" -dataloader@2.1.0, dataloader@^2.0.0: +dataloader@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/dataloader/-/dataloader-2.1.0.tgz#c69c538235e85e7ac6c6c444bae8ecabf5de9df7" integrity sha512-qTcEYLen3r7ojZNgVUaRggOI+KM7jrKxXeSHhogh/TWxYMeONEMqY+hmkobiYQozsGIyg9OYVzO4ZIfoB4I0pQ== +dataloader@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/dataloader/-/dataloader-2.2.2.tgz#216dc509b5abe39d43a9b9d97e6e5e473dfbe3e0" + integrity sha512-8YnDaaf7N3k/q5HnTJVuzSyLETjoZjVmHc4AeKAzOvKHEFQKcn64OKBfzHYtE9zGjctNM7V9I0MfnUVLpi7M5g== + date-fns@2.x: version "2.29.3" resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.3.tgz#27402d2fc67eb442b511b70bbdf98e6411cd68a8" From 2904c321e9a5ed2773ce52adb4d067f9bfe3fe8f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Apr 2024 23:03:08 +0800 Subject: [PATCH 02/15] batching highlights --- packages/api/src/resolvers/article/index.ts | 63 ++++++++++--------- .../api/src/resolvers/function_resolvers.ts | 28 +++++++-- packages/api/src/services/highlights.ts | 29 ++++++++- 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index f2fbe9c3a..a5c80caa8 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -64,7 +64,6 @@ import { libraryItemRepository } from '../../repository/library_item' import { userRepository } from '../../repository/user' import { clearCachedReadingPosition } from '../../services/cached_reading_position' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, createAndSaveLabelsInLibraryItem, @@ -104,7 +103,6 @@ import { userDataToUser, } from '../../utils/helpers' import { - contentConverter, getDistillerResult, htmlToMarkdown, ParsedContentPuppeteer, @@ -667,7 +665,7 @@ export const searchResolver = authorized< SearchSuccess, SearchError, QuerySearchArgs ->(async (_obj, params, { log, uid }) => { +>(async (_obj, params, { uid }) => { const startCursor = params.after || '' const first = Math.min(params.first || 10, 100) // limit to 100 items @@ -699,38 +697,41 @@ export const searchResolver = authorized< libraryItems.pop() } - const edges = await Promise.all( - libraryItems.map(async (libraryItem) => { - libraryItem.highlights = await findHighlightsByLibraryItemId( - libraryItem.id, - uid - ) + // const edges = await Promise.all( + // libraryItems.map(async (libraryItem) => { + // libraryItem.highlights = await findHighlightsByLibraryItemId( + // libraryItem.id, + // uid + // ) - if (params.includeContent && libraryItem.readableContent) { - // convert html to the requested format - const format = params.format || ArticleFormat.Html - try { - const converter = contentConverter(format) - if (converter) { - libraryItem.readableContent = converter( - libraryItem.readableContent, - libraryItem.highlights - ) - } - } catch (error) { - log.error('Error converting content', error) - } - } + // if (params.includeContent && libraryItem.readableContent) { + // // convert html to the requested format + // const format = params.format || ArticleFormat.Html + // try { + // const converter = contentConverter(format) + // if (converter) { + // libraryItem.readableContent = converter( + // libraryItem.readableContent, + // libraryItem.highlights + // ) + // } + // } catch (error) { + // log.error('Error converting content', error) + // } + // } - return { - node: libraryItemToSearchItem(libraryItem), - cursor: endCursor, - } - }) - ) + // return { + // node: libraryItemToSearchItem(libraryItem), + // cursor: endCursor, + // } + // }) + // ) return { - edges, + edges: libraryItems.map((item) => ({ + node: libraryItemToSearchItem(item), + cursor: endCursor, + })), pageInfo: { hasPreviousPage: false, startCursor, diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index a3a0d7f2a..2a6396474 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -24,7 +24,10 @@ import { } from '../generated/graphql' import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' -import { findHighlightsByLibraryItemId } from '../services/highlights' +import { + findHighlightsByLibraryItemId, + highlightsLoader, +} from '../services/highlights' import { labelsLoader } from '../services/labels' import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' @@ -433,13 +436,17 @@ export const functionResolvers = { return article.content ? wordsCount(article.content) : undefined }, async labels( - article: { id: string; labels?: Label[] }, + article: { id: string; labels?: Label[]; labelNames?: string[] }, _: unknown, ctx: WithDataSourcesContext ) { if (article.labels) return article.labels - return labelsLoader.load(article.id) + if (article.labelNames?.length) { + return labelsLoader.load(article.id) + } + + return [] }, ...readingProgressHandlers, }, @@ -511,7 +518,11 @@ export const functionResolvers = { ) { if (item.labels) return item.labels - return labelsLoader.load(item.id) + if (item.labelNames?.length) { + return labelsLoader.load(item.id) + } + + return [] }, async recommendations( item: { @@ -547,14 +558,19 @@ export const functionResolvers = { item: { id: string highlights?: Highlight[] + highlightAnnotations?: string[] }, _: unknown, ctx: WithDataSourcesContext ) { if (item.highlights) return item.highlights - const highlights = await findHighlightsByLibraryItemId(item.id, ctx.uid) - return highlights.map(highlightDataToHighlight) + if (item.highlightAnnotations?.length) { + const highlights = await highlightsLoader.load(item.id) + return highlights.map(highlightDataToHighlight) + } + + return [] }, ...readingProgressHandlers, }, diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 3893bdbc5..0b5c527d2 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -1,9 +1,11 @@ +import DataLoader from 'dataloader' import { diff_match_patch } from 'diff-match-patch' -import { DeepPartial } from 'typeorm' +import { DeepPartial, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' +import { LibraryItem } from '../entity/library_item' import { homePageURL } from '../env' import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { authTrx } from '../repository' @@ -20,6 +22,31 @@ export type HighlightEvent = Merge< EntityEvent > +const batchGetHighlightsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const libraryItems = await authTrx(async (tx) => + tx.getRepository(LibraryItem).find({ + where: { id: In(libraryItemIds as string[]) }, + relations: { + highlights: { + user: true, + }, + }, + }) + ) + + return libraryItemIds.map( + (libraryItemId) => + libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) + ?.highlights || [] + ) +} + +export const highlightsLoader = new DataLoader( + batchGetHighlightsFromLibraryItemIds +) + export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() const patches = dmp.patch_fromText(patch) From c7ecbd53f81d67bb7bf385705d44016d3a4b14d1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Apr 2024 23:26:55 +0800 Subject: [PATCH 03/15] put dataloaders in apollo context --- packages/api/src/apollo.ts | 7 +++++++ packages/api/src/resolvers/function_resolvers.ts | 11 +++-------- packages/api/src/resolvers/types.ts | 9 ++++++++- packages/api/src/services/highlights.ts | 6 +----- packages/api/src/services/labels.ts | 4 +--- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index 68e1c3645..26725b499 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -15,6 +15,7 @@ import { import { ApolloServer } from 'apollo-server-express' import { ExpressContext } from 'apollo-server-express/dist/ApolloServer' import { ApolloServerPlugin } from 'apollo-server-plugin-base' +import DataLoader from 'dataloader' import { Express } from 'express' import * as httpContext from 'express-http-context2' import type http from 'http' @@ -30,6 +31,8 @@ import { functionResolvers } from './resolvers/function_resolvers' import { ClaimsToSet, RequestContext, ResolverContext } from './resolvers/types' import ScalarResolvers from './scalars' import typeDefs from './schema' +import { batchGetHighlightsFromLibraryItemIds } from './services/highlights' +import { batchGetLabelsFromLibraryItemIds } from './services/labels' import { countDailyServiceUsage, createServiceUsage, @@ -100,6 +103,10 @@ const contextFunc: ContextFunction = async ({ dataSources: { readingProgress: new ReadingProgressDataSource(), }, + dataLoaders: { + labels: new DataLoader(batchGetLabelsFromLibraryItemIds), + highlights: new DataLoader(batchGetHighlightsFromLibraryItemIds), + }, } return ctx diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 2a6396474..e6b997332 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -24,11 +24,6 @@ import { } from '../generated/graphql' import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' -import { - findHighlightsByLibraryItemId, - highlightsLoader, -} from '../services/highlights' -import { labelsLoader } from '../services/labels' import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' import { @@ -443,7 +438,7 @@ export const functionResolvers = { if (article.labels) return article.labels if (article.labelNames?.length) { - return labelsLoader.load(article.id) + return ctx.dataLoaders.labels.load(article.id) } return [] @@ -519,7 +514,7 @@ export const functionResolvers = { if (item.labels) return item.labels if (item.labelNames?.length) { - return labelsLoader.load(item.id) + return ctx.dataLoaders.labels.load(item.id) } return [] @@ -566,7 +561,7 @@ export const functionResolvers = { if (item.highlights) return item.highlights if (item.highlightAnnotations?.length) { - const highlights = await highlightsLoader.load(item.id) + const highlights = await ctx.dataLoaders.highlights.load(item.id) return highlights.map(highlightDataToHighlight) } diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 6fc24405f..927dee9a1 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -1,11 +1,14 @@ /* eslint-disable @typescript-eslint/ban-types */ import { Span } from '@opentelemetry/api' import { Context as ApolloContext } from 'apollo-server-core' +import DataLoader from 'dataloader' import * as jwt from 'jsonwebtoken' import { EntityManager } from 'typeorm' import winston from 'winston' -import { PubsubClient } from '../pubsub' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' +import { Highlight } from '../entity/highlight' +import { Label } from '../entity/label' +import { PubsubClient } from '../pubsub' export interface Claims { uid: string @@ -41,6 +44,10 @@ export interface RequestContext { dataSources: { readingProgress: ReadingProgressDataSource } + dataLoaders: { + labels: DataLoader + highlights: DataLoader + } } export type ResolverContext = ApolloContext diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 0b5c527d2..5978ac676 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -22,7 +22,7 @@ export type HighlightEvent = Merge< EntityEvent > -const batchGetHighlightsFromLibraryItemIds = async ( +export const batchGetHighlightsFromLibraryItemIds = async ( libraryItemIds: readonly string[] ): Promise => { const libraryItems = await authTrx(async (tx) => @@ -43,10 +43,6 @@ const batchGetHighlightsFromLibraryItemIds = async ( ) } -export const highlightsLoader = new DataLoader( - batchGetHighlightsFromLibraryItemIds -) - export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() const patches = dmp.patch_fromText(patch) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 2d0d5c0af..5cbcbd372 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -24,7 +24,7 @@ export type LabelEvent = Merge< EntityEvent > -const batchGetLabelsFromLibraryItemIds = async ( +export const batchGetLabelsFromLibraryItemIds = async ( libraryItemIds: readonly string[] ): Promise => { const libraryItems = await authTrx((tx) => @@ -41,8 +41,6 @@ const batchGetLabelsFromLibraryItemIds = async ( ) } -export const labelsLoader = new DataLoader(batchGetLabelsFromLibraryItemIds) - export const findOrCreateLabels = async ( labels: CreateLabelInput[], userId: string From fbe816d2d2a9dad637502fc6c85ca40c7c0c714d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 11:07:41 +0800 Subject: [PATCH 04/15] convert content if requested --- packages/api/src/generated/graphql.ts | 2 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/resolvers/article/index.ts | 33 +------------- .../api/src/resolvers/function_resolvers.ts | 44 ++++++++++++++++++- packages/api/src/schema.ts | 1 + packages/api/src/utils/helpers.ts | 7 ++- packages/api/src/utils/parser.ts | 1 - 7 files changed, 55 insertions(+), 34 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index cd4b67b31..a632e26a8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2748,6 +2748,7 @@ export type SearchItem = { directionality?: Maybe; feedContent?: Maybe; folder: Scalars['String']; + format?: Maybe; highlights?: Maybe>; id: Scalars['ID']; image?: Maybe; @@ -6617,6 +6618,7 @@ export type SearchItemResolvers, ParentType, ContextType>; feedContent?: Resolver, ParentType, ContextType>; folder?: Resolver; + format?: Resolver, ParentType, ContextType>; highlights?: Resolver>, ParentType, ContextType>; id?: Resolver; image?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index e48fe3cff..815fc9502 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2102,6 +2102,7 @@ type SearchItem { directionality: DirectionalityType feedContent: String folder: String! + format: String highlights: [Highlight!] id: ID! image: String diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index a5c80caa8..942bae7be 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -88,6 +88,7 @@ import { setFileUploadComplete, } from '../../services/upload_file' import { traceAs } from '../../tracing' +import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' import { enqueueBulkAction } from '../../utils/createTask' @@ -697,39 +698,9 @@ export const searchResolver = authorized< libraryItems.pop() } - // const edges = await Promise.all( - // libraryItems.map(async (libraryItem) => { - // libraryItem.highlights = await findHighlightsByLibraryItemId( - // libraryItem.id, - // uid - // ) - - // if (params.includeContent && libraryItem.readableContent) { - // // convert html to the requested format - // const format = params.format || ArticleFormat.Html - // try { - // const converter = contentConverter(format) - // if (converter) { - // libraryItem.readableContent = converter( - // libraryItem.readableContent, - // libraryItem.highlights - // ) - // } - // } catch (error) { - // log.error('Error converting content', error) - // } - // } - - // return { - // node: libraryItemToSearchItem(libraryItem), - // cursor: endCursor, - // } - // }) - // ) - return { edges: libraryItems.map((item) => ({ - node: libraryItemToSearchItem(item), + node: libraryItemToSearchItem(item, params.format as ArticleFormat), cursor: endCursor, })), pageInfo: { diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index e6b997332..30cae9cc3 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -4,6 +4,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { createHmac } from 'crypto' +import { Highlight as HighlightEntity } from '../entity/highlight' import { EXISTING_NEWSLETTER_FOLDER, NewsletterEmail, @@ -34,11 +35,16 @@ import { wordsCount, } from '../utils/helpers' import { createImageProxyUrl } from '../utils/imageproxy' +import { contentConverter } from '../utils/parser' import { generateDownloadSignedUrl, generateUploadFilePathName, } from '../utils/uploads' -import { emptyTrashResolver, fetchContentResolver } from './article' +import { + ArticleFormat, + emptyTrashResolver, + fetchContentResolver, +} from './article' import { addDiscoverFeedResolver, deleteDiscoverArticleResolver, @@ -568,6 +574,42 @@ export const functionResolvers = { return [] }, ...readingProgressHandlers, + async content( + item: { + id: string + content?: string + highlightAnnotations?: string[] + format?: ArticleFormat + }, + _: unknown, + ctx: WithDataSourcesContext + ) { + // convert html to the requested format if requested + if (item.format && item.content) { + let highlights: HighlightEntity[] = [] + // load highlights if needed + if ( + item.format === 'highlightedMarkdown' && + item.highlightAnnotations?.length + ) { + highlights = await ctx.dataLoaders.highlights.load(item.id) + } + + try { + ctx.log.info(`Converting content to: ${item.format}`) + + // convert html to the requested format + const converter = contentConverter(item.format) + if (converter) { + return converter(item.content, highlights) + } + } catch (error) { + ctx.log.error('Error converting content', error) + } + } + + return item.content + }, }, Subscription: { newsletterEmail(subscription: Subscription) { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index acae6f520..8d112d509 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1657,6 +1657,7 @@ const schema = gql` folder: String! aiSummary: String directionality: DirectionalityType + format: String } type SearchItemEdge { diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 785e5d5ef..85abc5467 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -26,6 +26,7 @@ import { SearchItem, } from '../generated/graphql' import { createPubSubClient } from '../pubsub' +import { ArticleFormat } from '../resolvers' import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' @@ -230,7 +231,10 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ directionality: item.directionality as unknown as DirectionalityType, }) -export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ +export const libraryItemToSearchItem = ( + item: LibraryItem, + format?: ArticleFormat +): SearchItem => ({ ...item, url: item.originalUrl, state: item.state as unknown as ArticleSavingRequestStatus, @@ -247,6 +251,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ highlights: item.highlights?.map(highlightDataToHighlight), wordsCount: item.wordCount, directionality: item.directionality as unknown as DirectionalityType, + format, }) export const isParsingTimeout = (libraryItem: LibraryItem): boolean => { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 3122a72bd..e76a8ed15 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -647,7 +647,6 @@ export const contentConverter = ( return htmlToMarkdown case ArticleFormat.HighlightedMarkdown: return htmlToHighlightedMarkdown - case ArticleFormat.Html: default: return undefined } From 6493b8e2f6fe2886805bcf4532231030d3b1a82f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 11:17:52 +0800 Subject: [PATCH 05/15] batching recommendations --- packages/api/src/apollo.ts | 4 ++++ .../api/src/resolvers/function_resolvers.ts | 8 +++----- packages/api/src/resolvers/types.ts | 2 ++ packages/api/src/services/recommendation.ts | 19 ++++++++++++++++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index 26725b499..b53a47ff2 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -33,6 +33,7 @@ import ScalarResolvers from './scalars' import typeDefs from './schema' import { batchGetHighlightsFromLibraryItemIds } from './services/highlights' import { batchGetLabelsFromLibraryItemIds } from './services/labels' +import { batchGetRecommendationsFromLibraryItemIds } from './services/recommendation' import { countDailyServiceUsage, createServiceUsage, @@ -106,6 +107,9 @@ const contextFunc: ContextFunction = async ({ dataLoaders: { labels: new DataLoader(batchGetLabelsFromLibraryItemIds), highlights: new DataLoader(batchGetHighlightsFromLibraryItemIds), + recommendations: new DataLoader( + batchGetRecommendationsFromLibraryItemIds + ), }, } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 30cae9cc3..4177ed7cc 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -25,7 +25,6 @@ import { } from '../generated/graphql' import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' -import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' import { highlightDataToHighlight, @@ -536,10 +535,9 @@ export const functionResolvers = { ) { if (item.recommendations) return item.recommendations - if (item.recommenderNames && item.recommenderNames.length > 0) { - const recommendations = await findRecommendationsByLibraryItemId( - item.id, - ctx.uid + if (item.recommenderNames) { + const recommendations = await ctx.dataLoaders.recommendations.load( + item.id ) return recommendations.map(recommandationDataToRecommendation) } diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 927dee9a1..61ccf1d8d 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -8,6 +8,7 @@ import winston from 'winston' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' +import { Recommendation } from '../entity/recommendation' import { PubsubClient } from '../pubsub' export interface Claims { @@ -47,6 +48,7 @@ export interface RequestContext { dataLoaders: { labels: DataLoader highlights: DataLoader + recommendations: DataLoader } } diff --git a/packages/api/src/services/recommendation.ts b/packages/api/src/services/recommendation.ts index 72b6ff0bd..80f9efe30 100644 --- a/packages/api/src/services/recommendation.ts +++ b/packages/api/src/services/recommendation.ts @@ -1,5 +1,5 @@ import { nanoid } from 'nanoid' -import { DeepPartial } from 'typeorm' +import { DeepPartial, In } from 'typeorm' import { LibraryItem } from '../entity/library_item' import { Recommendation } from '../entity/recommendation' import { authTrx } from '../repository' @@ -12,6 +12,23 @@ import { updateLibraryItem, } from './library_item' +export const batchGetRecommendationsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const libraryItems = await authTrx(async (tx) => + tx.getRepository(LibraryItem).find({ + where: { id: In(libraryItemIds as string[]) }, + relations: ['recommendations'], + }) + ) + + return libraryItemIds.map( + (libraryItemId) => + libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) + ?.recommendations || [] + ) +} + export const addRecommendation = async ( item: LibraryItem, recommendation: Recommendation, From 364f8f102f097e273ec665c104eb0b23903581c3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 11:24:07 +0800 Subject: [PATCH 06/15] use enum --- packages/api/src/resolvers/function_resolvers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 4177ed7cc..2c1e42343 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -583,11 +583,11 @@ export const functionResolvers = { ctx: WithDataSourcesContext ) { // convert html to the requested format if requested - if (item.format && item.content) { + if (item.format && item.format !== ArticleFormat.Html && item.content) { let highlights: HighlightEntity[] = [] // load highlights if needed if ( - item.format === 'highlightedMarkdown' && + item.format === ArticleFormat.HighlightedMarkdown && item.highlightAnnotations?.length ) { highlights = await ctx.dataLoaders.highlights.load(item.id) From 2231f0394817234b459a0dc5b0a56295d1b65a50 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 11:36:16 +0800 Subject: [PATCH 07/15] reduce joins --- packages/api/src/services/highlights.ts | 20 ++++++-------------- packages/api/src/services/labels.ts | 18 ++++++++---------- packages/api/src/services/recommendation.ts | 16 ++++++++-------- 3 files changed, 22 insertions(+), 32 deletions(-) diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 5978ac676..dd6685362 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -1,11 +1,9 @@ -import DataLoader from 'dataloader' import { diff_match_patch } from 'diff-match-patch' import { DeepPartial, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' -import { LibraryItem } from '../entity/library_item' import { homePageURL } from '../env' import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { authTrx } from '../repository' @@ -25,21 +23,15 @@ export type HighlightEvent = Merge< export const batchGetHighlightsFromLibraryItemIds = async ( libraryItemIds: readonly string[] ): Promise => { - const libraryItems = await authTrx(async (tx) => - tx.getRepository(LibraryItem).find({ - where: { id: In(libraryItemIds as string[]) }, - relations: { - highlights: { - user: true, - }, - }, + const highlights = await authTrx(async (tx) => + tx.getRepository(Highlight).find({ + where: { libraryItem: { id: In(libraryItemIds as string[]) } }, + relations: ['user'], }) ) - return libraryItemIds.map( - (libraryItemId) => - libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) - ?.highlights || [] + return libraryItemIds.map((libraryItemId) => + highlights.filter((highlight) => highlight.libraryItemId === libraryItemId) ) } diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 5cbcbd372..e3400520e 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -1,9 +1,7 @@ -import DataLoader from 'dataloader' import { DeepPartial, FindOptionsWhere, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel, LabelSource } from '../entity/entity_label' import { Label } from '../entity/label' -import { LibraryItem } from '../entity/library_item' import { createPubSubClient, EntityEvent, @@ -27,17 +25,17 @@ export type LabelEvent = Merge< export const batchGetLabelsFromLibraryItemIds = async ( libraryItemIds: readonly string[] ): Promise => { - const libraryItems = await authTrx((tx) => - tx.getRepository(LibraryItem).find({ - where: { id: In(libraryItemIds as string[]) }, - relations: ['labels'], + const labels = await authTrx(async (tx) => + tx.getRepository(EntityLabel).find({ + where: { libraryItemId: In(libraryItemIds as string[]) }, + relations: ['label'], }) ) - return libraryItemIds.map( - (libraryItemId) => - libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) - ?.labels || [] + return libraryItemIds.map((libraryItemId) => + labels + .filter((label) => label.libraryItemId === libraryItemId) + .map((label) => label.label) ) } diff --git a/packages/api/src/services/recommendation.ts b/packages/api/src/services/recommendation.ts index 80f9efe30..a098e7e25 100644 --- a/packages/api/src/services/recommendation.ts +++ b/packages/api/src/services/recommendation.ts @@ -15,17 +15,17 @@ import { export const batchGetRecommendationsFromLibraryItemIds = async ( libraryItemIds: readonly string[] ): Promise => { - const libraryItems = await authTrx(async (tx) => - tx.getRepository(LibraryItem).find({ - where: { id: In(libraryItemIds as string[]) }, - relations: ['recommendations'], + const recommendations = await authTrx(async (tx) => + tx.getRepository(Recommendation).find({ + where: { libraryItem: { id: In(libraryItemIds as string[]) } }, + relations: ['group', 'recommender'], }) ) - return libraryItemIds.map( - (libraryItemId) => - libraryItems.find((libraryItem) => libraryItem.id === libraryItemId) - ?.recommendations || [] + return libraryItemIds.map((libraryItemId) => + recommendations.filter( + (recommendation) => recommendation.libraryItem.id === libraryItemId + ) ) } From 66d764962000e04a39ce41e81e94a58376c23b02 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 11:54:55 +0800 Subject: [PATCH 08/15] fix tests --- packages/api/src/entity/recommendation.ts | 3 ++ .../api/src/resolvers/function_resolvers.ts | 38 +++++-------------- packages/api/src/services/recommendation.ts | 2 +- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/packages/api/src/entity/recommendation.ts b/packages/api/src/entity/recommendation.ts index e4007a515..0b0d2ae13 100644 --- a/packages/api/src/entity/recommendation.ts +++ b/packages/api/src/entity/recommendation.ts @@ -23,6 +23,9 @@ export class Recommendation { @JoinColumn({ name: 'library_item_id' }) libraryItem!: LibraryItem + @Column('uuid') + libraryItemId!: string + @ManyToOne(() => Group, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'group_id' }) group!: Group diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 2c1e42343..f9c10b516 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -436,17 +436,13 @@ export const functionResolvers = { return article.content ? wordsCount(article.content) : undefined }, async labels( - article: { id: string; labels?: Label[]; labelNames?: string[] }, + article: { id: string; labels?: Label[] }, _: unknown, ctx: WithDataSourcesContext ) { if (article.labels) return article.labels - if (article.labelNames?.length) { - return ctx.dataLoaders.labels.load(article.id) - } - - return [] + return ctx.dataLoaders.labels.load(article.id) }, ...readingProgressHandlers, }, @@ -512,37 +508,28 @@ export const functionResolvers = { return item.siteIcon }, async labels( - item: { id: string; labels?: Label[]; labelNames?: string[] }, + item: { id: string; labels?: Label[] }, _: unknown, ctx: WithDataSourcesContext ) { if (item.labels) return item.labels - if (item.labelNames?.length) { - return ctx.dataLoaders.labels.load(item.id) - } - - return [] + return ctx.dataLoaders.labels.load(item.id) }, async recommendations( item: { id: string recommendations?: Recommendation[] - recommenderNames?: string[] | null }, _: unknown, ctx: WithDataSourcesContext ) { if (item.recommendations) return item.recommendations - if (item.recommenderNames) { - const recommendations = await ctx.dataLoaders.recommendations.load( - item.id - ) - return recommendations.map(recommandationDataToRecommendation) - } - - return [] + const recommendations = await ctx.dataLoaders.recommendations.load( + item.id + ) + return recommendations.map(recommandationDataToRecommendation) }, async aiSummary(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { return ( @@ -557,19 +544,14 @@ export const functionResolvers = { item: { id: string highlights?: Highlight[] - highlightAnnotations?: string[] }, _: unknown, ctx: WithDataSourcesContext ) { if (item.highlights) return item.highlights - if (item.highlightAnnotations?.length) { - const highlights = await ctx.dataLoaders.highlights.load(item.id) - return highlights.map(highlightDataToHighlight) - } - - return [] + const highlights = await ctx.dataLoaders.highlights.load(item.id) + return highlights.map(highlightDataToHighlight) }, ...readingProgressHandlers, async content( diff --git a/packages/api/src/services/recommendation.ts b/packages/api/src/services/recommendation.ts index a098e7e25..9e58224f4 100644 --- a/packages/api/src/services/recommendation.ts +++ b/packages/api/src/services/recommendation.ts @@ -24,7 +24,7 @@ export const batchGetRecommendationsFromLibraryItemIds = async ( return libraryItemIds.map((libraryItemId) => recommendations.filter( - (recommendation) => recommendation.libraryItem.id === libraryItemId + (recommendation) => recommendation.libraryItemId === libraryItemId ) ) } From 01246207cba9adf33ddda21fee133f98c92c4ad5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 12:06:20 +0800 Subject: [PATCH 09/15] fix logger --- packages/api/src/pubsub.ts | 4 +--- packages/api/src/utils/logger.ts | 2 +- packages/api/src/utils/parser.ts | 7 +++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 4cb0116d4..fea9139b7 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -7,13 +7,11 @@ import { enqueueProcessYouTubeVideo, enqueueTriggerRuleJob, } from './utils/createTask' -import { buildLogger } from './utils/logger' +import { logger } from './utils/logger' import { isYouTubeVideoURL } from './utils/youtube' export type EntityEvent = { id: string } -const logger = buildLogger('pubsub') - const client = new PubSub() export const createPubSubClient = (): PubsubClient => { diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 364cfffa3..8de7659ac 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -31,7 +31,7 @@ export class CustomTypeOrmLogger constructor(options?: TypeOrmLoggerOptions) { super(options) - this.logger = buildLogger('typeorm') + this.logger = logger } logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index e76a8ed15..06241d726 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -33,7 +33,7 @@ import { makeHighlightNodeAttributes, } from './highlightGenerator' import { createImageProxyUrl } from './imageproxy' -import { buildLogger, LogRecord } from './logger' +import { logger, LogRecord } from './logger' interface Feed { title: string @@ -43,7 +43,6 @@ interface Feed { description?: string } -const logger = buildLogger('utils.parse') const signToken = promisify(jwt.sign) const axiosInstance = axios.create({ @@ -670,7 +669,7 @@ export const htmlToHighlightedMarkdown = ( throw new Error('Invalid html content') } } catch (err) { - logger.info(err) + logger.error(err) return nhm.translate(/* html */ html) } @@ -690,7 +689,7 @@ export const htmlToHighlightedMarkdown = ( articleTextNodes ) } catch (err) { - logger.info(err) + logger.error(err) } }) html = document.documentElement.outerHTML From 342db3928cc246c96374b71cb0dc5f8da5daf6e6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 14:44:17 +0800 Subject: [PATCH 10/15] batching upload_files --- packages/api/src/apollo.ts | 2 ++ packages/api/src/resolvers/function_resolvers.ts | 7 ++++--- packages/api/src/resolvers/types.ts | 2 ++ packages/api/src/services/upload_file.ts | 13 ++++++++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index b53a47ff2..11008c18b 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -38,6 +38,7 @@ import { countDailyServiceUsage, createServiceUsage, } from './services/service_usage' +import { batchGetUploadFilesByIds } from './services/upload_file' import { tracer } from './tracing' import { getClaimsByToken, setAuthInCookie } from './utils/auth' import { SetClaimsRole } from './utils/dictionary' @@ -110,6 +111,7 @@ const contextFunc: ContextFunction = async ({ recommendations: new DataLoader( batchGetRecommendationsFromLibraryItemIds ), + uploadFiles: new DataLoader(batchGetUploadFilesByIds), }, } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index f9c10b516..b4ace7f7b 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -25,7 +25,6 @@ import { } from '../generated/graphql' import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' -import { findUploadFileById } from '../services/upload_file' import { highlightDataToHighlight, isBase64Image, @@ -407,7 +406,9 @@ export const functionResolvers = { ctx.claims && article.uploadFileId ) { - const upload = await findUploadFileById(article.uploadFileId) + const upload = await ctx.dataLoaders.uploadFiles.load( + article.uploadFileId + ) if (!upload || !upload.fileName) { return undefined } @@ -481,7 +482,7 @@ export const functionResolvers = { ctx.claims && item.uploadFileId ) { - const upload = await findUploadFileById(item.uploadFileId) + const upload = await ctx.dataLoaders.uploadFiles.load(item.uploadFileId) if (!upload || !upload.fileName) { return undefined } diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 61ccf1d8d..f880a0145 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -9,6 +9,7 @@ import { ReadingProgressDataSource } from '../datasources/reading_progress_data_ import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { Recommendation } from '../entity/recommendation' +import { UploadFile } from '../entity/upload_file' import { PubsubClient } from '../pubsub' export interface Claims { @@ -49,6 +50,7 @@ export interface RequestContext { labels: DataLoader highlights: DataLoader recommendations: DataLoader + uploadFiles: DataLoader } } diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index 148614352..fd96f313c 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -1,5 +1,7 @@ import normalizeUrl from 'normalize-url' import path from 'path' +import { In } from 'typeorm' +import { v4 as uuid } from 'uuid' import { LibraryItemState } from '../entity/library_item' import { UploadFile } from '../entity/upload_file' import { @@ -18,7 +20,16 @@ import { } from '../utils/uploads' import { validateUrl } from './create_page_save_request' import { createOrUpdateLibraryItem } from './library_item' -import { v4 as uuid } from 'uuid' + +export const batchGetUploadFilesByIds = async ( + ids: readonly string[] +): Promise<(UploadFile | undefined)[]> => { + const uploadFiles = await getRepository(UploadFile).findBy({ + id: In(ids as string[]), + }) + + return ids.map((id) => uploadFiles.find((uploadFile) => uploadFile.id === id)) +} const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) From 89d6564d74fd635420bdbb4299f4aea9066da1b6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Apr 2024 15:18:34 +0800 Subject: [PATCH 11/15] increase log metadata size to 10000 characters --- packages/api/src/utils/logger.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 8de7659ac..50606a069 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -132,8 +132,8 @@ const truncateObjectDeep = (object: any, length: number): any => { class GcpLoggingTransport extends LoggingWinston { log(info: any, callback: (err: Error | null, apiResponse?: any) => void) { - // reduce the size of the log entry by truncating any string values to 500 characters - info = truncateObjectDeep(info, 500) as never + // reduce the size of the log entry by truncating any string values to 10000 characters + info = truncateObjectDeep(info, 10000) as never super.log(info, callback) } } From 7ef6973a476ef62f47100e62a015eb3785339c3d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sun, 28 Apr 2024 14:25:11 +0800 Subject: [PATCH 12/15] load labels when getting item --- packages/api/src/entity/library_item.ts | 1 + packages/api/src/resolvers/article/index.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index fb92fa618..ef2c339f4 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -9,6 +9,7 @@ import { OneToMany, OneToOne, PrimaryGeneratedColumn, + Unique, UpdateDateColumn, } from 'typeorm' import { Highlight } from './highlight' diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 942bae7be..451e36a0a 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -88,7 +88,6 @@ import { setFileUploadComplete, } from '../../services/upload_file' import { traceAs } from '../../tracing' -import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' import { enqueueBulkAction } from '../../utils/createTask' @@ -393,10 +392,12 @@ export const getArticleResolver = authorized< tx.withRepository(libraryItemRepository).findOne({ select: selectColumns, where: { + user: { id: uid }, ...where, deletedAt: IsNull(), }, relations: { + labels: true, highlights: { user: true, labels: true, From a78d438e2106bd52783e0176db730dd6528dd460 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sun, 28 Apr 2024 15:58:55 +0800 Subject: [PATCH 13/15] fix: findOne with relations will execute query twice --- packages/api/src/entity/profile.ts | 1 - packages/api/src/repository/user.ts | 8 ++- packages/api/src/resolvers/article/index.ts | 57 +++++++++++---------- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/api/src/entity/profile.ts b/packages/api/src/entity/profile.ts index 9fdc96b76..a1a0372ea 100644 --- a/packages/api/src/entity/profile.ts +++ b/packages/api/src/entity/profile.ts @@ -7,7 +7,6 @@ import { PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' - import { User } from './user' @Entity({ name: 'user_profile' }) diff --git a/packages/api/src/repository/user.ts b/packages/api/src/repository/user.ts index a8f327b0d..d8592222a 100644 --- a/packages/api/src/repository/user.ts +++ b/packages/api/src/repository/user.ts @@ -16,7 +16,13 @@ export const MAX_RECORDS_LIMIT = 1000 export const userRepository = appDataSource.getRepository(User).extend({ findById(id: string) { - return this.findOneBy({ id, status: StatusType.Active }) + return this.createQueryBuilder('user') + .leftJoinAndSelect('user.profile', 'profile') + .where('user.id = :id AND user.status = :status', { + id, + status: StatusType.Active, + }) + .getOne() }, findByEmail(email: string) { diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 451e36a0a..452b4d00b 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -5,7 +5,6 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ import { Readability } from '@omnivore/readability' import graphqlFields from 'graphql-fields' -import { IsNull } from 'typeorm' import { LibraryItem, LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { @@ -383,33 +382,35 @@ export const getArticleResolver = authorized< if (!includeOriginalHtml) { selectColumns.splice(selectColumns.indexOf('originalContent'), 1) } - // We allow the backend to use the ID instead of a slug to fetch the article - // query against id if slug is a uuid - const where = slug.match(/^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/i) - ? { id: slug } - : { slug } - const libraryItem = await authTrx((tx) => - tx.withRepository(libraryItemRepository).findOne({ - select: selectColumns, - where: { - user: { id: uid }, - ...where, - deletedAt: IsNull(), - }, - relations: { - labels: true, - highlights: { - user: true, - labels: true, - }, - uploadFile: true, - recommendations: { - recommender: true, - group: true, - }, - }, - }) - ) + + const libraryItem = await authTrx((tx) => { + const qb = tx + .createQueryBuilder(LibraryItem, 'libraryItem') + .select(selectColumns.map((column) => `libraryItem.${column}`)) + .where('libraryItem.user_id = :uid', { uid }) + + // We allow the backend to use the ID instead of a slug to fetch the article + // query against id if slug is a uuid + slug.match(/^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/i) + ? qb.andWhere('libraryItem.id = :id', { id: slug }) + : qb.andWhere('libraryItem.slug = :slug', { slug }) + + return qb + .andWhere('libraryItem.deleted_at IS NULL') + .leftJoinAndSelect('libraryItem.labels', 'labels') + .leftJoinAndSelect('libraryItem.highlights', 'highlights') + .leftJoinAndSelect('highlights.labels', 'highlights_labels') + .leftJoinAndSelect('highlights.user', 'highlights_user') + .leftJoinAndSelect('highlights_user.profile', 'highlights_user_profile') + .leftJoinAndSelect('libraryItem.uploadFile', 'uploadFile') + .leftJoinAndSelect('libraryItem.recommendations', 'recommendations') + .leftJoinAndSelect('recommendations.group', 'recommendations_group') + .leftJoinAndSelect( + 'recommendations.recommender', + 'recommendations_recommender' + ) + .getOne() + }) if (!libraryItem) { return { errorCodes: [ArticleErrorCode.NotFound] } From ef2ff36f0e920fc67ae3ca84af40d5e774704373 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sun, 28 Apr 2024 21:51:30 +0800 Subject: [PATCH 14/15] fetch api key once --- packages/api/src/resolvers/article/index.ts | 20 +++++++++----------- packages/api/src/utils/auth.ts | 11 ++++++----- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 452b4d00b..874d235da 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -387,16 +387,6 @@ export const getArticleResolver = authorized< const qb = tx .createQueryBuilder(LibraryItem, 'libraryItem') .select(selectColumns.map((column) => `libraryItem.${column}`)) - .where('libraryItem.user_id = :uid', { uid }) - - // We allow the backend to use the ID instead of a slug to fetch the article - // query against id if slug is a uuid - slug.match(/^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/i) - ? qb.andWhere('libraryItem.id = :id', { id: slug }) - : qb.andWhere('libraryItem.slug = :slug', { slug }) - - return qb - .andWhere('libraryItem.deleted_at IS NULL') .leftJoinAndSelect('libraryItem.labels', 'labels') .leftJoinAndSelect('libraryItem.highlights', 'highlights') .leftJoinAndSelect('highlights.labels', 'highlights_labels') @@ -409,7 +399,15 @@ export const getArticleResolver = authorized< 'recommendations.recommender', 'recommendations_recommender' ) - .getOne() + .where('libraryItem.user_id = :uid', { uid }) + + // We allow the backend to use the ID instead of a slug to fetch the article + // query against id if slug is a uuid + slug.match(/^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/i) + ? qb.andWhere('libraryItem.id = :id', { id: slug }) + : qb.andWhere('libraryItem.slug = :slug', { slug }) + + return qb.andWhere('libraryItem.deleted_at IS NULL').getOne() }) if (!libraryItem) { diff --git a/packages/api/src/utils/auth.ts b/packages/api/src/utils/auth.ts index c3269aec8..2dbc56d1f 100644 --- a/packages/api/src/utils/auth.ts +++ b/packages/api/src/utils/auth.ts @@ -36,12 +36,13 @@ export const claimsFromApiKey = async (key: string): Promise => { const apiKeyRepo = getRepository(ApiKey) - const apiKey = await apiKeyRepo.findOne({ - where: { + const apiKey = await apiKeyRepo + .createQueryBuilder('apiKey') + .innerJoinAndSelect('apiKey.user', 'user') + .where({ key: hashedKey, - }, - relations: ['user'], - }) + }) + .getOne() if (!apiKey) { throw new Error('api key not found') } From 8549c16ceb56b0d2ffa4cbe7bbf22032bb8163f7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sun, 28 Apr 2024 21:55:09 +0800 Subject: [PATCH 15/15] fix import --- packages/api/src/entity/library_item.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index ef2c339f4..fb92fa618 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -9,7 +9,6 @@ import { OneToMany, OneToOne, PrimaryGeneratedColumn, - Unique, UpdateDateColumn, } from 'typeorm' import { Highlight } from './highlight'