From 32097de62f8eaa50dd837d462269ba4845ce891b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 15 Aug 2024 21:24:57 +0800
Subject: [PATCH] perf: reduce number of column returned by search query

---
 packages/api/src/entity/library_item.ts   | 20 ++++++++------------
 packages/api/src/pubsub.ts                | 21 ---------------------
 packages/api/src/services/library_item.ts |  5 ++++-
 3 files changed, 12 insertions(+), 34 deletions(-)

diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
index 46d056310..c646aa193 100644
--- a/packages/api/src/entity/library_item.ts
+++ b/packages/api/src/entity/library_item.ts
@@ -58,7 +58,7 @@ export class LibraryItem {
   @Column('text')
   originalUrl!: string
 
-  @Column('text', { nullable: true })
+  @Column('text', { nullable: true, select: false })
   downloadUrl?: string | null
 
   @Column('text')
@@ -138,10 +138,6 @@ export class LibraryItem {
   @Column('text')
   readableContent!: string
 
-  // NOT SUPPORTED IN TYPEORM
-  // @Column('vector', { nullable: true })
-  // embedding?: number[]
-
   @Column('text', { nullable: true })
   textContentHash?: string | null
 
@@ -187,13 +183,13 @@ export class LibraryItem {
   @Column('text', { nullable: true })
   recommenderNames?: string[] | null
 
-  @Column('jsonb')
+  @Column('jsonb', { select: false })
   links?: any | null
 
   @Column('text')
   feedContent?: string | null
 
-  @Column('text')
+  @Column('text', { select: false })
   previewContentType?: string | null
 
   @Column('text')
@@ -205,18 +201,18 @@ export class LibraryItem {
   @Column('text')
   highlightAnnotations?: string[]
 
-  @Column('timestamptz')
+  @Column('timestamptz', { select: false })
   seenAt?: Date
 
-  @Column('ltree')
+  @Column('ltree', { select: false })
   topic?: string
 
-  @Column('timestamptz')
+  @Column('timestamptz', { select: false })
   digestedAt?: Date
 
-  @Column('float')
+  @Column('float', { select: false })
   score?: number
 
-  @Column('text')
+  @Column('text', { select: false })
   previewContent?: string
 }
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index f51c30f1b..501480b1c 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -4,9 +4,7 @@ import { RuleEventType } from './entity/rule'
 import { env } from './env'
 import { ReportType } from './generated/graphql'
 import {
-  enqueueGeneratePreviewContentJob,
   enqueueProcessYouTubeVideo,
-  enqueueScoreJob,
   enqueueThumbnailJob,
   enqueueTriggerRuleJob,
 } from './utils/createTask'
@@ -94,25 +92,6 @@ export const createPubSubClient = (): PubsubClient => {
             logger.error('Failed to enqueue thumbnail job', e)
           }
         }
-
-        const hasPreviewContent = (
-          data: any
-        ): data is { previewContent: string | null } => {
-          return 'previewContent' in data
-        }
-
-        // generate preview content if it is less than 180 characters
-        if (
-          !hasPreviewContent(data) ||
-          (data.previewContent && data.previewContent.length < 180)
-        ) {
-          try {
-            const job = await enqueueGeneratePreviewContentJob(data.id, userId)
-            logger.info('Generate preview job created', { id: job?.id })
-          } catch (e) {
-            logger.error('Failed to enqueue generate preview job', e)
-          }
-        }
       }
     },
     entityUpdated: async <T extends EntityEvent>(
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 9292284fc..5b59d904f 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -642,7 +642,10 @@ export const createSearchQueryBuilder = (
   // exclude content if not requested
   const selects: Select[] = getColumns(libraryItemRepository)
     .filter(
-      (select) => args.includeContent || select !== 'readableContent' //
+      (select) =>
+        args.includeContent ||
+        ['readableContent', 'feedContent', 'previewContent'].indexOf(select) ===
+          -1
     )
     .map((column) => ({ column: `library_item.${column}` }))