reduce number of db calls for search library items

packages/api/src/jobs/ai/create_digest.ts

@@ -113,18 +113,17 @@ const getPreferencesList = async (userId: string): Promise<LibraryItem[]> => {
   //   reason: "some older items that were interacted with"
 
   const preferences = await Promise.all(
-    digestDefinition.preferenceSelectors.map(async (selector) => {
-      // use the selector to fetch items
-      const results = await searchLibraryItems(
-        {
-          query: selector.query,
-          size: selector.count,
-        },
-        userId
-      )
-
-      return results.libraryItems
-    })
+    digestDefinition.preferenceSelectors.map(
+      async (selector) =>
+        // use the selector to fetch items
+        await searchLibraryItems(
+          {
+            query: selector.query,
+            size: selector.count,
+          },
+          userId
+        )
+    )
   )
 
   // deduplicate and flatten the items
@@ -160,21 +159,20 @@ const getCandidatesList = async (
   logger.info('existingCandidateIds: ', { existingCandidateIds })
 
   const candidates = await Promise.all(
-    digestDefinition.candidateSelectors.map(async (selector) => {
-      // use the selector to fetch items
-      const results = await searchLibraryItems(
-        {
-          includeContent: true,
-          query: existingCandidateIds
-            ? `(${selector.query}) -includes:${existingCandidateIds}` // exclude the existing candidates
-            : selector.query,
-          size: selector.count,
-        },
-        userId
-      )
-
-      return results.libraryItems
-    })
+    digestDefinition.candidateSelectors.map(
+      async (selector) =>
+        // use the selector to fetch items
+        await searchLibraryItems(
+          {
+            includeContent: true,
+            query: existingCandidateIds
+              ? `(${selector.query}) -includes:${existingCandidateIds}` // exclude the existing candidates
+              : selector.query,
+            size: selector.count,
+          },
+          userId
+        )
+    )
   )
 
   // deduplicate and flatten the items

packages/api/src/jobs/trigger_rule.ts

@@ -243,14 +243,13 @@ const triggerActions = async (
     } catch (error) {
       if (error instanceof RequiresSearchQueryError) {
         logger.info('Failed to filter items by metadata, running search query')
-        const searchResult = await searchLibraryItems(
+        results = await searchLibraryItems(
           {
             query: `includes:${data.id} AND (${rule.filter})`,
             size: 1,
           },
           userId
         )
-        results = searchResult.libraryItems
       } else {
         logger.error('Error filtering item events', error)
         await markRuleAsFailed(rule.id, userId)

packages/api/src/resolvers/article/index.ts

@@ -74,7 +74,7 @@ import {
   countLibraryItems,
   createOrUpdateLibraryItem,
   findLibraryItemsByPrefix,
-  searchLibraryItems,
+  searchAndCountLibraryItems,
   softDeleteLibraryItem,
   sortParamsToSort,
   updateLibraryItem,
@@ -675,7 +675,7 @@ export const searchResolver = authorized<
     return { errorCodes: [SearchErrorCode.QueryTooLong] }
   }
 
-  const { libraryItems, count } = await searchLibraryItems(
+  const { libraryItems, count } = await searchAndCountLibraryItems(
     {
       from: Number(startCursor),
       size: first + 1, // fetch one more item to get next cursor
@@ -752,7 +752,7 @@ export const updatesSinceResolver = authorized<
     folder ? ' in:' + folder : ''
   } sort:${sort.by}-${sort.order}`
 
-  const { libraryItems, count } = await searchLibraryItems(
+  const { libraryItems, count } = await searchAndCountLibraryItems(
     {
       from: Number(startCursor),
       size: size + 1, // fetch one more item to get next cursor

packages/api/src/services/library_item.ts

@@ -6,10 +6,10 @@ import {
   EntityManager,
   FindOptionsWhere,
   ObjectLiteral,
-  SelectQueryBuilder,
 } from 'typeorm'
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
 import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
+import { appDataSource } from '../data_source'
 import { EntityLabel } from '../entity/entity_label'
 import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
@@ -18,12 +18,7 @@ import { env } from '../env'
 import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
 import { createPubSubClient, EntityEvent, EntityType } from '../pubsub'
 import { redisDataSource } from '../redis_data_source'
-import {
-  authTrx,
-  getColumns,
-  getRepository,
-  queryBuilderToRawSql,
-} from '../repository'
+import { authTrx, getColumns, queryBuilderToRawSql } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
 import { Merge, PickTuple } from '../util'
 import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers'
@@ -619,11 +614,13 @@ export const buildQueryString = (
   return serialize(searchQuery)
 }
 
-export const buildQuery = (
-  queryBuilder: SelectQueryBuilder<LibraryItem>,
+export const createSearchQueryBuilder = (
   args: SearchArgs,
-  userId: string
+  userId: string,
+  em = appDataSource.manager
 ) => {
+  const queryBuilder = em.createQueryBuilder(LibraryItem, 'library_item')
+
   // select all columns except content
   const selects: Select[] = getColumns(libraryItemRepository)
     .filter(
@@ -688,44 +685,54 @@ export const buildQuery = (
   orders.forEach((order) => {
     queryBuilder.addOrderBy(order.by, order.order, order.nulls)
   })
+
+  return queryBuilder
 }
 
 export const countLibraryItems = async (args: SearchArgs, userId: string) => {
-  const queryBuilder =
-    getRepository(LibraryItem).createQueryBuilder('library_item')
-
-  buildQuery(queryBuilder, args, userId)
-
-  return queryBuilder.getCount()
+  return authTrx(
+    async (tx) => createSearchQueryBuilder(args, userId, tx).getCount(),
+    undefined,
+    userId
+  )
 }
 
 export const searchLibraryItems = async (
   args: SearchArgs,
   userId: string
-): Promise<{ libraryItems: LibraryItem[]; count: number }> => {
+): Promise<LibraryItem[]> => {
   const { from = 0, size = 10 } = args
 
+  if (size === 0) {
+    // return only count if size is 0 because limit 0 is not allowed in typeorm
+    return []
+  }
+
   return authTrx(
-    async (tx) => {
-      const queryBuilder = tx.createQueryBuilder(LibraryItem, 'library_item')
-      buildQuery(queryBuilder, args, userId)
-
-      const count = await queryBuilder.getCount()
-      if (size === 0) {
-        // return only count if size is 0 because limit 0 is not allowed in typeorm
-        return { libraryItems: [], count }
-      }
-
-      // add pagination
-      const libraryItems = await queryBuilder.skip(from).take(size).getMany()
-
-      return { libraryItems, count }
-    },
+    async (tx) =>
+      createSearchQueryBuilder(args, userId, tx)
+        .skip(from)
+        .take(size)
+        .getMany(),
     undefined,
     userId
   )
 }
 
+export const searchAndCountLibraryItems = async (
+  args: SearchArgs,
+  userId: string
+): Promise<{ libraryItems: LibraryItem[]; count: number }> => {
+  const count = await countLibraryItems(args, userId)
+  if (count === 0) {
+    return { libraryItems: [], count }
+  }
+
+  const libraryItems = await searchLibraryItems(args, userId)
+
+  return { libraryItems, count }
+}
+
 export const findRecentLibraryItems = async (
   userId: string,
   limit = 1000,

packages/api/test/routers/auth.test.ts

@@ -6,7 +6,7 @@ import { userRepository } from '../../src/repository/user'
 import { isValidSignupRequest } from '../../src/routers/auth/auth_router'
 import { AuthProvider } from '../../src/routers/auth/auth_types'
 import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers'
-import { searchLibraryItems } from '../../src/services/library_item'
+import { searchAndCountLibraryItems } from '../../src/services/library_item'
 import { deleteUser, updateUser } from '../../src/services/user'
 import {
   comparePassword,
@@ -528,7 +528,7 @@ describe('auth router', () => {
           'web'
         ).expect(200)
         const user = await userRepository.findOneByOrFail({ name })
-        const { count } = await searchLibraryItems(
+        const { count } = await searchAndCountLibraryItems(
           { query: 'in:inbox sort:read-desc is:reading' },
           user.id
         )
@@ -552,7 +552,10 @@ describe('auth router', () => {
           'ios'
         ).expect(200)
         const user = await userRepository.findOneByOrFail({ name })
-        const { count } = await searchLibraryItems({ query: 'in:all' }, user.id)
+        const { count } = await searchAndCountLibraryItems(
+          { query: 'in:all' },
+          user.id
+        )
 
         expect(count).to.eql(4)
       })