diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index 25f10dd94..2a6e155d4 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -113,18 +113,17 @@ const getPreferencesList = async (userId: string): Promise => { // reason: "some older items that were interacted with" const preferences = await Promise.all( - digestDefinition.preferenceSelectors.map(async (selector) => { - // use the selector to fetch items - const results = await searchLibraryItems( - { - query: selector.query, - size: selector.count, - }, - userId - ) - - return results.libraryItems - }) + digestDefinition.preferenceSelectors.map( + async (selector) => + // use the selector to fetch items + await searchLibraryItems( + { + query: selector.query, + size: selector.count, + }, + userId + ) + ) ) // deduplicate and flatten the items @@ -160,21 +159,20 @@ const getCandidatesList = async ( logger.info('existingCandidateIds: ', { existingCandidateIds }) const candidates = await Promise.all( - digestDefinition.candidateSelectors.map(async (selector) => { - // use the selector to fetch items - const results = await searchLibraryItems( - { - includeContent: true, - query: existingCandidateIds - ? `(${selector.query}) -includes:${existingCandidateIds}` // exclude the existing candidates - : selector.query, - size: selector.count, - }, - userId - ) - - return results.libraryItems - }) + digestDefinition.candidateSelectors.map( + async (selector) => + // use the selector to fetch items + await searchLibraryItems( + { + includeContent: true, + query: existingCandidateIds + ? `(${selector.query}) -includes:${existingCandidateIds}` // exclude the existing candidates + : selector.query, + size: selector.count, + }, + userId + ) + ) ) // deduplicate and flatten the items diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index dcc1c902f..27b3a6a64 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -243,14 +243,13 @@ const triggerActions = async ( } catch (error) { if (error instanceof RequiresSearchQueryError) { logger.info('Failed to filter items by metadata, running search query') - const searchResult = await searchLibraryItems( + results = await searchLibraryItems( { query: `includes:${data.id} AND (${rule.filter})`, size: 1, }, userId ) - results = searchResult.libraryItems } else { logger.error('Error filtering item events', error) await markRuleAsFailed(rule.id, userId) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 874d235da..660474c7b 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -74,7 +74,7 @@ import { countLibraryItems, createOrUpdateLibraryItem, findLibraryItemsByPrefix, - searchLibraryItems, + searchAndCountLibraryItems, softDeleteLibraryItem, sortParamsToSort, updateLibraryItem, @@ -675,7 +675,7 @@ export const searchResolver = authorized< return { errorCodes: [SearchErrorCode.QueryTooLong] } } - const { libraryItems, count } = await searchLibraryItems( + const { libraryItems, count } = await searchAndCountLibraryItems( { from: Number(startCursor), size: first + 1, // fetch one more item to get next cursor @@ -752,7 +752,7 @@ export const updatesSinceResolver = authorized< folder ? ' in:' + folder : '' } sort:${sort.by}-${sort.order}` - const { libraryItems, count } = await searchLibraryItems( + const { libraryItems, count } = await searchAndCountLibraryItems( { from: Number(startCursor), size: size + 1, // fetch one more item to get next cursor diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 47ec7bba8..d91cfa10d 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -6,10 +6,10 @@ import { EntityManager, FindOptionsWhere, ObjectLiteral, - SelectQueryBuilder, } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' +import { appDataSource } from '../data_source' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' @@ -18,12 +18,7 @@ import { env } from '../env' import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { redisDataSource } from '../redis_data_source' -import { - authTrx, - getColumns, - getRepository, - queryBuilderToRawSql, -} from '../repository' +import { authTrx, getColumns, queryBuilderToRawSql } from '../repository' import { libraryItemRepository } from '../repository/library_item' import { Merge, PickTuple } from '../util' import { deepDelete, setRecentlySavedItemInRedis } from '../utils/helpers' @@ -619,11 +614,13 @@ export const buildQueryString = ( return serialize(searchQuery) } -export const buildQuery = ( - queryBuilder: SelectQueryBuilder, +export const createSearchQueryBuilder = ( args: SearchArgs, - userId: string + userId: string, + em = appDataSource.manager ) => { + const queryBuilder = em.createQueryBuilder(LibraryItem, 'library_item') + // select all columns except content const selects: Select[] = getColumns(libraryItemRepository) .filter( @@ -688,44 +685,54 @@ export const buildQuery = ( orders.forEach((order) => { queryBuilder.addOrderBy(order.by, order.order, order.nulls) }) + + return queryBuilder } export const countLibraryItems = async (args: SearchArgs, userId: string) => { - const queryBuilder = - getRepository(LibraryItem).createQueryBuilder('library_item') - - buildQuery(queryBuilder, args, userId) - - return queryBuilder.getCount() + return authTrx( + async (tx) => createSearchQueryBuilder(args, userId, tx).getCount(), + undefined, + userId + ) } export const searchLibraryItems = async ( args: SearchArgs, userId: string -): Promise<{ libraryItems: LibraryItem[]; count: number }> => { +): Promise => { const { from = 0, size = 10 } = args + if (size === 0) { + // return only count if size is 0 because limit 0 is not allowed in typeorm + return [] + } + return authTrx( - async (tx) => { - const queryBuilder = tx.createQueryBuilder(LibraryItem, 'library_item') - buildQuery(queryBuilder, args, userId) - - const count = await queryBuilder.getCount() - if (size === 0) { - // return only count if size is 0 because limit 0 is not allowed in typeorm - return { libraryItems: [], count } - } - - // add pagination - const libraryItems = await queryBuilder.skip(from).take(size).getMany() - - return { libraryItems, count } - }, + async (tx) => + createSearchQueryBuilder(args, userId, tx) + .skip(from) + .take(size) + .getMany(), undefined, userId ) } +export const searchAndCountLibraryItems = async ( + args: SearchArgs, + userId: string +): Promise<{ libraryItems: LibraryItem[]; count: number }> => { + const count = await countLibraryItems(args, userId) + if (count === 0) { + return { libraryItems: [], count } + } + + const libraryItems = await searchLibraryItems(args, userId) + + return { libraryItems, count } +} + export const findRecentLibraryItems = async ( userId: string, limit = 1000, diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 32b438047..415231149 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -6,7 +6,7 @@ import { userRepository } from '../../src/repository/user' import { isValidSignupRequest } from '../../src/routers/auth/auth_router' import { AuthProvider } from '../../src/routers/auth/auth_types' import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers' -import { searchLibraryItems } from '../../src/services/library_item' +import { searchAndCountLibraryItems } from '../../src/services/library_item' import { deleteUser, updateUser } from '../../src/services/user' import { comparePassword, @@ -528,7 +528,7 @@ describe('auth router', () => { 'web' ).expect(200) const user = await userRepository.findOneByOrFail({ name }) - const { count } = await searchLibraryItems( + const { count } = await searchAndCountLibraryItems( { query: 'in:inbox sort:read-desc is:reading' }, user.id ) @@ -552,7 +552,10 @@ describe('auth router', () => { 'ios' ).expect(200) const user = await userRepository.findOneByOrFail({ name }) - const { count } = await searchLibraryItems({ query: 'in:all' }, user.id) + const { count } = await searchAndCountLibraryItems( + { query: 'in:all' }, + user.id + ) expect(count).to.eql(4) })