From 835b61064f78d314ca9650faee25ee19d76e790b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 7 Oct 2023 21:30:23 +0800 Subject: [PATCH 1/4] allow filter by wordsCount --- packages/api/src/resolvers/article/index.ts | 1 + .../api/src/services/integrations/readwise.ts | 20 +++++--- packages/api/src/services/library_item.ts | 29 ++++++++++- packages/api/src/utils/search.ts | 49 +++++++++++++++++++ 4 files changed, 92 insertions(+), 7 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index f2ae8f800..aa57c1543 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -649,6 +649,7 @@ export const searchResolver = authorized< size: first + 1, // fetch one more item to get next cursor sort: searchQuery.sort, includePending: true, + includeContent: params.includeContent || false, ...searchQuery, }, uid diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 28c444cfc..f111599a8 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -6,7 +6,7 @@ import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { wait } from '../../utils/helpers' import { logger } from '../../utils/logger' -import { getHighlightUrl } from '../highlights' +import { findHighlightsByLibraryItemId, getHighlightUrl } from '../highlights' import { IntegrationService } from './integration' interface ReadwiseHighlight { @@ -64,10 +64,12 @@ export class ReadwiseIntegration extends IntegrationService { ): Promise => { let result = true - const highlights = items.flatMap(this.libraryItemToReadwiseHighlight) + const highlights = await Promise.all( + items.map((item) => this.libraryItemToReadwiseHighlight(item)) + ) // If there are no highlights, we will skip the sync if (highlights.length > 0) { - result = await this.syncWithReadwise(integration.token, highlights) + result = await this.syncWithReadwise(integration.token, highlights.flat()) } // update integration syncedAt if successful @@ -84,10 +86,16 @@ export class ReadwiseIntegration extends IntegrationService { return result } - libraryItemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => { - if (!item.highlights) return [] + libraryItemToReadwiseHighlight = async ( + item: LibraryItem + ): Promise => { + let highlights = item.highlights + if (!highlights) { + highlights = await findHighlightsByLibraryItemId(item.id, item.user.id) + } + const category = item.siteName === 'Twitter' ? 'tweets' : 'articles' - return item.highlights + return highlights .map((highlight) => { // filter out highlights that are not of type highlight or have no quote if ( diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 1dcdb146d..1d869fbfc 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -6,7 +6,7 @@ import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { BulkActionType } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' -import { authTrx } from '../repository' +import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' import { wordsCount } from '../utils/helpers' import { @@ -17,6 +17,7 @@ import { LabelFilter, LabelFilterType, NoFilter, + RangeFilter, ReadFilter, Sort, SortBy, @@ -42,6 +43,7 @@ export interface SearchArgs { recommendedBy?: string includeContent?: boolean noFilters?: NoFilter[] + rangeFilters?: RangeFilter[] } export interface SearchResultItem { @@ -258,6 +260,22 @@ const buildWhereClause = ( ) } } + + if (args.includeContent) { + queryBuilder.addSelect('library_item.readableContent') + } + + if (args.rangeFilters && args.rangeFilters.length > 0) { + args.rangeFilters.forEach((filter, i) => { + const param = `range_${filter.field}_${i}` + queryBuilder.andWhere( + `library_item.${filter.field} ${filter.operator} ${param}`, + { + [param]: filter.value, + } + ) + }) + } } export const searchLibraryItems = async ( @@ -271,11 +289,20 @@ export const searchLibraryItems = async ( // default sort by saved_at const sortField = sort?.by || SortBy.SAVED + const selectColumns = getColumns(libraryItemRepository) + .map((column) => `library_item.${column}`) + .filter( + (column) => + column !== 'library_item.readableContent' && + column !== 'library_item.originalContent' + ) + // add pagination and sorting return authTrx( async (tx) => { const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') + .select(selectColumns) .where('library_item.user_id = :userId', { userId }) // build the where clause diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 5bbe09cde..f5d9e9b6c 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -40,6 +40,7 @@ export interface SearchFilter { ids: string[] recommendedBy?: string noFilters: NoFilter[] + rangeFilters: RangeFilter[] } export enum LabelFilterType { @@ -63,6 +64,12 @@ export interface DateFilter { endDate?: Date } +export interface RangeFilter { + field: string + operator: string + value: number +} + export enum SortBy { SAVED = 'savedAt', UPDATED = 'updatedAt', @@ -255,6 +262,40 @@ const parseDateFilter = ( } } +const parseRangeFilter = ( + field: string, + str?: string +): RangeFilter | undefined => { + if (str === undefined) { + return undefined + } + + switch (field.toUpperCase()) { + case 'WORDSCOUNT': + field = 'wordCount' + break + default: + return undefined + } + + const operatorRegex = /([<>]=?)/ + const operator = str.match(operatorRegex)?.[0] + if (!operator) { + return undefined + } + + const value = str.replace(operatorRegex, '') + if (!value) { + return undefined + } + + return { + field, + operator, + value: Number(value), + } +} + const parseFieldFilter = ( field: string, str?: string @@ -323,6 +364,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { matchFilters: [], ids: [], noFilters: [], + rangeFilters: [], } if (!searchQuery) { @@ -337,6 +379,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { matchFilters: [], ids: [], noFilters: [], + rangeFilters: [], } } @@ -364,6 +407,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { 'site', 'note', 'rss', + 'wordCount', ], tokenize: true, }) @@ -460,6 +504,11 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { case 'mode': // mode is ignored and used only by the frontend break + case 'wordCount': { + const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value) + rangeFilter && result.rangeFilters.push(rangeFilter) + break + } } } } From 3e91f3836b40f2a0f3769b951571fc22a2fdd94b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 9 Oct 2023 12:25:37 +0800 Subject: [PATCH 2/4] fix tests --- packages/api/src/services/integrations/readwise.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index f111599a8..0c6676ebd 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -65,7 +65,9 @@ export class ReadwiseIntegration extends IntegrationService { let result = true const highlights = await Promise.all( - items.map((item) => this.libraryItemToReadwiseHighlight(item)) + items.map((item) => + this.libraryItemToReadwiseHighlight(item, integration.user.id) + ) ) // If there are no highlights, we will skip the sync if (highlights.length > 0) { @@ -87,11 +89,12 @@ export class ReadwiseIntegration extends IntegrationService { } libraryItemToReadwiseHighlight = async ( - item: LibraryItem + item: LibraryItem, + userId: string ): Promise => { let highlights = item.highlights if (!highlights) { - highlights = await findHighlightsByLibraryItemId(item.id, item.user.id) + highlights = await findHighlightsByLibraryItemId(item.id, userId) } const category = item.siteName === 'Twitter' ? 'tweets' : 'articles' From 93c410a56414b73cf9a3fcc0c29c56fd4e263a34 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 Oct 2023 12:21:32 +0800 Subject: [PATCH 3/4] fix is:read filter and add is:reading filter --- .../api/src/events/user/profile_created.ts | 34 ----------- packages/api/src/services/create_user.ts | 6 +- packages/api/src/services/library_item.ts | 60 +++++++++++++------ packages/api/src/utils/search.ts | 3 + packages/api/test/resolvers/article.test.ts | 44 +++++++++++--- 5 files changed, 85 insertions(+), 62 deletions(-) delete mode 100644 packages/api/src/events/user/profile_created.ts diff --git a/packages/api/src/events/user/profile_created.ts b/packages/api/src/events/user/profile_created.ts deleted file mode 100644 index 0af9326e0..000000000 --- a/packages/api/src/events/user/profile_created.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { - EntitySubscriberInterface, - EventSubscriber, - InsertEvent, -} from 'typeorm' -import { Profile } from '../../entity/profile' -import { createDefaultFiltersForUser } from '../../services/create_user' -import { addPopularReadsForNewUser } from '../../services/popular_reads' - -@EventSubscriber() -export class AddPopularReadsToNewUser - implements EntitySubscriberInterface -{ - listenTo() { - return Profile - } - - async afterInsert(event: InsertEvent): Promise { - await addPopularReadsForNewUser(event.entity.user.id, event.manager) - } -} - -@EventSubscriber() -export class AddDefaultFiltersToNewUser - implements EntitySubscriberInterface -{ - listenTo() { - return Profile - } - - async afterInsert(event: InsertEvent): Promise { - await createDefaultFiltersForUser(event.manager)(event.entity.user.id) - } -} diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index 4648658c8..6ded934b4 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -15,6 +15,7 @@ import { analytics } from '../utils/analytics' import { IntercomClient } from '../utils/intercom' import { logger } from '../utils/logger' import { validateUsername } from '../utils/usernamePolicy' +import { addPopularReadsForNewUser } from './popular_reads' import { sendConfirmationEmail } from './send_emails' export const MAX_RECORDS_LIMIT = 1000 @@ -103,6 +104,9 @@ export const createUser = async (input: { }) } + await addPopularReadsForNewUser(user.id, t) + await createDefaultFiltersForUser(t)(user.id) + return [user, profile] } ) @@ -146,7 +150,7 @@ export const createUser = async (input: { return [user, profile] } -export const createDefaultFiltersForUser = +const createDefaultFiltersForUser = (t: EntityManager) => async (userId: string): Promise => { const defaultFilters = [ diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 1d869fbfc..f9f8d5cb7 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,4 +1,13 @@ -import { DeepPartial, SelectQueryBuilder } from 'typeorm' +import { + Between, + DeepPartial, + In, + IsNull, + LessThan, + MoreThan, + Not, + SelectQueryBuilder, +} from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' @@ -106,10 +115,14 @@ const buildWhereClause = ( if (args.inFilter !== InFilter.ALL) { switch (args.inFilter) { case InFilter.INBOX: - queryBuilder.andWhere('library_item.archived_at IS NULL') + queryBuilder.andWhere({ + archivedAt: IsNull(), + }) break case InFilter.ARCHIVE: - queryBuilder.andWhere('library_item.archived_at IS NOT NULL') + queryBuilder.andWhere({ + archivedAt: Not(IsNull()), + }) break case InFilter.TRASH: // return only deleted pages within 14 days @@ -119,16 +132,20 @@ const buildWhereClause = ( break case InFilter.SUBSCRIPTION: queryBuilder - .andWhere('library_item.subscription IS NOT NULL') .andWhere("NOT ('library' ILIKE ANY (library_item.label_names))") - .andWhere('library_item.archived_at IS NULL') + .andWhere({ + subscription: Not(IsNull()), + archivedAt: IsNull(), + }) break case InFilter.LIBRARY: queryBuilder .andWhere( "(library_item.subscription IS NULL OR 'library' ILIKE ANY (library_item.label_names))" ) - .andWhere('library_item.archived_at IS NULL') + .andWhere({ + archivedAt: IsNull(), + }) break } } @@ -136,10 +153,17 @@ const buildWhereClause = ( if (args.readFilter !== ReadFilter.ALL) { switch (args.readFilter) { case ReadFilter.READ: - queryBuilder.andWhere('library_item.reading_progress_top_percent >= 98') + queryBuilder.andWhere({ + readingProgressBottomPercent: MoreThan(98), + }) + break + case ReadFilter.READING: + queryBuilder.andWhere({ readingProgressBottomPercent: Between(2, 98) }) break case ReadFilter.UNREAD: - queryBuilder.andWhere('library_item.reading_progress_top_percent < 98') + queryBuilder.andWhere({ + readingProgressBottomPercent: LessThan(2), + }) break } } @@ -148,12 +172,10 @@ const buildWhereClause = ( args.hasFilters.forEach((filter) => { switch (filter) { case HasFilter.HIGHLIGHTS: - queryBuilder.andWhere( - 'array_length(library_item.highlight_annotations, 1) > 0' - ) + queryBuilder.andWhere("library_item.highlight_annotations <> '{}'") break case HasFilter.LABELS: - queryBuilder.andWhere('array_length(library_item.label_names, 1) > 0') + queryBuilder.andWhere("library_item.label_names <> '{}'") break } }) @@ -225,18 +247,20 @@ const buildWhereClause = ( } if (args.ids && args.ids.length > 0) { - queryBuilder.andWhere('library_item.id IN (:...ids)', { ids: args.ids }) + queryBuilder.andWhere({ + id: In(args.ids), + }) } if (!args.includePending) { - queryBuilder.andWhere('library_item.state != :state', { - state: LibraryItemState.Processing, + queryBuilder.andWhere({ + state: Not(LibraryItemState.Processing), }) } if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere('library_item.state != :state', { - state: LibraryItemState.Deleted, + queryBuilder.andWhere({ + state: Not(LibraryItemState.Deleted), }) } @@ -303,7 +327,7 @@ export const searchLibraryItems = async ( const queryBuilder = tx .createQueryBuilder(LibraryItem, 'library_item') .select(selectColumns) - .where('library_item.user_id = :userId', { userId }) + .where({ user: { id: userId } }) // build the where clause buildWhereClause(queryBuilder, args) diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index f5d9e9b6c..c3af64881 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -14,6 +14,7 @@ import { InputMaybe, PageType, SortParams } from '../generated/graphql' export enum ReadFilter { ALL, READ, + READING, UNREAD, } @@ -110,6 +111,8 @@ const parseIsFilter = (str: string | undefined): ReadFilter => { switch (str?.toUpperCase()) { case 'READ': return ReadFilter.READ + case 'READING': + return ReadFilter.READING case 'UNREAD': return ReadFilter.UNREAD } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 2959a2072..85b788f90 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -818,6 +818,7 @@ describe('Article API', () => { let keyword = '' before(async () => { + const readingProgressArray = [0, 2, 97, 98, 100] // Create some test items for (let i = 0; i < 5; i++) { const itemToSave: DeepPartial = { @@ -827,6 +828,7 @@ describe('Article API', () => { slug: 'test slug', originalUrl: `${url}/${i}`, siteName: 'Example', + readingProgressBottomPercent: readingProgressArray[i], } const item = await createLibraryItem(itemToSave, user.id) items.push(item) @@ -887,15 +889,39 @@ describe('Article API', () => { keyword = `'${searchedKeyword}' is:unread` }) - it('should return unread articles in descending order', async () => { + it('returns unread articles in descending order', async () => { const res = await graphqlRequest(query, authToken).expect(200) - expect(res.body.data.search.edges.length).to.eq(5) + expect(res.body.data.search.edges.length).to.eq(1) + expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id) + }) + }) + + context('when is:reading is in the query', () => { + before(() => { + keyword = `'${searchedKeyword}' is:reading` + }) + + it('returns reading articles in descending order', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.edges.length).to.eq(3) + expect(res.body.data.search.edges[0].node.id).to.eq(items[3].id) + expect(res.body.data.search.edges[1].node.id).to.eq(items[2].id) + expect(res.body.data.search.edges[2].node.id).to.eq(items[1].id) + }) + }) + + context('when is:read is in the query', () => { + before(() => { + keyword = `'${searchedKeyword}' is:read` + }) + + it('returns fully read articles in descending order', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.edges.length).to.eq(1) expect(res.body.data.search.edges[0].node.id).to.eq(items[4].id) - expect(res.body.data.search.edges[1].node.id).to.eq(items[3].id) - expect(res.body.data.search.edges[2].node.id).to.eq(items[2].id) - expect(res.body.data.search.edges[3].node.id).to.eq(items[1].id) - expect(res.body.data.search.edges[4].node.id).to.eq(items[0].id) }) }) @@ -1122,7 +1148,7 @@ describe('Article API', () => { slug: 'test slug 2', originalUrl: `${url}/test2`, archivedAt: new Date(), - readingProgressTopPercent: 100, + readingProgressBottomPercent: 100, }, { user, @@ -1140,7 +1166,7 @@ describe('Article API', () => { await deleteLibraryItems(items, user.id) }) - it('returns unfinished archived items', async () => { + it('returns unread archived items', async () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.search.pageInfo.totalCount).to.eq(1) @@ -1221,7 +1247,7 @@ describe('Article API', () => { slug: 'test slug 2', originalUrl: `${url}/test2`, deletedAt: new Date(), - readingProgressTopPercent: 100, + readingProgressBottomPercent: 100, }, { user, From 62ffaa924bd8b3873d01fbc7731fb7d906f91eb9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 Oct 2023 12:52:49 +0800 Subject: [PATCH 4/4] add wordsCount filter --- packages/api/src/services/library_item.ts | 2 +- packages/api/src/utils/search.ts | 11 ++- packages/api/test/resolvers/article.test.ts | 98 +++++++++++++++++++++ 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index f9f8d5cb7..0fcf6940e 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -293,7 +293,7 @@ const buildWhereClause = ( args.rangeFilters.forEach((filter, i) => { const param = `range_${filter.field}_${i}` queryBuilder.andWhere( - `library_item.${filter.field} ${filter.operator} ${param}`, + `library_item.${filter.field} ${filter.operator} :${param}`, { [param]: filter.value, } diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index c3af64881..357875449 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -275,7 +275,10 @@ const parseRangeFilter = ( switch (field.toUpperCase()) { case 'WORDSCOUNT': - field = 'wordCount' + field = 'word_count' + break + case 'READPOSITION': + field = 'reading_progress_bottom_percent' break default: return undefined @@ -410,7 +413,8 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { 'site', 'note', 'rss', - 'wordCount', + 'wordsCount', + 'readPosition', ], tokenize: true, }) @@ -507,7 +511,8 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { case 'mode': // mode is ignored and used only by the frontend break - case 'wordCount': { + case 'readPosition': + case 'wordsCount': { const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value) rangeFilter && result.rangeFilters.push(rangeFilter) break diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 85b788f90..d8c2545d9 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -1272,6 +1272,104 @@ describe('Article API', () => { expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id) }) }) + + context('when readPosition:>20 readPosition:<50 is in the query', () => { + let items: LibraryItem[] = [] + + before(async () => { + keyword = 'readPosition:>20 readPosition:<50' + // Create some test items + items = await createLibraryItems( + [ + { + user, + title: 'test title 1', + readableContent: '

test 1

', + slug: 'test slug 1', + originalUrl: `${url}/test1`, + readingProgressBottomPercent: 40, + }, + { + user, + title: 'test title 2', + readableContent: '

test 2

', + slug: 'test slug 2', + originalUrl: `${url}/test2`, + readingProgressBottomPercent: 10, + }, + { + user, + title: 'test title 3', + readableContent: '

test 3

', + slug: 'test slug 3', + originalUrl: `${url}/test3`, + readingProgressBottomPercent: 100, + }, + ], + user.id + ) + }) + + after(async () => { + await deleteLibraryItems(items, user.id) + }) + + it('returns items with reading progress between 20% and 50% exclusively', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.pageInfo.totalCount).to.eq(1) + expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id) + }) + }) + + context('when wordsCount:>=10000 wordsCount:<=20000 is in the query', () => { + let items: LibraryItem[] = [] + + before(async () => { + keyword = 'wordsCount:>=10000 wordsCount:<=20000' + // Create some test items + items = await createLibraryItems( + [ + { + user, + title: 'test title 1', + readableContent: '

test 1

', + slug: 'test slug 1', + originalUrl: `${url}/test1`, + wordCount: 10000, + }, + { + user, + title: 'test title 2', + readableContent: '

test 2

', + slug: 'test slug 2', + originalUrl: `${url}/test2`, + wordCount: 8000, + }, + { + user, + title: 'test title 3', + readableContent: '

test 3

', + slug: 'test slug 3', + originalUrl: `${url}/test3`, + wordCount: 100000, + }, + ], + user.id + ) + }) + + after(async () => { + await deleteLibraryItems(items, user.id) + }) + + it('returns items with words count between 10000 and 20000 inclusively', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.pageInfo.totalCount).to.eq(1) + expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id) + }) + }) }) describe('TypeaheadSearch API', () => {