Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2879 from omnivore-app/feature/filter-by-word-count

Browse Source 
allow filter by wordsCount
This commit is contained in:
Hongbo Wu
2023-10-10 13:06:34 +08:00
committed by GitHub
parent fe3093834c 62ffaa924b
commit 29c8786057
7 changed files with 283 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/api/src/events/user/profile_created.ts
View File

@ -1,34 +0,0 @@
import {
EntitySubscriberInterface,
EventSubscriber,
InsertEvent,
} from 'typeorm'
import { Profile } from '../../entity/profile'
import { createDefaultFiltersForUser } from '../../services/create_user'
import { addPopularReadsForNewUser } from '../../services/popular_reads'
@EventSubscriber()
export class AddPopularReadsToNewUser
implements EntitySubscriberInterface<Profile>
{
listenTo() {
return Profile
}
async afterInsert(event: InsertEvent<Profile>): Promise<void> {
await addPopularReadsForNewUser(event.entity.user.id, event.manager)
}
}
@EventSubscriber()
export class AddDefaultFiltersToNewUser
implements EntitySubscriberInterface<Profile>
{
listenTo() {
return Profile
}
async afterInsert(event: InsertEvent<Profile>): Promise<void> {
await createDefaultFiltersForUser(event.manager)(event.entity.user.id)
}
}

1
packages/api/src/resolvers/article/index.ts
View File

@ -649,6 +649,7 @@ export const searchResolver = authorized<
size: first + 1, // fetch one more item to get next cursor
sort: searchQuery.sort,
includePending: true,
includeContent: params.includeContent || false,
...searchQuery,
},
uid

6
packages/api/src/services/create_user.ts
View File

@ -15,6 +15,7 @@ import { analytics } from '../utils/analytics'
import { IntercomClient } from '../utils/intercom'
import { logger } from '../utils/logger'
import { validateUsername } from '../utils/usernamePolicy'
import { addPopularReadsForNewUser } from './popular_reads'
import { sendConfirmationEmail } from './send_emails'
export const MAX_RECORDS_LIMIT = 1000
@ -103,6 +104,9 @@ export const createUser = async (input: {
})
}
await addPopularReadsForNewUser(user.id, t)
await createDefaultFiltersForUser(t)(user.id)
return [user, profile]
}
)
@ -146,7 +150,7 @@ export const createUser = async (input: {
return [user, profile]
}
export const createDefaultFiltersForUser =
const createDefaultFiltersForUser =
(t: EntityManager) =>
async (userId: string): Promise<Filter[]> => {
const defaultFilters = [

23
packages/api/src/services/integrations/readwise.ts
View File

@ -6,7 +6,7 @@ import { LibraryItem } from '../../entity/library_item'
import { env } from '../../env'
import { wait } from '../../utils/helpers'
import { logger } from '../../utils/logger'
import { getHighlightUrl } from '../highlights'
import { findHighlightsByLibraryItemId, getHighlightUrl } from '../highlights'
import { IntegrationService } from './integration'
interface ReadwiseHighlight {
@ -64,10 +64,14 @@ export class ReadwiseIntegration extends IntegrationService {
): Promise<boolean> => {
let result = true
const highlights = items.flatMap(this.libraryItemToReadwiseHighlight)
const highlights = await Promise.all(
items.map((item) =>
this.libraryItemToReadwiseHighlight(item, integration.user.id)
)
)
// If there are no highlights, we will skip the sync
if (highlights.length > 0) {
result = await this.syncWithReadwise(integration.token, highlights)
result = await this.syncWithReadwise(integration.token, highlights.flat())
}
// update integration syncedAt if successful
@ -84,10 +88,17 @@ export class ReadwiseIntegration extends IntegrationService {
return result
}
libraryItemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => {
if (!item.highlights) return []
libraryItemToReadwiseHighlight = async (
item: LibraryItem,
userId: string
): Promise<ReadwiseHighlight[]> => {
let highlights = item.highlights
if (!highlights) {
highlights = await findHighlightsByLibraryItemId(item.id, userId)
}
const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
return item.highlights
return highlights
.map((highlight) => {
// filter out highlights that are not of type highlight or have no quote
if (

89
packages/api/src/services/library_item.ts
View File

@ -1,4 +1,13 @@
import { DeepPartial, SelectQueryBuilder } from 'typeorm'
import {
Between,
DeepPartial,
In,
IsNull,
LessThan,
MoreThan,
Not,
SelectQueryBuilder,
} from 'typeorm'
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
import { EntityLabel } from '../entity/entity_label'
import { Highlight } from '../entity/highlight'
@ -6,7 +15,7 @@ import { Label } from '../entity/label'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { BulkActionType } from '../generated/graphql'
import { createPubSubClient, EntityType } from '../pubsub'
import { authTrx } from '../repository'
import { authTrx, getColumns } from '../repository'
import { libraryItemRepository } from '../repository/library_item'
import { wordsCount } from '../utils/helpers'
import {
@ -17,6 +26,7 @@ import {
LabelFilter,
LabelFilterType,
NoFilter,
RangeFilter,
ReadFilter,
Sort,
SortBy,
@ -42,6 +52,7 @@ export interface SearchArgs {
recommendedBy?: string
includeContent?: boolean
noFilters?: NoFilter[]
rangeFilters?: RangeFilter[]
}
export interface SearchResultItem {
@ -104,10 +115,14 @@ const buildWhereClause = (
if (args.inFilter !== InFilter.ALL) {
switch (args.inFilter) {
case InFilter.INBOX:
queryBuilder.andWhere('library_item.archived_at IS NULL')
queryBuilder.andWhere({
archivedAt: IsNull(),
})
break
case InFilter.ARCHIVE:
queryBuilder.andWhere('library_item.archived_at IS NOT NULL')
queryBuilder.andWhere({
archivedAt: Not(IsNull()),
})
break
case InFilter.TRASH:
// return only deleted pages within 14 days
@ -117,16 +132,20 @@ const buildWhereClause = (
break
case InFilter.SUBSCRIPTION:
queryBuilder
.andWhere('library_item.subscription IS NOT NULL')
.andWhere("NOT ('library' ILIKE ANY (library_item.label_names))")
.andWhere('library_item.archived_at IS NULL')
.andWhere({
subscription: Not(IsNull()),
archivedAt: IsNull(),
})
break
case InFilter.LIBRARY:
queryBuilder
.andWhere(
"(library_item.subscription IS NULL OR 'library' ILIKE ANY (library_item.label_names))"
)
.andWhere('library_item.archived_at IS NULL')
.andWhere({
archivedAt: IsNull(),
})
break
}
}
@ -134,10 +153,17 @@ const buildWhereClause = (
if (args.readFilter !== ReadFilter.ALL) {
switch (args.readFilter) {
case ReadFilter.READ:
queryBuilder.andWhere('library_item.reading_progress_top_percent >= 98')
queryBuilder.andWhere({
readingProgressBottomPercent: MoreThan(98),
})
break
case ReadFilter.READING:
queryBuilder.andWhere({ readingProgressBottomPercent: Between(2, 98) })
break
case ReadFilter.UNREAD:
queryBuilder.andWhere('library_item.reading_progress_top_percent < 98')
queryBuilder.andWhere({
readingProgressBottomPercent: LessThan(2),
})
break
}
}
@ -146,12 +172,10 @@ const buildWhereClause = (
args.hasFilters.forEach((filter) => {
switch (filter) {
case HasFilter.HIGHLIGHTS:
queryBuilder.andWhere(
'array_length(library_item.highlight_annotations, 1) > 0'
)
queryBuilder.andWhere("library_item.highlight_annotations <> '{}'")
break
case HasFilter.LABELS:
queryBuilder.andWhere('array_length(library_item.label_names, 1) > 0')
queryBuilder.andWhere("library_item.label_names <> '{}'")
break
}
})
@ -223,18 +247,20 @@ const buildWhereClause = (
}
if (args.ids && args.ids.length > 0) {
queryBuilder.andWhere('library_item.id IN (:...ids)', { ids: args.ids })
queryBuilder.andWhere({
id: In(args.ids),
})
}
if (!args.includePending) {
queryBuilder.andWhere('library_item.state != :state', {
state: LibraryItemState.Processing,
queryBuilder.andWhere({
state: Not(LibraryItemState.Processing),
})
}
if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) {
queryBuilder.andWhere('library_item.state != :state', {
state: LibraryItemState.Deleted,
queryBuilder.andWhere({
state: Not(LibraryItemState.Deleted),
})
}
@ -258,6 +284,22 @@ const buildWhereClause = (
)
}
}
if (args.includeContent) {
queryBuilder.addSelect('library_item.readableContent')
}
if (args.rangeFilters && args.rangeFilters.length > 0) {
args.rangeFilters.forEach((filter, i) => {
const param = `range_${filter.field}_${i}`
queryBuilder.andWhere(
`library_item.${filter.field} ${filter.operator} :${param}`,
{
[param]: filter.value,
}
)
})
}
}
export const searchLibraryItems = async (
@ -271,12 +313,21 @@ export const searchLibraryItems = async (
// default sort by saved_at
const sortField = sort?.by || SortBy.SAVED
const selectColumns = getColumns(libraryItemRepository)
.map((column) => `library_item.${column}`)
.filter(
(column) =>
column !== 'library_item.readableContent' &&
column !== 'library_item.originalContent'
)
// add pagination and sorting
return authTrx(
async (tx) => {
const queryBuilder = tx
.createQueryBuilder(LibraryItem, 'library_item')
.where('library_item.user_id = :userId', { userId })
.select(selectColumns)
.where({ user: { id: userId } })
// build the where clause
buildWhereClause(queryBuilder, args)

57
packages/api/src/utils/search.ts
View File

@ -14,6 +14,7 @@ import { InputMaybe, PageType, SortParams } from '../generated/graphql'
export enum ReadFilter {
ALL,
READ,
READING,
UNREAD,
}
@ -40,6 +41,7 @@ export interface SearchFilter {
ids: string[]
recommendedBy?: string
noFilters: NoFilter[]
rangeFilters: RangeFilter[]
}
export enum LabelFilterType {
@ -63,6 +65,12 @@ export interface DateFilter {
endDate?: Date
}
export interface RangeFilter {
field: string
operator: string
value: number
}
export enum SortBy {
SAVED = 'savedAt',
UPDATED = 'updatedAt',
@ -103,6 +111,8 @@ const parseIsFilter = (str: string | undefined): ReadFilter => {
switch (str?.toUpperCase()) {
case 'READ':
return ReadFilter.READ
case 'READING':
return ReadFilter.READING
case 'UNREAD':
return ReadFilter.UNREAD
}
@ -255,6 +265,43 @@ const parseDateFilter = (
}
}
const parseRangeFilter = (
field: string,
str?: string
): RangeFilter | undefined => {
if (str === undefined) {
return undefined
}
switch (field.toUpperCase()) {
case 'WORDSCOUNT':
field = 'word_count'
break
case 'READPOSITION':
field = 'reading_progress_bottom_percent'
break
default:
return undefined
}
const operatorRegex = /([<>]=?)/
const operator = str.match(operatorRegex)?.[0]
if (!operator) {
return undefined
}
const value = str.replace(operatorRegex, '')
if (!value) {
return undefined
}
return {
field,
operator,
value: Number(value),
}
}
const parseFieldFilter = (
field: string,
str?: string
@ -323,6 +370,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
matchFilters: [],
ids: [],
noFilters: [],
rangeFilters: [],
}
if (!searchQuery) {
@ -337,6 +385,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
matchFilters: [],
ids: [],
noFilters: [],
rangeFilters: [],
}
}
@ -364,6 +413,8 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
'site',
'note',
'rss',
'wordsCount',
'readPosition',
],
tokenize: true,
})
@ -460,6 +511,12 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
case 'mode':
// mode is ignored and used only by the frontend
break
case 'readPosition':
case 'wordsCount': {
const rangeFilter = parseRangeFilter(keyword.keyword, keyword.value)
rangeFilter && result.rangeFilters.push(rangeFilter)
break
}
}
}
}

142
packages/api/test/resolvers/article.test.ts
View File

@ -818,6 +818,7 @@ describe('Article API', () => {
let keyword = ''
before(async () => {
const readingProgressArray = [0, 2, 97, 98, 100]
// Create some test items
for (let i = 0; i < 5; i++) {
const itemToSave: DeepPartial<LibraryItem> = {
@ -827,6 +828,7 @@ describe('Article API', () => {
slug: 'test slug',
originalUrl: `${url}/${i}`,
siteName: 'Example',
readingProgressBottomPercent: readingProgressArray[i],
}
const item = await createLibraryItem(itemToSave, user.id)
items.push(item)
@ -887,15 +889,39 @@ describe('Article API', () => {
keyword = `'${searchedKeyword}' is:unread`
})
it('should return unread articles in descending order', async () => {
it('returns unread articles in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eq(5)
expect(res.body.data.search.edges.length).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when is:reading is in the query', () => {
before(() => {
keyword = `'${searchedKeyword}' is:reading`
})
it('returns reading articles in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eq(3)
expect(res.body.data.search.edges[0].node.id).to.eq(items[3].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[2].id)
expect(res.body.data.search.edges[2].node.id).to.eq(items[1].id)
})
})
context('when is:read is in the query', () => {
before(() => {
keyword = `'${searchedKeyword}' is:read`
})
it('returns fully read articles in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[4].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[3].id)
expect(res.body.data.search.edges[2].node.id).to.eq(items[2].id)
expect(res.body.data.search.edges[3].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[4].node.id).to.eq(items[0].id)
})
})
@ -1122,7 +1148,7 @@ describe('Article API', () => {
slug: 'test slug 2',
originalUrl: `${url}/test2`,
archivedAt: new Date(),
readingProgressTopPercent: 100,
readingProgressBottomPercent: 100,
},
{
user,
@ -1140,7 +1166,7 @@ describe('Article API', () => {
await deleteLibraryItems(items, user.id)
})
it('returns unfinished archived items', async () => {
it('returns unread archived items', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
@ -1221,7 +1247,7 @@ describe('Article API', () => {
slug: 'test slug 2',
originalUrl: `${url}/test2`,
deletedAt: new Date(),
readingProgressTopPercent: 100,
readingProgressBottomPercent: 100,
},
{
user,
@ -1246,6 +1272,104 @@ describe('Article API', () => {
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when readPosition:>20 readPosition:<50 is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'readPosition:>20 readPosition:<50'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
readingProgressBottomPercent: 40,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
readingProgressBottomPercent: 10,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
readingProgressBottomPercent: 100,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns items with reading progress between 20% and 50% exclusively', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when wordsCount:>=10000 wordsCount:<=20000 is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'wordsCount:>=10000 wordsCount:<=20000'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
wordCount: 10000,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
wordCount: 8000,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
wordCount: 100000,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns items with words count between 10000 and 20000 inclusively', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
})
describe('TypeaheadSearch API', () => {