From 24457d90365903471d6ebe10d54e8970abfaf859 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 6 Jun 2024 16:12:07 +0800 Subject: [PATCH] search highlights by labels --- packages/api/src/repository/index.ts | 20 ++ packages/api/src/services/highlights.ts | 189 +++++++++++++++++- packages/api/src/services/library_item.ts | 30 +-- packages/api/test/resolvers/highlight.test.ts | 57 +++++- 4 files changed, 260 insertions(+), 36 deletions(-) diff --git a/packages/api/src/repository/index.ts b/packages/api/src/repository/index.ts index 3a5db7e72..21c59becd 100644 --- a/packages/api/src/repository/index.ts +++ b/packages/api/src/repository/index.ts @@ -12,6 +12,26 @@ import { appDataSource } from '../data_source' import { Claims } from '../resolvers/types' import { SetClaimsRole } from '../utils/dictionary' +export enum SortOrder { + ASCENDING = 'ASC', + DESCENDING = 'DESC', +} + +export interface Sort { + by: string + order?: SortOrder + nulls?: 'NULLS FIRST' | 'NULLS LAST' +} + +export interface Select { + column: string + alias?: string +} + +export const paramtersToObject = (parameters: ObjectLiteral[]) => { + return parameters.reduce((a, b) => ({ ...a, ...b }), {}) +} + export const getColumns = ( repository: Repository ): (keyof T)[] => { diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index d6ad393c0..e154faf71 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -1,16 +1,18 @@ +import { ExpressionToken, LiqeQuery } from '@omnivore/liqe' import { diff_match_patch } from 'diff-match-patch' -import { DeepPartial, In } from 'typeorm' +import { DeepPartial, In, ObjectLiteral } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { homePageURL } from '../env' import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' -import { authTrx } from '../repository' +import { authTrx, paramtersToObject } from '../repository' import { highlightRepository } from '../repository/highlight' import { Merge } from '../util' import { enqueueUpdateHighlight } from '../utils/createTask' import { deepDelete } from '../utils/helpers' +import { parseSearchQuery } from '../utils/search' import { ItemEvent } from './library_item' const columnsToDelete = ['user', 'sharedAt', 'libraryItem'] as const @@ -279,6 +281,149 @@ export const findHighlightsByLibraryItemId = async ( ) } +export const buildQueryString = ( + searchQuery: LiqeQuery, + parameters: ObjectLiteral[] = [] +) => { + const escapeQueryWithParameters = ( + query: string, + parameter: ObjectLiteral + ) => { + parameters.push(parameter) + return query + } + + const serializeImplicitField = ( + expression: ExpressionToken + ): string | null => { + if (expression.type !== 'LiteralExpression') { + throw new Error('Expected a literal expression') + } + + // not implemented yet + return null + } + + const serializeTagExpression = (ast: LiqeQuery): string | null => { + if (ast.type !== 'Tag') { + throw new Error('Expected a tag expression') + } + + const { field, expression } = ast + + if (field.type === 'ImplicitField') { + return serializeImplicitField(expression) + } else { + if (expression.type !== 'LiteralExpression') { + // ignore empty values + return null + } + + const value = expression.value?.toString() + if (!value) { + // ignore empty values + return null + } + + switch (field.name.toLowerCase()) { + case 'label': { + const labels = value.toLowerCase().split(',') + return ( + labels + .map((label) => { + const param = `label_${parameters.length}` + + const hasWildcard = label.includes('*') + if (hasWildcard) { + return escapeQueryWithParameters( + `label.name ILIKE :${param}`, + { + [param]: label.replace(/\*/g, '%'), + } + ) + } + + return escapeQueryWithParameters( + `LOWER(label.name) = :${param}`, + { + [param]: label.toLowerCase(), + } + ) + }) + .join(' OR ') + // wrap in brackets to avoid precedence issues + .replace(/^(.*)$/, '($1)') + ) + } + default: + // treat unknown fields as implicit fields + return serializeImplicitField({ + ...expression, + value: `${field.name}:${value}`, + }) + } + } + } + + const serialize = (ast: LiqeQuery): string | null => { + if (ast.type === 'Tag') { + return serializeTagExpression(ast) + } + + if (ast.type === 'LogicalExpression') { + let operator = '' + if (ast.operator.operator === 'AND') { + operator = 'AND' + } else if (ast.operator.operator === 'OR') { + operator = 'OR' + } else { + throw new Error('Unexpected operator') + } + + const left = serialize(ast.left) + const right = serialize(ast.right) + + if (!left && !right) { + return null + } + + if (!left) { + return right + } + + if (!right) { + return left + } + + return `${left} ${operator} ${right}` + } + + if (ast.type === 'UnaryOperator') { + const serialized = serialize(ast.operand) + + if (!serialized) { + return null + } + + return `NOT ${serialized}` + } + + if (ast.type === 'ParenthesizedExpression') { + const serialized = serialize(ast.expression) + + if (!serialized) { + return null + } + + return `(${serialized})` + } + + return null + } + + return serialize(searchQuery) +} + export const searchHighlights = async ( userId: string, query?: string, @@ -286,15 +431,37 @@ export const searchHighlights = async ( offset?: number ): Promise> => { return authTrx( - async (tx) => - tx.withRepository(highlightRepository).find({ - where: { user: { id: userId } }, - order: { - updatedAt: 'DESC', - }, - take: limit, - skip: offset, - }), + async (tx) => { + // TODO: parse query and search by it + const queryBuilder = tx + .getRepository(Highlight) + .createQueryBuilder('highlight') + + queryBuilder + .andWhere('highlight.userId = :userId', { userId }) + .orderBy('highlight.updatedAt', 'DESC') + .take(limit) + .skip(offset) + + if (query) { + const parameters: ObjectLiteral[] = [] + + const searchQuery = parseSearchQuery(query) + + // build query string and save parameters + const queryString = buildQueryString(searchQuery, parameters) + + if (queryString) { + // add where clause from query string + queryBuilder + .innerJoinAndSelect('highlight.labels', 'label') + .andWhere(`(${queryString})`) + .setParameters(paramtersToObject(parameters)) + } + } + + return queryBuilder.getMany() + }, undefined, userId ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4390e6128..c6ecbfc9f 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -18,7 +18,15 @@ import { env } from '../env' import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' import { createPubSubClient, EntityEvent, EntityType } from '../pubsub' import { redisDataSource } from '../redis_data_source' -import { authTrx, getColumns, queryBuilderToRawSql } from '../repository' +import { + authTrx, + getColumns, + paramtersToObject, + queryBuilderToRawSql, + Select, + Sort, + SortOrder, +} from '../repository' import { libraryItemRepository } from '../repository/library_item' import { Merge, PickTuple } from '../util' import { enqueueBulkUploadContentJob } from '../utils/createTask' @@ -122,22 +130,6 @@ export enum SortBy { WORDS_COUNT = 'wordscount', } -export enum SortOrder { - ASCENDING = 'ASC', - DESCENDING = 'DESC', -} - -export interface Sort { - by: string - order?: SortOrder - nulls?: 'NULLS FIRST' | 'NULLS LAST' -} - -interface Select { - column: string - alias?: string -} - const readingProgressDataSource = new ReadingProgressDataSource() export const batchGetLibraryItems = async (ids: readonly string[]) => { @@ -197,10 +189,6 @@ const handleNoCase = (value: string) => { throw new Error(`Unexpected keyword: ${value}`) } -const paramtersToObject = (parameters: ObjectLiteral[]) => { - return parameters.reduce((a, b) => ({ ...a, ...b }), {}) -} - export const sortParamsToSort = ( sortParams: InputMaybe | undefined ) => { diff --git a/packages/api/test/resolvers/highlight.test.ts b/packages/api/test/resolvers/highlight.test.ts index df2dd9bad..dc0a2c08e 100644 --- a/packages/api/test/resolvers/highlight.test.ts +++ b/packages/api/test/resolvers/highlight.test.ts @@ -12,7 +12,11 @@ import { deleteHighlightsByIds, findHighlightById, } from '../../src/services/highlights' -import { createLabel, saveLabelsInHighlight } from '../../src/services/labels' +import { + createLabel, + deleteLabels, + saveLabelsInHighlight, +} from '../../src/services/labels' import { deleteUser } from '../../src/services/user' import { createTestLibraryItem, createTestUser } from '../db' import { @@ -358,14 +362,24 @@ describe('Highlights API', () => { describe('Get highlights API', () => { const query = ` - query Highlights ($first: Int, $after: String) { - highlights (first: $first, after: $after) { + query Highlights ($first: Int, $after: String, $query: String) { + highlights (first: $first, after: $after, query: $query) { ... on HighlightsSuccess { edges { node { id user { id + name + } + labels { + id + name + color + } + libraryItem { + id + title } } cursor @@ -416,12 +430,47 @@ describe('Highlights API', () => { ) }) - it('returns highlights', async () => { + it('returns highlights in descending order', async () => { const res = await graphqlRequest(query, authToken).expect(200) const highlights = res.body.data.highlights.edges as Array expect(highlights).to.have.lengthOf(existingHighlights.length) expect(highlights[0].node.id).to.eq(existingHighlights[1].id) expect(highlights[1].node.id).to.eq(existingHighlights[0].id) + expect(highlights[0].node.user.id).to.eq(user.id) + expect(highlights[1].node.libraryItem.id).to.eq( + existingHighlights[0].libraryItemId + ) + }) + + it('returns highlights with pagination', async () => { + const res = await graphqlRequest(query, authToken, { + first: 1, + }).expect(200) + + const highlights = res.body.data.highlights.edges as Array + expect(highlights).to.have.lengthOf(1) + }) + + it('returns highlights with labels', async () => { + // create labels + const labelName = 'test_label' + const label = await createLabel(labelName, '#ff0000', user.id) + const labelName1 = 'test_label_1' + const label1 = await createLabel(labelName1, '#ff0001', user.id) + + // save labels in highlights + await saveLabelsInHighlight([label], existingHighlights[0].id, user.id) + await saveLabelsInHighlight([label1], existingHighlights[1].id, user.id) + + const res = await graphqlRequest(query, authToken, { + query: `label:${labelName},${labelName1}`, + }).expect(200) + const highlights = res.body.data.highlights.edges as Array + expect(highlights).to.have.lengthOf(2) + expect(highlights[1].node.labels?.[0].name).to.eq(labelName) + expect(highlights[0].node.labels?.[0].name).to.eq(labelName1) + + await deleteLabels([label.id, label1.id], user.id) }) }) })