diff --git a/docker-compose-test.yml b/docker-compose-test.yml new file mode 100644 index 000000000..8772ffc50 --- /dev/null +++ b/docker-compose-test.yml @@ -0,0 +1,73 @@ +version: '3' +services: + postgres-test: + image: "postgres:12.8" + container_name: "omnivore-postgres-test" + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=omnivore_test + - PG_POOL_MAX=20 + healthcheck: + test: "exit 0" + interval: 2s + timeout: 12s + retries: 3 + expose: + - 5432 + + elastic-test: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 + container_name: "omnivore-elastic-test" + healthcheck: + test: curl 0.0.0.0:9201/_cat/health >/dev/null || exit 1 + interval: 2s + timeout: 2s + retries: 5 + environment: + - discovery.type=single-node + - http.cors.allow-origin=* + - http.cors.enabled=true + - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization + - http.cors.allow-credentials=true + - http.port=9201 + volumes: + - ./.docker/elastic-data:/usr/share/elasticsearch/data + ports: + - "9201:9201" + + api-test: + build: + context: . + dockerfile: ./packages/api/Dockerfile-test + container_name: "omnivore-api-test" + environment: + - API_ENV=local + - PG_HOST=postgres-test + - PG_USER=postgres + - PG_PASSWORD=postgres + - PG_DB=omnivore_test + - PG_PORT=5432 + - PG_POOL_MAX=20 + - ELASTIC_URL=http://elastic-test:9201 + - IMAGE_PROXY_URL=http://localhost:9999 + - IMAGE_PROXY_SECRET=some-secret + - JWT_SECRET=some_secret + - SSO_JWT_SECRET=some_sso_secret + - CLIENT_URL=http://localhost:3000 + - GATEWAY_URL=http://localhost:8080/api + - PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/ + - REMINDER_TASK_HANDLER_URL=/svc/reminders/trigger + - BOOKMARKLET_JWT_SECRET=some_bookmarklet_secret + - BOOKMARKLET_VERSION=1.0.0 + - PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper' + - GCP_PROJECT_ID=omnivore-local + - GAUTH_CLIENT_ID='notset' + - GAUTH_SECRET='notset' + - SEGMENT_WRITE_KEY='test' + - PUBSUB_VERIFICATION_TOKEN='123456' + depends_on: + postgres-test: + condition: service_healthy + elastic-test: + condition: service_healthy diff --git a/packages/api/.env.example b/packages/api/.env.example index 2590b876d..7c3204150 100644 --- a/packages/api/.env.example +++ b/packages/api/.env.example @@ -23,4 +23,5 @@ GCS_UPLOAD_BUCKET= GCS_UPLOAD_SA_KEY_FILE_PATH= TWITTER_BEARER_TOKEN= PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper' -REMINDER_TASK_HANDLER_URL= \ No newline at end of file +REMINDER_TASK_HANDLER_URL= +ELASTIC_URL=http://localhost:9200 diff --git a/packages/api/.env.test b/packages/api/.env.test index d3b940a69..4d10b8316 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -28,4 +28,4 @@ SEGMENT_WRITE_KEY='test' REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger PUBSUB_VERIFICATION_TOKEN='123456' PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/ - +ELASTIC_URL=http://localhost:9200 diff --git a/packages/api/Dockerfile-test b/packages/api/Dockerfile-test index 262cb5a29..bdeb25b6d 100644 --- a/packages/api/Dockerfile-test +++ b/packages/api/Dockerfile-test @@ -18,6 +18,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true RUN yarn install +ADD /packages/db ./packages/db ADD /packages/readabilityjs ./packages/readabilityjs ADD /packages/api ./packages/api diff --git a/packages/api/add_highlight_to_elastic.py b/packages/api/add_highlight_to_elastic.py new file mode 100644 index 000000000..46d73ba8a --- /dev/null +++ b/packages/api/add_highlight_to_elastic.py @@ -0,0 +1,188 @@ +#!/usr/bin/python +import os +import json +import psycopg2 +from psycopg2.extras import RealDictCursor +from elasticsearch import Elasticsearch, NotFoundError + +PG_HOST = os.getenv('PG_HOST', 'localhost') +PG_PORT = os.getenv('PG_PORT', 5432) +PG_USER = os.getenv('PG_USER', 'app_user') +PG_PASSWORD = os.getenv('PG_PASSWORD', 'app_pass') +PG_DB = os.getenv('PG_DB', 'omnivore') +ES_URL = os.getenv('ES_URL', 'http://localhost:9200') +ES_USERNAME = os.getenv('ES_USERNAME', 'elastic') +ES_PASSWORD = os.getenv('ES_PASSWORD', 'password') +UPDATE_TIME = os.getenv('UPDATE_TIME', '2019-01-01 00:00:00') +INDEX_SETTINGS = os.getenv('INDEX_SETTINGS', 'index_settings.json') +DATETIME_FORMAT = 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"' + + +def update_mappings(client: Elasticsearch): + print('updating mappings') + try: + with open(INDEX_SETTINGS, 'r') as f: + settings = json.load(f) + client.indices.put_mapping( + index='pages_alias', + body=settings['mappings']) + print('mappings updated') + except Exception as err: + print('update mappings ERROR:', err) + exit(1) + + +def assertData(conn, client: Elasticsearch, pages): + # get all users from postgres + try: + success = 0 + failure = 0 + skip = 0 + cursor = conn.cursor(cursor_factory=RealDictCursor) + for page in pages: + pageId = page['pageId'] + cursor.execute( + f'''SELECT COUNT(*) FROM omnivore.highlight + WHERE elastic_page_id = \'{pageId}\' AND deleted = false''') + countInPostgres = cursor.fetchone()['count'] + try: + countInElastic = len(client.get( + index='pages_alias', + id=pageId, + _source=['highlights'])['_source']['highlights']) + except NotFoundError as err: + print('Elasticsearch get ERROR:', err) + # if page is not found in elasticsearch, skip testing + skip += 1 + continue + + if countInPostgres == countInElastic: + success += 1 + print(f'Page {pageId} OK') + else: + failure += 1 + print( + f'Page {pageId} ERROR: postgres: {countInPostgres}, elastic: {countInElastic}') + cursor.close() + print( + f'Asserted data, success: {success}, failure: {failure}, skip: {skip}') + except Exception as err: + print('Assert data ERROR:', err) + exit(1) + + +def ingest_highlights(conn, pages): + try: + import_count = 0 + cursor = conn.cursor(cursor_factory=RealDictCursor) + + for page in pages: + pageId = page['pageId'] + query = ''' + SELECT + id, + quote, + prefix, + to_char(created_at, '{DATETIME_FORMAT}') as "createdAt", + to_char(COALESCE(updated_at, current_timestamp), '{DATETIME_FORMAT}') as "updatedAt", + suffix, + patch, + annotation, + short_id as "shortId", + user_id as "userId", + to_char(shared_at, '{DATETIME_FORMAT}') as "sharedAt" + FROM omnivore.highlight + WHERE + elastic_page_id = \'{pageId}\' + AND deleted = false + AND created_at > '{UPDATE_TIME}' + '''.format(pageId=pageId, DATETIME_FORMAT=DATETIME_FORMAT, UPDATE_TIME=UPDATE_TIME) + + cursor.execute(query) + result = cursor.fetchall() + import_count += import_highlights_to_es(client, result, pageId) + + print(f'Imported total {import_count} highlights to es') + + cursor.close() + except Exception as err: + print('Export data to json ERROR:', err) + + +def import_highlights_to_es(client, highlights, pageId) -> int: + # import highlights to elasticsearch + print(f'Writing {len(highlights)} highlights to page {pageId}') + + if len(highlights) == 0: + print('No highlights to import') + return 0 + + try: + resp = client.update( + index='pages_alias', + id=pageId, + body={'doc': {'highlights': highlights}}) + + count = 0 + if resp['result'] == 'updated': + count = len(highlights) + + print(f'Added {count} highlights to page {pageId}') + + return count + except Exception as err: + print('Elasticsearch update ERROR:', err) + return 0 + + +def get_pages_with_highlights(conn): + try: + query = f''' + SELECT DISTINCT + elastic_page_id as "pageId" + FROM omnivore.highlight + WHERE + elastic_page_id IS NOT NULL + AND deleted = false + AND created_at > '{UPDATE_TIME}' + ''' + cursor = conn.cursor(cursor_factory=RealDictCursor) + cursor.execute(query) + result = cursor.fetchall() + cursor.close() + print('Found pages with highlights:', len(result)) + return result + except Exception as err: + print('Get pages with highlights ERROR:', err) + + +print('Starting migration') + +# test elastic client +client = Elasticsearch(ES_URL, http_auth=( + ES_USERNAME, ES_PASSWORD), retry_on_timeout=True) +try: + print('Elasticsearch client connected', client.info()) +except Exception as err: + print('Elasticsearch client ERROR:', err) + exit(1) + +# test postgres client +conn = psycopg2.connect( + f'host={PG_HOST} port={PG_PORT} dbname={PG_DB} user={PG_USER} \ + password={PG_PASSWORD}') +print('Postgres connection:', conn.info) + + +update_mappings(client) + +pages = get_pages_with_highlights(conn) + +ingest_highlights(conn, pages) + +assertData(conn, client, pages) + +client.close() +conn.close() + +print('Migration complete') diff --git a/packages/api/elastic_migrate.py b/packages/api/elastic_migrate.py index d0be65ede..3fd7003d5 100644 --- a/packages/api/elastic_migrate.py +++ b/packages/api/elastic_migrate.py @@ -89,7 +89,7 @@ def assertData(conn, client): f'SELECT COUNT(*) FROM omnivore.links WHERE user_id = \'{userId}\'''') countInPostgres = cursor.fetchone()['count'] countInElastic = client.count( - index='pages', body={'query': {'term': {'userId': userId}}})['count'] + index='pages_alias', body={'query': {'term': {'userId': userId}}})['count'] if countInPostgres == countInElastic: success += 1 @@ -197,7 +197,7 @@ def import_data_to_es(client, docs) -> int: doc['publishedAt'] = validated_date(doc['publishedAt']) # convert the string to a dict object dict_doc = { - '_index': 'pages', + '_index': 'pages_alias', '_id': doc['id'], '_source': doc } diff --git a/packages/api/index_settings.json b/packages/api/index_settings.json index c0c3aab57..2e540dcde 100644 --- a/packages/api/index_settings.json +++ b/packages/api/index_settings.json @@ -56,6 +56,30 @@ } } }, + "highlights": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "userId": { + "type": "keyword" + }, + "quote": { + "type": "text", + "analyzer": "strip_html_analyzer" + }, + "annotation": { + "type": "text" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + } + } + }, "readingProgressPercent": { "type": "float" }, diff --git a/packages/api/package.json b/packages/api/package.json index 80c7c523a..312f97c91 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -8,7 +8,7 @@ "start": "node dist/server.js", "lint": "eslint src --ext ts,js,tsx,jsx", "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx", - "test": "nyc mocha -r ts-node/register --config mocha-config.json --exit --timeout 10000" + "test": "nyc mocha -r ts-node/register --config mocha-config.json --timeout 10000" }, "dependencies": { "@elastic/elasticsearch": "~7.12.0", diff --git a/packages/api/src/datalayer/links/share_info.ts b/packages/api/src/datalayer/links/share_info.ts index 4bc45acd3..2646cfc57 100644 --- a/packages/api/src/datalayer/links/share_info.ts +++ b/packages/api/src/datalayer/links/share_info.ts @@ -3,7 +3,7 @@ import Knex from 'knex' import { LinkShareInfo } from '../../generated/graphql' import { DataModels } from '../../resolvers/types' -import { getPageByParam } from '../../elastic' +import { getPageByParam } from '../../elastic/pages' // once we have links setup properly in the API we will remove this method // and have a getShareInfoForLink method diff --git a/packages/api/src/elastic/highlights.ts b/packages/api/src/elastic/highlights.ts new file mode 100644 index 000000000..5a499237e --- /dev/null +++ b/packages/api/src/elastic/highlights.ts @@ -0,0 +1,281 @@ +import { + Highlight, + Page, + PageContext, + SearchItem, + SearchResponse, +} from './types' +import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import { client, INDEX_ALIAS } from './index' +import { SortBy, SortOrder, SortParams } from '../utils/search' + +export const addHighlightToPage = async ( + id: string, + highlight: Highlight, + ctx: PageContext +): Promise => { + try { + const { body } = await client.update({ + index: INDEX_ALIAS, + id, + body: { + script: { + source: `if (ctx._source.highlights == null) { + ctx._source.highlights = [params.highlight] + } else { + ctx._source.highlights.add(params.highlight) + }`, + lang: 'painless', + params: { + highlight: highlight, + }, + }, + }, + refresh: ctx.refresh, + retry_on_conflict: 3, + }) + + return body.result === 'updated' + } catch (e) { + if ( + e instanceof ResponseError && + e.message === 'document_missing_exception' + ) { + console.log('page has been deleted', id) + return false + } + console.error('failed to add highlight to a page in elastic', e) + return false + } +} + +export const getHighlightById = async ( + id: string +): Promise => { + try { + const { body } = await client.search({ + index: INDEX_ALIAS, + body: { + query: { + nested: { + path: 'highlights', + query: { + match: { + 'highlights.id': id, + }, + }, + inner_hits: {}, + }, + }, + _source: false, + }, + }) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + if (body.hits.total.value === 0) { + return undefined + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-return + return body.hits.hits[0].inner_hits.highlights.hits.hits[0]._source + } catch (e) { + console.error('failed to get highlight from a page in elastic', e) + return undefined + } +} + +export const deleteHighlight = async ( + highlightId: string, + ctx: PageContext +): Promise => { + try { + const { body } = await client.updateByQuery({ + index: INDEX_ALIAS, + body: { + script: { + source: + 'ctx._source.highlights.removeIf(h -> h.id == params.highlightId)', + lang: 'painless', + params: { + highlightId: highlightId, + }, + }, + query: { + bool: { + filter: [ + { + term: { + userId: ctx.uid, + }, + }, + { + nested: { + path: 'highlights', + query: { + term: { + 'highlights.id': highlightId, + }, + }, + }, + }, + ], + }, + }, + }, + refresh: ctx.refresh, + }) + + return !!body.updated + } catch (e) { + console.error('failed to delete a highlight in elastic', e) + + return false + } +} + +export const searchHighlights = async ( + args: { + from?: number + size?: number + sort?: SortParams + query?: string + }, + userId: string +): Promise<[SearchItem[], number] | undefined> => { + try { + const { from = 0, size = 10, sort, query } = args + const sortOrder = sort?.order || SortOrder.DESCENDING + // default sort by updatedAt + const sortField = + sort?.by === SortBy.SCORE ? SortBy.SCORE : 'highlights.updatedAt' + + const searchBody = { + query: { + nested: { + path: 'highlights', + query: { + bool: { + filter: [ + { + term: { + 'highlights.userId': userId, + }, + }, + ], + should: [ + { + multi_match: { + query: query || '', + fields: ['highlights.quote', 'highlights.annotation'], + operator: 'and', + type: 'cross_fields', + }, + }, + ], + minimum_should_match: query ? 1 : 0, + }, + }, + inner_hits: {}, + }, + }, + sort: [ + { + [sortField]: { + order: sortOrder, + nested: { + path: 'highlights', + }, + }, + }, + ], + from, + size, + _source: ['title', 'slug', 'url', 'createdAt'], + } + + console.log('searching highlights in elastic', JSON.stringify(searchBody)) + + const response = await client.search>({ + index: INDEX_ALIAS, + body: searchBody, + }) + + if (response.body.hits.total.value === 0) { + return [[], 0] + } + + const results: SearchItem[] = [] + response.body.hits.hits.forEach((hit) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-member-access + hit.inner_hits.highlights.hits.hits.forEach( + (innerHit: { _source: Highlight }) => { + results.push({ + ...hit._source, + ...innerHit._source, + pageId: hit._id, + }) + } + ) + }) + + return [results, response.body.hits.total.value] + } catch (e) { + console.error('failed to search highlights in elastic', e) + return undefined + } +} + +export const updateHighlight = async ( + highlight: Highlight, + ctx: PageContext +): Promise => { + try { + const { body } = await client.updateByQuery({ + index: INDEX_ALIAS, + body: { + script: { + source: `ctx._source.highlights.removeIf(h -> h.id == params.highlight.id); + ctx._source.highlights.add(params.highlight)`, + lang: 'painless', + params: { + highlight: highlight, + }, + }, + query: { + bool: { + filter: [ + { + term: { + userId: ctx.uid, + }, + }, + { + nested: { + path: 'highlights', + query: { + term: { + 'highlights.id': highlight.id, + }, + }, + }, + }, + ], + }, + }, + }, + refresh: ctx.refresh, + }) + + return !!body.updated + } catch (e) { + if ( + e instanceof ResponseError && + e.message === 'document_missing_exception' + ) { + console.log('page has been deleted') + return false + } + console.error('failed to update highlight in elastic', e) + return false + } +} diff --git a/packages/api/src/elastic/index.ts b/packages/api/src/elastic/index.ts index 1108c1619..118132069 100644 --- a/packages/api/src/elastic/index.ts +++ b/packages/api/src/elastic/index.ts @@ -1,35 +1,11 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ import { env } from '../env' import { Client } from '@elastic/elasticsearch' -import { - Label, - PageType, - SortBy, - SortOrder, - SortParams, -} from '../generated/graphql' -import { - InFilter, - LabelFilter, - LabelFilterType, - ReadFilter, -} from '../utils/search' -import { - Page, - PageContext, - ParamSet, - SearchBody, - SearchResponse, -} from './types' import { readFileSync } from 'fs' import { join } from 'path' -import { ResponseError } from '@elastic/elasticsearch/lib/errors' -const INDEX_NAME = 'pages' -const INDEX_ALIAS = 'pages_alias' -const client = new Client({ +export const INDEX_NAME = 'pages' +export const INDEX_ALIAS = 'pages_alias' +export const client = new Client({ node: env.elastic.url, maxRetries: 3, requestTimeout: 50000, @@ -52,484 +28,6 @@ const ingest = async (): Promise => { }) } -const appendQuery = (body: SearchBody, query: string): void => { - body.query.bool.should.push({ - multi_match: { - query, - fields: ['title', 'content', 'author', 'description', 'siteName'], - operator: 'and', - type: 'cross_fields', - }, - }) - body.query.bool.minimum_should_match = 1 -} - -const appendTypeFilter = (body: SearchBody, filter: PageType): void => { - body.query.bool.filter.push({ - term: { - pageType: filter, - }, - }) -} - -const appendReadFilter = (body: SearchBody, filter: ReadFilter): void => { - switch (filter) { - case ReadFilter.UNREAD: - body.query.bool.filter.push({ - range: { - readingProgress: { - gte: 98, - }, - }, - }) - break - case ReadFilter.READ: - body.query.bool.filter.push({ - range: { - readingProgress: { - lt: 98, - }, - }, - }) - } -} - -const appendInFilter = (body: SearchBody, filter: InFilter): void => { - switch (filter) { - case InFilter.ARCHIVE: - body.query.bool.filter.push({ - exists: { - field: 'archivedAt', - }, - }) - break - case InFilter.INBOX: - body.query.bool.must_not.push({ - exists: { - field: 'archivedAt', - }, - }) - } -} - -const appendNotNullField = (body: SearchBody, field: string): void => { - body.query.bool.filter.push({ - exists: { - field, - }, - }) -} - -const appendExcludeLabelFilter = ( - body: SearchBody, - filters: LabelFilter[] -): void => { - body.query.bool.must_not.push({ - nested: { - path: 'labels', - query: filters.map((filter) => { - return { - terms: { - 'labels.name': filter.labels, - }, - } - }), - }, - }) -} - -const appendIncludeLabelFilter = ( - body: SearchBody, - filters: LabelFilter[] -): void => { - body.query.bool.filter.push({ - nested: { - path: 'labels', - query: { - bool: { - filter: filters.map((filter) => { - return { - terms: { - 'labels.name': filter.labels, - }, - } - }), - }, - }, - }, - }) -} - -export const createPage = async ( - page: Page, - ctx: PageContext -): Promise => { - try { - const { body } = await client.index({ - id: page.id || undefined, - index: INDEX_ALIAS, - body: { - ...page, - updatedAt: new Date(), - savedAt: new Date(), - }, - refresh: ctx.refresh, - }) - - await ctx.pubsub.pageCreated(page) - - return body._id as string - } catch (e) { - console.error('failed to create a page in elastic', e) - return undefined - } -} - -export const updatePage = async ( - id: string, - page: Partial, - ctx: PageContext -): Promise => { - try { - const { body } = await client.update({ - index: INDEX_ALIAS, - id, - body: { - doc: { - ...page, - updatedAt: new Date(), - }, - }, - refresh: ctx.refresh, - retry_on_conflict: 3, - }) - - if (body.result !== 'updated') return false - - await ctx.pubsub.pageUpdated({ ...page, id }, ctx.uid) - - return true - } catch (e) { - if ( - e instanceof ResponseError && - e.message === 'document_missing_exception' - ) { - console.info('page has been deleted', id) - return false - } - console.error('failed to update a page in elastic', e) - return false - } -} - -export const addLabelInPage = async ( - id: string, - label: Label, - ctx: PageContext -): Promise => { - try { - const { body } = await client.update({ - index: INDEX_ALIAS, - id, - body: { - script: { - source: `if (ctx._source.labels == null) { - ctx._source.labels = [params.label] - } else if (!ctx._source.labels.any(label -> label.name == params.label.name)) { - ctx._source.labels.add(params.label) - } else { ctx.op = 'none' }`, - lang: 'painless', - params: { - label: label, - }, - }, - }, - refresh: ctx.refresh, - retry_on_conflict: 3, - }) - - return body.result === 'updated' - } catch (e) { - console.error('failed to update a page in elastic', e) - return false - } -} - -export const deletePage = async ( - id: string, - ctx: PageContext -): Promise => { - try { - const { body } = await client.delete({ - index: INDEX_ALIAS, - id, - refresh: ctx.refresh, - }) - - if (body.deleted === 0) return false - - await ctx.pubsub.pageDeleted(id, ctx.uid) - - return true - } catch (e) { - console.error('failed to delete a page in elastic', e) - return false - } -} - -export const deleteLabelInPages = async ( - userId: string, - label: string, - ctx: PageContext -): Promise => { - try { - await client.updateByQuery({ - index: INDEX_ALIAS, - body: { - script: { - source: - 'ctx._source.labels.removeIf(label -> label.name == params.label)', - lang: 'painless', - params: { - label: label, - }, - }, - query: { - bool: { - filter: [ - { - term: { - userId, - }, - }, - { - nested: { - path: 'labels', - query: { - term: { - 'labels.name': label, - }, - }, - }, - }, - ], - }, - }, - }, - refresh: ctx.refresh, - }) - } catch (e) { - console.error('failed to delete a page in elastic', e) - } -} - -export const getPageByParam = async ( - param: Record -): Promise => { - try { - const params = { - query: { - bool: { - filter: Object.keys(param).map((key) => { - return { - term: { - [key]: param[key as K], - }, - } - }), - }, - }, - size: 1, - _source: { - excludes: ['originalHtml'], - }, - } - - const { body } = await client.search({ - index: INDEX_ALIAS, - body: params, - }) - - if (body.hits.total.value === 0) { - return undefined - } - - return { - ...body.hits.hits[0]._source, - id: body.hits.hits[0]._id, - } as Page - } catch (e) { - console.error('failed to search pages in elastic', e) - return undefined - } -} - -export const getPageById = async (id: string): Promise => { - try { - const { body } = await client.get({ - index: INDEX_ALIAS, - id, - }) - - return { - ...body._source, - id: body._id, - } as Page - } catch (e) { - console.error('failed to search pages in elastic', e) - return undefined - } -} - -export const searchPages = async ( - args: { - from?: number - size?: number - sort?: SortParams - query?: string - inFilter: InFilter - readFilter: ReadFilter - typeFilter?: PageType - labelFilters: LabelFilter[] - }, - userId: string, - notNullField: string | null = null -): Promise<[Page[], number] | undefined> => { - try { - const { - from = 0, - size = 10, - sort, - query, - readFilter, - typeFilter, - labelFilters, - inFilter, - } = args - const sortOrder = sort?.order === SortOrder.Ascending ? 'asc' : 'desc' - // default sort by saved_at - const sortField = sort?.by === SortBy.Score ? '_score' : 'savedAt' - const includeLabels = labelFilters.filter( - (filter) => filter.type === LabelFilterType.INCLUDE - ) - const excludeLabels = labelFilters.filter( - (filter) => filter.type === LabelFilterType.EXCLUDE - ) - - const body: SearchBody = { - query: { - bool: { - filter: [ - { - term: { - userId, - }, - }, - ], - should: [], - must_not: [], - }, - }, - sort: [ - { - [sortField]: { - order: sortOrder, - }, - }, - ], - from, - size, - _source: { - excludes: ['originalHtml', 'content'], - }, - } - - // append filters - if (query) { - appendQuery(body, query) - } - if (typeFilter) { - appendTypeFilter(body, typeFilter) - } - if (inFilter !== InFilter.ALL) { - appendInFilter(body, inFilter) - } - if (readFilter !== ReadFilter.ALL) { - appendReadFilter(body, readFilter) - } - if (notNullField) { - appendNotNullField(body, notNullField) - } - if (includeLabels.length > 0) { - appendIncludeLabelFilter(body, includeLabels) - } - if (excludeLabels.length > 0) { - appendExcludeLabelFilter(body, excludeLabels) - } - - console.log('searching pages in elastic', JSON.stringify(body)) - - const response = await client.search, SearchBody>({ - index: INDEX_ALIAS, - body, - }) - - if (response.body.hits.total.value === 0) { - return [[], 0] - } - - return [ - response.body.hits.hits.map((hit: { _source: Page; _id: string }) => ({ - ...hit._source, - content: '', - id: hit._id, - })), - response.body.hits.total.value, - ] - } catch (e) { - console.error('failed to search pages in elastic', e) - return undefined - } -} - -export const countByCreatedAt = async ( - userId: string, - from?: number, - to?: number -): Promise => { - try { - const { body } = await client.count({ - index: INDEX_ALIAS, - body: { - query: { - bool: { - filter: [ - { - term: { - userId, - }, - }, - { - range: { - createdAt: { - gte: from, - lte: to, - }, - }, - }, - ], - }, - }, - }, - }) - - return body.count as number - } catch (e) { - console.error('failed to count pages in elastic', e) - return 0 - } -} - export const initElasticsearch = async (): Promise => { try { const response = await client.info() diff --git a/packages/api/src/elastic/labels.ts b/packages/api/src/elastic/labels.ts new file mode 100644 index 000000000..ec1ac5188 --- /dev/null +++ b/packages/api/src/elastic/labels.ts @@ -0,0 +1,81 @@ +import { Label, PageContext } from './types' +import { client, INDEX_ALIAS } from './index' + +export const addLabelInPage = async ( + id: string, + label: Label, + ctx: PageContext +): Promise => { + try { + const { body } = await client.update({ + index: INDEX_ALIAS, + id, + body: { + script: { + source: `if (ctx._source.labels == null) { + ctx._source.labels = [params.label] + } else if (!ctx._source.labels.any(label -> label.name == params.label.name)) { + ctx._source.labels.add(params.label) + } else { ctx.op = 'none' }`, + lang: 'painless', + params: { + label: label, + }, + }, + }, + refresh: ctx.refresh, + retry_on_conflict: 3, + }) + + return body.result === 'updated' + } catch (e) { + console.error('failed to update a page in elastic', e) + return false + } +} + +export const deleteLabelInPages = async ( + userId: string, + label: string, + ctx: PageContext +): Promise => { + try { + await client.updateByQuery({ + index: INDEX_ALIAS, + body: { + script: { + source: + 'ctx._source.labels.removeIf(label -> label.name == params.label)', + lang: 'painless', + params: { + label: label, + }, + }, + query: { + bool: { + filter: [ + { + term: { + userId, + }, + }, + { + nested: { + path: 'labels', + query: { + term: { + 'labels.name': label, + }, + }, + }, + }, + ], + }, + }, + }, + refresh: ctx.refresh, + }) + } catch (e) { + console.error('failed to delete a page in elastic', e) + } +} diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts new file mode 100644 index 000000000..96f1b5d4a --- /dev/null +++ b/packages/api/src/elastic/pages.ts @@ -0,0 +1,470 @@ +import { + Page, + PageContext, + PageType, + ParamSet, + SearchBody, + SearchResponse, +} from './types' +import { + DateRangeFilter, + HasFilter, + InFilter, + LabelFilter, + LabelFilterType, + ReadFilter, + SortBy, + SortOrder, + SortParams, +} from '../utils/search' +import { client, INDEX_ALIAS } from './index' + +const appendQuery = (body: SearchBody, query: string): void => { + body.query.bool.should.push({ + multi_match: { + query, + fields: ['title', 'content', 'author', 'description', 'siteName'], + operator: 'and', + type: 'cross_fields', + }, + }) + body.query.bool.minimum_should_match = 1 +} + +const appendTypeFilter = (body: SearchBody, filter: PageType): void => { + body.query.bool.filter.push({ + term: { + pageType: filter, + }, + }) +} + +const appendReadFilter = (body: SearchBody, filter: ReadFilter): void => { + switch (filter) { + case ReadFilter.UNREAD: + body.query.bool.filter.push({ + range: { + readingProgress: { + gte: 98, + }, + }, + }) + break + case ReadFilter.READ: + body.query.bool.filter.push({ + range: { + readingProgress: { + lt: 98, + }, + }, + }) + } +} + +const appendInFilter = (body: SearchBody, filter: InFilter): void => { + switch (filter) { + case InFilter.ARCHIVE: + body.query.bool.filter.push({ + exists: { + field: 'archivedAt', + }, + }) + break + case InFilter.INBOX: + body.query.bool.must_not.push({ + exists: { + field: 'archivedAt', + }, + }) + } +} + +const appendHasFilters = (body: SearchBody, filters: HasFilter[]): void => { + filters.forEach((filter) => { + switch (filter) { + case HasFilter.HIGHLIGHTS: + body.query.bool.filter.push({ + nested: { + path: 'highlights', + query: { + exists: { + field: 'highlights', + }, + }, + }, + }) + break + case HasFilter.SHARED_AT: + body.query.bool.filter.push({ + exists: { + field: 'sharedAt', + }, + }) + break + } + }) +} + +const appendExcludeLabelFilter = ( + body: SearchBody, + filters: LabelFilter[] +): void => { + body.query.bool.must_not.push({ + nested: { + path: 'labels', + query: filters.map((filter) => { + return { + terms: { + 'labels.name': filter.labels, + }, + } + }), + }, + }) +} + +const appendIncludeLabelFilter = ( + body: SearchBody, + filters: LabelFilter[] +): void => { + body.query.bool.filter.push({ + nested: { + path: 'labels', + query: { + bool: { + filter: filters.map((filter) => { + return { + terms: { + 'labels.name': filter.labels, + }, + } + }), + }, + }, + }, + }) +} + +const appendSavedDateFilter = ( + body: SearchBody, + filter: DateRangeFilter +): void => { + body.query.bool.filter.push({ + range: { + savedAt: { + gt: filter.startDate, + lt: filter.endDate, + }, + }, + }) +} + +const appendPublishedDateFilter = ( + body: SearchBody, + filter: DateRangeFilter +): void => { + body.query.bool.filter.push({ + range: { + publishedAt: { + gt: filter.startDate, + lt: filter.endDate, + }, + }, + }) +} + +export const createPage = async ( + page: Page, + ctx: PageContext +): Promise => { + try { + const { body } = await client.index({ + id: page.id || undefined, + index: INDEX_ALIAS, + body: { + ...page, + updatedAt: new Date(), + savedAt: new Date(), + }, + refresh: ctx.refresh, + }) + + await ctx.pubsub.pageCreated(page) + + return body._id as string + } catch (e) { + console.error('failed to create a page in elastic', e) + return undefined + } +} + +export const updatePage = async ( + id: string, + page: Partial, + ctx: PageContext +): Promise => { + try { + const { body } = await client.update({ + index: INDEX_ALIAS, + id, + body: { + doc: { + ...page, + updatedAt: new Date(), + }, + }, + refresh: ctx.refresh, + retry_on_conflict: 3, + }) + + if (body.result !== 'updated') return false + + await ctx.pubsub.pageUpdated({ ...page, id }, ctx.uid) + + return true + } catch (e) { + console.error('failed to update a page in elastic', e) + return false + } +} + +export const deletePage = async ( + id: string, + ctx: PageContext +): Promise => { + try { + const { body } = await client.delete({ + index: INDEX_ALIAS, + id, + refresh: ctx.refresh, + }) + + if (body.deleted === 0) return false + + await ctx.pubsub.pageDeleted(id, ctx.uid) + + return true + } catch (e) { + console.error('failed to delete a page in elastic', e) + return false + } +} + +export const getPageByParam = async ( + param: Record +): Promise => { + try { + const params = { + query: { + bool: { + filter: Object.keys(param).map((key) => { + return { + term: { + [key]: param[key as K], + }, + } + }), + }, + }, + size: 1, + _source: { + excludes: ['originalHtml'], + }, + } + + const { body } = await client.search>({ + index: INDEX_ALIAS, + body: params, + }) + + if (body.hits.total.value === 0) { + return undefined + } + + return { + ...body.hits.hits[0]._source, + id: body.hits.hits[0]._id, + } as Page + } catch (e) { + console.error('failed to search pages in elastic', e) + return undefined + } +} + +export const getPageById = async (id: string): Promise => { + try { + const { body } = await client.get({ + index: INDEX_ALIAS, + id, + }) + + return { + ...body._source, + id: body._id as string, + } as Page + } catch (e) { + console.error('failed to search pages in elastic', e) + return undefined + } +} + +export const searchPages = async ( + args: { + from?: number + size?: number + sort?: SortParams + query?: string + inFilter?: InFilter + readFilter?: ReadFilter + typeFilter?: PageType + labelFilters?: LabelFilter[] + hasFilters?: HasFilter[] + savedDateFilter?: DateRangeFilter + publishedDateFilter?: DateRangeFilter + }, + userId: string +): Promise<[Page[], number] | undefined> => { + try { + const { + from = 0, + size = 10, + sort, + query, + readFilter = ReadFilter.ALL, + typeFilter, + labelFilters = [], + inFilter = InFilter.ALL, + hasFilters = [], + savedDateFilter, + publishedDateFilter, + } = args + // default order is descending + const sortOrder = sort?.order || SortOrder.DESCENDING + // default sort by saved_at + const sortField = sort?.by || SortBy.SAVED + const includeLabels = labelFilters.filter( + (filter) => filter.type === LabelFilterType.INCLUDE + ) + const excludeLabels = labelFilters.filter( + (filter) => filter.type === LabelFilterType.EXCLUDE + ) + + const body: SearchBody = { + query: { + bool: { + filter: [ + { + term: { + userId, + }, + }, + ], + should: [], + must_not: [], + }, + }, + sort: [ + { + [sortField]: { + order: sortOrder, + }, + }, + ], + from, + size, + _source: { + excludes: ['originalHtml', 'content', 'highlights'], + }, + } + + // append filters + if (query) { + appendQuery(body, query) + } + if (typeFilter) { + appendTypeFilter(body, typeFilter) + } + if (inFilter !== InFilter.ALL) { + appendInFilter(body, inFilter) + } + if (readFilter !== ReadFilter.ALL) { + appendReadFilter(body, readFilter) + } + if (hasFilters.length > 0) { + appendHasFilters(body, hasFilters) + } + if (includeLabels.length > 0) { + appendIncludeLabelFilter(body, includeLabels) + } + if (excludeLabels.length > 0) { + appendExcludeLabelFilter(body, excludeLabels) + } + if (savedDateFilter) { + appendSavedDateFilter(body, savedDateFilter) + } + if (publishedDateFilter) { + appendPublishedDateFilter(body, publishedDateFilter) + } + + console.log('searching pages in elastic', JSON.stringify(body)) + + const response = await client.search, SearchBody>({ + index: INDEX_ALIAS, + body, + }) + + if (response.body.hits.total.value === 0) { + return [[], 0] + } + + return [ + response.body.hits.hits.map((hit: { _source: Page; _id: string }) => ({ + ...hit._source, + content: '', + id: hit._id, + })), + response.body.hits.total.value, + ] + } catch (e) { + console.error('failed to search pages in elastic', e) + return undefined + } +} + +export const countByCreatedAt = async ( + userId: string, + from?: number, + to?: number +): Promise => { + try { + const { body } = await client.count({ + index: INDEX_ALIAS, + body: { + query: { + bool: { + filter: [ + { + term: { + userId, + }, + }, + { + range: { + createdAt: { + gte: from, + lte: to, + }, + }, + }, + ], + }, + }, + }, + }) + + return body.count as number + } catch (e) { + console.error('failed to count pages in elastic', e) + return 0 + } +} diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 0ec857678..9d9e5f74a 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -1,5 +1,4 @@ // Define the type of the body for the Search request -import { Label, PageType } from '../generated/graphql' import { PickTuple } from '../util' import { PubsubClient } from '../datalayer/pubsub' @@ -19,6 +18,16 @@ export interface SearchBody { readingProgress: { gte: number } | { lt: number } } } + | { + range: { + savedAt: { gt: Date | undefined } | { lt: Date | undefined } + } + } + | { + range: { + publishedAt: { gt: Date | undefined } | { lt: Date | undefined } + } + } | { nested: { path: 'labels' @@ -33,6 +42,16 @@ export interface SearchBody { } } } + | { + nested: { + path: 'highlights' + query: { + exists: { + field: 'highlights' + } + } + } + } )[] should: { multi_match: { @@ -109,7 +128,7 @@ export interface SearchResponse { _explanation?: Explanation fields?: never highlight?: never - inner_hits?: never + inner_hits?: any matched_queries?: string[] sort?: string[] }> @@ -117,6 +136,38 @@ export interface SearchResponse { aggregations?: never } +export enum PageType { + Article = 'ARTICLE', + Book = 'BOOK', + File = 'FILE', + Profile = 'PROFILE', + Unknown = 'UNKNOWN', + Website = 'WEBSITE', + Highlights = 'HIGHLIGHTS', +} + +export interface Label { + id: string + name: string + color: string + description?: string + createdAt?: Date +} + +export interface Highlight { + id: string + shortId: string + patch: string + quote: string + userId: string + createdAt: Date + prefix?: string | null + suffix?: string | null + annotation?: string | null + sharedAt?: Date | null + updatedAt?: Date | null +} + export interface Page { id: string userId: string @@ -143,6 +194,29 @@ export interface Page { siteName?: string _id?: string siteIcon?: string + highlights?: Highlight[] +} + +export interface SearchItem { + annotation?: string | null + author?: string | null + createdAt: Date + description?: string | null + id: string + image?: string | null + pageId?: string + pageType: PageType + publishedAt?: Date + quote?: string | null + shortId?: string | null + slug: string + title: string + uploadFileId?: string | null + url: string + archivedAt?: Date | null + readingProgressPercent?: number + readingProgressAnchorIndex?: number + userId: string } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId'] as const diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 3d392ec03..872991d7f 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -615,7 +615,6 @@ export type GoogleSignupSuccess = { export type Highlight = { __typename?: 'Highlight'; annotation?: Maybe; - article: Article; createdAt: Scalars['Date']; createdByMe: Scalars['Boolean']; id: Scalars['ID']; @@ -1087,6 +1086,7 @@ export enum PageType { Article = 'ARTICLE', Book = 'BOOK', File = 'FILE', + Highlights = 'HIGHLIGHTS', Profile = 'PROFILE', Unknown = 'UNKNOWN', Website = 'WEBSITE' @@ -1120,6 +1120,7 @@ export type Query = { me?: Maybe; newsletterEmails: NewsletterEmailsResult; reminder: ReminderResult; + search: SearchResult; sharedArticle: SharedArticleResult; user: UserResult; users: UsersResult; @@ -1170,6 +1171,13 @@ export type QueryReminderArgs = { }; +export type QuerySearchArgs = { + after?: InputMaybe; + first?: InputMaybe; + query?: InputMaybe; +}; + + export type QuerySharedArticleArgs = { selectedHighlightId?: InputMaybe; slug: Scalars['String']; @@ -1323,6 +1331,55 @@ export type SaveUrlInput = { url: Scalars['String']; }; +export type SearchError = { + __typename?: 'SearchError'; + errorCodes: Array; +}; + +export enum SearchErrorCode { + Unauthorized = 'UNAUTHORIZED' +} + +export type SearchItem = { + __typename?: 'SearchItem'; + annotation?: Maybe; + author?: Maybe; + contentReader: ContentReader; + createdAt: Scalars['Date']; + description?: Maybe; + id: Scalars['ID']; + image?: Maybe; + isArchived: Scalars['Boolean']; + labels?: Maybe>; + originalArticleUrl?: Maybe; + ownedByViewer?: Maybe; + pageId?: Maybe; + pageType: PageType; + publishedAt?: Maybe; + quote?: Maybe; + readingProgressAnchorIndex?: Maybe; + readingProgressPercent?: Maybe; + shortId?: Maybe; + slug: Scalars['String']; + title: Scalars['String']; + uploadFileId?: Maybe; + url: Scalars['String']; +}; + +export type SearchItemEdge = { + __typename?: 'SearchItemEdge'; + cursor: Scalars['String']; + node: SearchItem; +}; + +export type SearchResult = SearchError | SearchSuccess; + +export type SearchSuccess = { + __typename?: 'SearchSuccess'; + edges: Array; + pageInfo: PageInfo; +}; + export type SetBookmarkArticleError = { __typename?: 'SetBookmarkArticleError'; errorCodes: Array; @@ -2094,6 +2151,12 @@ export type ResolversTypes = { SaveResult: ResolversTypes['SaveError'] | ResolversTypes['SaveSuccess']; SaveSuccess: ResolverTypeWrapper; SaveUrlInput: SaveUrlInput; + SearchError: ResolverTypeWrapper; + SearchErrorCode: SearchErrorCode; + SearchItem: ResolverTypeWrapper; + SearchItemEdge: ResolverTypeWrapper; + SearchResult: ResolversTypes['SearchError'] | ResolversTypes['SearchSuccess']; + SearchSuccess: ResolverTypeWrapper; SetBookmarkArticleError: ResolverTypeWrapper; SetBookmarkArticleErrorCode: SetBookmarkArticleErrorCode; SetBookmarkArticleInput: SetBookmarkArticleInput; @@ -2346,6 +2409,11 @@ export type ResolversParentTypes = { SaveResult: ResolversParentTypes['SaveError'] | ResolversParentTypes['SaveSuccess']; SaveSuccess: SaveSuccess; SaveUrlInput: SaveUrlInput; + SearchError: SearchError; + SearchItem: SearchItem; + SearchItemEdge: SearchItemEdge; + SearchResult: ResolversParentTypes['SearchError'] | ResolversParentTypes['SearchSuccess']; + SearchSuccess: SearchSuccess; SetBookmarkArticleError: SetBookmarkArticleError; SetBookmarkArticleInput: SetBookmarkArticleInput; SetBookmarkArticleResult: ResolversParentTypes['SetBookmarkArticleError'] | ResolversParentTypes['SetBookmarkArticleSuccess']; @@ -2864,7 +2932,6 @@ export type GoogleSignupSuccessResolvers = { annotation?: Resolver, ParentType, ContextType>; - article?: Resolver; createdAt?: Resolver; createdByMe?: Resolver; id?: Resolver; @@ -3100,6 +3167,7 @@ export type QueryResolvers, ParentType, ContextType>; newsletterEmails?: Resolver; reminder?: Resolver>; + search?: Resolver>; sharedArticle?: Resolver>; user?: Resolver>; users?: Resolver; @@ -3180,6 +3248,53 @@ export type SaveSuccessResolvers; }; +export type SearchErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type SearchItemResolvers = { + annotation?: Resolver, ParentType, ContextType>; + author?: Resolver, ParentType, ContextType>; + contentReader?: Resolver; + createdAt?: Resolver; + description?: Resolver, ParentType, ContextType>; + id?: Resolver; + image?: Resolver, ParentType, ContextType>; + isArchived?: Resolver; + labels?: Resolver>, ParentType, ContextType>; + originalArticleUrl?: Resolver, ParentType, ContextType>; + ownedByViewer?: Resolver, ParentType, ContextType>; + pageId?: Resolver, ParentType, ContextType>; + pageType?: Resolver; + publishedAt?: Resolver, ParentType, ContextType>; + quote?: Resolver, ParentType, ContextType>; + readingProgressAnchorIndex?: Resolver, ParentType, ContextType>; + readingProgressPercent?: Resolver, ParentType, ContextType>; + shortId?: Resolver, ParentType, ContextType>; + slug?: Resolver; + title?: Resolver; + uploadFileId?: Resolver, ParentType, ContextType>; + url?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type SearchItemEdgeResolvers = { + cursor?: Resolver; + node?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type SearchResultResolvers = { + __resolveType: TypeResolveFn<'SearchError' | 'SearchSuccess', ParentType, ContextType>; +}; + +export type SearchSuccessResolvers = { + edges?: Resolver, ParentType, ContextType>; + pageInfo?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type SetBookmarkArticleErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -3620,6 +3735,11 @@ export type Resolvers = { SaveError?: SaveErrorResolvers; SaveResult?: SaveResultResolvers; SaveSuccess?: SaveSuccessResolvers; + SearchError?: SearchErrorResolvers; + SearchItem?: SearchItemResolvers; + SearchItemEdge?: SearchItemEdgeResolvers; + SearchResult?: SearchResultResolvers; + SearchSuccess?: SearchSuccessResolvers; SetBookmarkArticleError?: SetBookmarkArticleErrorResolvers; SetBookmarkArticleResult?: SetBookmarkArticleResultResolvers; SetBookmarkArticleSuccess?: SetBookmarkArticleSuccessResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 74a59f9de..29d101a6c 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -540,7 +540,6 @@ type GoogleSignupSuccess { type Highlight { annotation: String - article: Article! createdAt: Date! createdByMe: Boolean! id: ID! @@ -788,6 +787,7 @@ enum PageType { ARTICLE BOOK FILE + HIGHLIGHTS PROFILE UNKNOWN WEBSITE @@ -819,6 +819,7 @@ type Query { me: User newsletterEmails: NewsletterEmailsResult! reminder(linkId: ID!): ReminderResult! + search(after: String, first: Int, query: String): SearchResult! sharedArticle(selectedHighlightId: String, slug: String!, username: String!): SharedArticleResult! user(userId: ID, username: String): UserResult! users: UsersResult! @@ -951,6 +952,51 @@ input SaveUrlInput { url: String! } +type SearchError { + errorCodes: [SearchErrorCode!]! +} + +enum SearchErrorCode { + UNAUTHORIZED +} + +type SearchItem { + annotation: String + author: String + contentReader: ContentReader! + createdAt: Date! + description: String + id: ID! + image: String + isArchived: Boolean! + labels: [Label!] + originalArticleUrl: String + ownedByViewer: Boolean + pageId: ID + pageType: PageType! + publishedAt: Date + quote: String + readingProgressAnchorIndex: Int + readingProgressPercent: Float + shortId: String + slug: String! + title: String! + uploadFileId: ID + url: String! +} + +type SearchItemEdge { + cursor: String! + node: SearchItem! +} + +union SearchResult = SearchError | SearchSuccess + +type SearchSuccess { + edges: [SearchItemEdge!]! + pageInfo: PageInfo! +} + type SetBookmarkArticleError { errorCodes: [SetBookmarkArticleErrorCode!]! } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index b85ae7d05..aed133929 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -22,10 +22,14 @@ import { PageType, QueryArticleArgs, QueryArticlesArgs, + QuerySearchArgs, ResolverFn, SaveArticleReadingProgressError, SaveArticleReadingProgressErrorCode, SaveArticleReadingProgressSuccess, + SearchError, + SearchItem, + SearchSuccess, SetBookmarkArticleError, SetBookmarkArticleErrorCode, SetBookmarkArticleSuccess, @@ -67,6 +71,8 @@ import { createPageSaveRequest } from '../../services/create_page_save_request' import { createIntercomEvent } from '../../utils/intercom' import { analytics } from '../../utils/analytics' import { env } from '../../env' + +import { Page, SearchItem as SearchItemData } from '../../elastic/types' import { createPage, deletePage, @@ -74,8 +80,8 @@ import { getPageByParam, searchPages, updatePage, -} from '../../elastic' -import { Page } from '../../elastic/types' +} from '../../elastic/pages' +import { searchHighlights } from '../../elastic/highlights' export type PartialArticle = Omit< Article, @@ -438,44 +444,45 @@ export const getArticlesResolver = authorized< ArticlesError, QueryArticlesArgs >(async (_obj, params, { claims }) => { - const notNullField = params.sharedOnly ? 'sharedAt' : null const startCursor = params.after || '' const first = params.first || 10 - // Perform basic sanitization. Right now we just allow alphanumeric, space and quote - // so queries can contain phrases like "human race"; - // We can also split out terms like "label:unread". const searchQuery = parseSearchQuery(params.query || undefined) analytics.track({ userId: claims.uid, - event: 'search', + event: 'get_articles', properties: { + env: env.server.apiEnv, query: searchQuery.query, inFilter: searchQuery.inFilter, readFilter: searchQuery.readFilter, typeFilter: searchQuery.typeFilter, labelFilters: searchQuery.labelFilters, sortParams: searchQuery.sortParams, - env: env.server.apiEnv, + hasFilters: searchQuery.hasFilters, + savedDateFilter: searchQuery.savedDateFilter, + publishedDateFilter: searchQuery.publishedDateFilter, }, }) - await createIntercomEvent('search', claims.uid) + await createIntercomEvent('get_articles', claims.uid) const [pages, totalCount] = (await searchPages( { from: Number(startCursor), size: first + 1, // fetch one more item to get next cursor - sort: searchQuery.sortParams || params.sort || undefined, + sort: searchQuery.sortParams, query: searchQuery.query, inFilter: searchQuery.inFilter, readFilter: searchQuery.readFilter, typeFilter: searchQuery.typeFilter, labelFilters: searchQuery.labelFilters, + hasFilters: searchQuery.hasFilters, + savedDateFilter: searchQuery.savedDateFilter, + publishedDateFilter: searchQuery.publishedDateFilter, }, - claims.uid, - notNullField + claims.uid )) || [[], 0] const start = @@ -792,3 +799,106 @@ export const getReadingProgressAnchorIndexForArticleResolver: ResolverFn< return articleReadingProgressAnchorIndex || 0 } + +export const searchResolver = authorized< + SearchSuccess, + SearchError, + QuerySearchArgs +>(async (_obj, params, { claims }) => { + const startCursor = params.after || '' + const first = params.first || 10 + + const searchQuery = parseSearchQuery(params.query || undefined) + + analytics.track({ + userId: claims.uid, + event: 'search', + properties: { + query: searchQuery.query, + inFilter: searchQuery.inFilter, + readFilter: searchQuery.readFilter, + typeFilter: searchQuery.typeFilter, + labelFilters: searchQuery.labelFilters, + sortParams: searchQuery.sortParams, + hasFilters: searchQuery.hasFilters, + savedDateFilter: searchQuery.savedDateFilter, + publishedDateFilter: searchQuery.publishedDateFilter, + env: env.server.apiEnv, + }, + }) + + await createIntercomEvent('search', claims.uid) + + let results: (SearchItemData | Page)[] + let totalCount: number + + const searchType = searchQuery.typeFilter + // search highlights if type:highlights + if (searchType === PageType.Highlights) { + ;[results, totalCount] = (await searchHighlights( + { + from: Number(startCursor), + size: first + 1, // fetch one more item to get next cursor + sort: searchQuery.sortParams, + query: searchQuery.query, + }, + claims.uid + )) || [[], 0] + } else { + // otherwise, search pages + ;[results, totalCount] = (await searchPages( + { + from: Number(startCursor), + size: first + 1, // fetch one more item to get next cursor + sort: searchQuery.sortParams, + query: searchQuery.query, + inFilter: searchQuery.inFilter, + readFilter: searchQuery.readFilter, + typeFilter: searchQuery.typeFilter, + labelFilters: searchQuery.labelFilters, + hasFilters: searchQuery.hasFilters, + savedDateFilter: searchQuery.savedDateFilter, + publishedDateFilter: searchQuery.publishedDateFilter, + }, + claims.uid + )) || [[], 0] + } + + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const hasNextPage = results.length > first + const endCursor = String(start + results.length - (hasNextPage ? 1 : 0)) + + if (hasNextPage) { + // remove an extra if exists + results.pop() + } + + const edges = results.map((r) => { + return { + node: { + ...r, + image: r.image && createImageProxyUrl(r.image, 88, 88), + isArchived: !!r.archivedAt, + contentReader: + r.pageType === PageType.File ? ContentReader.Pdf : ContentReader.Web, + originalArticleUrl: r.url, + publishedAt: validatedDate(r.publishedAt), + ownedByViewer: r.userId === claims.uid, + pageType: r.pageType || PageType.Highlights, + } as SearchItem, + cursor: endCursor, + } + }) + + return { + edges, + pageInfo: { + hasPreviousPage: false, + startCursor, + hasNextPage: hasNextPage, + endCursor, + totalCount, + }, + } +}) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 000ccf22e..7a99e06c0 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -16,6 +16,7 @@ import { LinkShareInfo, PageType, Reaction, + SearchItem, User, } from './../generated/graphql' @@ -54,6 +55,7 @@ import { saveFileResolver, savePageResolver, saveUrlResolver, + searchResolver, setBookmarkArticleResolver, setDeviceTokenResolver, setFollowResolver, @@ -78,7 +80,7 @@ import { generateDownloadSignedUrl, generateUploadFilePathName, } from '../utils/uploads' -import { getPageById, getPageByParam } from '../elastic' +import { getPageByParam } from '../elastic/pages' import { generateApiKeyResolver } from './api_key' /* eslint-disable @typescript-eslint/naming-convention */ @@ -157,6 +159,7 @@ export const functionResolvers = { newsletterEmails: newsletterEmailsResolver, reminder: reminderResolver, labels: labelsResolver, + search: searchResolver, }, User: { async sharedArticles( @@ -394,32 +397,33 @@ export const functionResolvers = { : ContentReader.Web }, async highlights( - article: { id: string; userId?: string }, + article: { id: string; userId?: string; highlights?: Highlight[] }, _: { input: ArticleHighlightsInput }, ctx: WithDataSourcesContext ) { - const includeFriends = false - // TODO: this is a temporary solution until we figure out how collaborative approach would look like - // article has userId only if it's returned by getSharedArticle resolver - if (article.userId) { - const result = await ctx.models.highlight.getForUserArticle( - article.userId, - article.id - ) - return result - } - - const friendsIds = - ctx.claims?.uid && includeFriends - ? await ctx.models.userFriends.getFriends(ctx.claims?.uid) - : [] - - // FIXME: Move this filtering logic to the datalayer - return (await ctx.models.highlight.batchGet(article.id)).filter((h) => - [...(includeFriends ? friendsIds : []), ctx.claims?.uid || ''].some( - (u) => u === h.userId - ) - ) + // const includeFriends = false + // // TODO: this is a temporary solution until we figure out how collaborative approach would look like + // // article has userId only if it's returned by getSharedArticle resolver + // if (article.userId) { + // const result = await ctx.models.highlight.getForUserArticle( + // article.userId, + // article.id + // ) + // return result + // } + // + // const friendsIds = + // ctx.claims?.uid && includeFriends + // ? await ctx.models.userFriends.getFriends(ctx.claims?.uid) + // : [] + // + // // FIXME: Move this filtering logic to the datalayer + // return (await ctx.models.highlight.batchGet(article.id)).filter((h) => + // [...(includeFriends ? friendsIds : []), ctx.claims?.uid || ''].some( + // (u) => u === h.userId + // ) + // ) + return article.highlights || [] }, async shareInfo( article: { id: string; sharedBy?: User; shareInfo?: LinkShareInfo }, @@ -447,9 +451,6 @@ export const functionResolvers = { }, }, Highlight: { - async article(highlight: { articleId: string }, __: unknown) { - return getPageById(highlight.articleId) - }, async user( highlight: { userId: string }, __: unknown, @@ -484,6 +485,19 @@ export const functionResolvers = { return userDataToUser(await ctx.models.user.get(reaction.userId)) }, }, + SearchItem: { + async url(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { + if (item.pageType == PageType.File && ctx.claims && item.uploadFileId) { + const upload = await ctx.models.uploadFile.get(item.uploadFileId) + if (!upload || !upload.fileName) { + return undefined + } + const filePath = generateUploadFilePathName(upload.id, upload.fileName) + return generateDownloadSignedUrl(filePath) + } + return item.url + }, + }, ...resultResolveTypeResolver('Login'), ...resultResolveTypeResolver('LogOut'), ...resultResolveTypeResolver('GoogleSignup'), @@ -532,4 +546,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('Signup'), ...resultResolveTypeResolver('SetLabels'), ...resultResolveTypeResolver('GenerateApiKey'), + ...resultResolveTypeResolver('Search'), } diff --git a/packages/api/src/resolvers/highlight/index.ts b/packages/api/src/resolvers/highlight/index.ts index 69163de53..d14419c03 100644 --- a/packages/api/src/resolvers/highlight/index.ts +++ b/packages/api/src/resolvers/highlight/index.ts @@ -3,7 +3,6 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ import { authorized } from '../../utils/helpers' import { - Article, CreateHighlightError, CreateHighlightErrorCode, CreateHighlightSuccess, @@ -27,15 +26,20 @@ import { UpdateHighlightSuccess, User, } from '../../generated/graphql' -import { HighlightData } from '../../datalayer/highlight/model' import { env } from '../../env' import { analytics } from '../../utils/analytics' -import { getPageById } from '../../elastic' +import { Highlight as HighlightData } from '../../elastic/types' +import { getPageById, updatePage } from '../../elastic/pages' +import { + addHighlightToPage, + deleteHighlight, + getHighlightById, + updateHighlight, +} from '../../elastic/highlights' const highlightDataToHighlight = (highlight: HighlightData): Highlight => ({ ...highlight, user: highlight.userId as unknown as User, - article: highlight.articleId as unknown as Article, updatedAt: highlight.updatedAt || highlight.createdAt, replies: [], reactions: [], @@ -46,11 +50,10 @@ export const createHighlightResolver = authorized< CreateHighlightSuccess, CreateHighlightError, MutationCreateHighlightArgs ->(async (_, { input }, { models, claims, log }) => { - const { articleId } = input - const article = await getPageById(articleId) - - if (!article) { +>(async (_, { input }, { claims, log, pubsub }) => { + const { articleId: pageId } = input + const page = await getPageById(pageId) + if (!page) { return { errorCodes: [CreateHighlightErrorCode.NotFound], } @@ -60,7 +63,7 @@ export const createHighlightResolver = authorized< userId: claims.uid, event: 'highlight_created', properties: { - articleId: article.id, + pageId, env: env.server.apiEnv, }, }) @@ -72,12 +75,23 @@ export const createHighlightResolver = authorized< } try { - const highlight = await models.highlight.create({ - ...input, - articleId: undefined, + const highlight: HighlightData = { + updatedAt: new Date(), + createdAt: new Date(), userId: claims.uid, - elasticPageId: article.id, - }) + ...input, + } + + if ( + !(await addHighlightToPage(pageId, highlight, { + pubsub, + uid: claims.uid, + })) + ) { + return { + errorCodes: [CreateHighlightErrorCode.NotFound], + } + } log.info('Creating a new highlight', { highlight, @@ -101,22 +115,27 @@ export const mergeHighlightResolver = authorized< MergeHighlightSuccess, MergeHighlightError, MutationMergeHighlightArgs ->(async (_, { input }, { authTrx, models, claims, log }) => { - const { articleId } = input +>(async (_, { input }, { claims, log, pubsub }) => { + const { articleId: pageId } = input const { overlapHighlightIdList, ...newHighlightInput } = input - const articleHighlights = await models.highlight.batchGet(articleId) - - if (!articleHighlights.length) { + const page = await getPageById(pageId) + if (!page || !page.highlights) { return { errorCodes: [MergeHighlightErrorCode.NotFound], } } + const articleHighlights = page.highlights + /* Compute merged annotation form the order of highlights appearing on page */ const overlapAnnotations: { [id: string]: string } = {} - articleHighlights.forEach((highlight) => { - if (overlapHighlightIdList.includes(highlight.id) && highlight.annotation) { - overlapAnnotations[highlight.id] = highlight.annotation + articleHighlights.forEach((highlight, index) => { + if (overlapHighlightIdList.includes(highlight.id)) { + articleHighlights.splice(index, 1) + + if (highlight.annotation) { + overlapAnnotations[highlight.id] = highlight.annotation + } } }) const mergedAnnotation: string[] = [] @@ -127,17 +146,20 @@ export const mergeHighlightResolver = authorized< }) try { - const highlight = await authTrx(async (tx) => { - await models.highlight.deleteMany(overlapHighlightIdList, tx) - return await models.highlight.create({ - ...newHighlightInput, - articleId: undefined, - annotation: mergedAnnotation ? mergedAnnotation.join('\n') : null, - userId: claims.uid, - elasticPageId: newHighlightInput.articleId, - }) - }) - if (!highlight) { + const highlight: HighlightData = { + ...newHighlightInput, + updatedAt: new Date(), + createdAt: new Date(), + userId: claims.uid, + annotation: mergedAnnotation ? mergedAnnotation.join('\n') : null, + } + + const merged = await updatePage( + pageId, + { highlights: articleHighlights.concat(highlight) }, + { pubsub, uid: claims.uid } + ) + if (!merged) { throw new Error('Failed to create merged highlight') } @@ -147,7 +169,7 @@ export const mergeHighlightResolver = authorized< source: 'resolver', resolver: 'mergeHighlightResolver', uid: claims.uid, - articleId: articleId, + pageId, }, }) @@ -175,9 +197,9 @@ export const updateHighlightResolver = authorized< UpdateHighlightSuccess, UpdateHighlightError, MutationUpdateHighlightArgs ->(async (_, { input }, { authTrx, models, claims, log }) => { +>(async (_, { input }, { pubsub, claims, log }) => { const { highlightId } = input - const highlight = await models.highlight.get(highlightId) + const highlight = await getHighlightById(highlightId) if (!highlight?.id) { return { @@ -197,16 +219,11 @@ export const updateHighlightResolver = authorized< } } - const updatedHighlight = await authTrx((tx) => - models.highlight.update( - highlightId, - { - annotation: input.annotation, - sharedAt: input.sharedAt, - }, - tx - ) - ) + const updatedHighlight: HighlightData = { + ...highlight, + annotation: input.annotation, + updatedAt: new Date(), + } log.info('Updating a highlight', { updatedHighlight, @@ -217,6 +234,17 @@ export const updateHighlightResolver = authorized< }, }) + const updated = await updateHighlight(updatedHighlight, { + pubsub, + uid: claims.uid, + }) + + if (!updated) { + return { + errorCodes: [UpdateHighlightErrorCode.NotFound], + } + } + return { highlight: highlightDataToHighlight(updatedHighlight) } }) @@ -224,8 +252,8 @@ export const deleteHighlightResolver = authorized< DeleteHighlightSuccess, DeleteHighlightError, MutationDeleteHighlightArgs ->(async (_, { highlightId }, { authTrx, models, claims, log }) => { - const highlight = await models.highlight.get(highlightId) +>(async (_, { highlightId }, { claims, log, pubsub }) => { + const highlight = await getHighlightById(highlightId) if (!highlight?.id) { return { @@ -239,18 +267,19 @@ export const deleteHighlightResolver = authorized< } } - const deletedHighlight = await authTrx((tx) => - models.highlight.delete(highlightId, tx) - ) + const deleted = await deleteHighlight(highlightId, { + pubsub, + uid: claims.uid, + }) - if ('error' in deletedHighlight) { + if (!deleted) { return { errorCodes: [DeleteHighlightErrorCode.NotFound], } } log.info('Deleting a highlight', { - deletedHighlight, + highlight, labels: { source: 'resolver', resolver: 'deleteHighlightResolver', @@ -258,15 +287,15 @@ export const deleteHighlightResolver = authorized< }, }) - return { highlight: highlightDataToHighlight(deletedHighlight) } + return { highlight: highlightDataToHighlight(highlight) } }) export const setShareHighlightResolver = authorized< SetShareHighlightSuccess, SetShareHighlightError, MutationSetShareHighlightArgs ->(async (_, { input: { id, share } }, { authTrx, models, claims, log }) => { - const highlight = await models.highlight.get(id) +>(async (_, { input: { id, share } }, { pubsub, claims, log }) => { + const highlight = await getHighlightById(id) if (!highlight?.id) { return { @@ -287,16 +316,22 @@ export const setShareHighlightResolver = authorized< labels: { source: 'resolver', resolver: 'setShareHighlightResolver', - articleId: highlight.articleId, userId: highlight.userId, }, }) - const updatedHighlight = await authTrx((tx) => - models.highlight.update(id, { sharedAt }, tx) - ) + const updatedHighlight: HighlightData = { + ...highlight, + sharedAt, + updatedAt: new Date(), + } - if (!updatedHighlight || 'error' in updatedHighlight) { + const updated = await updateHighlight(updatedHighlight, { + pubsub, + uid: claims.uid, + }) + + if (!updated) { return { errorCodes: [SetShareHighlightErrorCode.NotFound], } diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index 3f428c8f2..e2813861e 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -26,9 +26,10 @@ import { User } from '../../entity/user' import { Label } from '../../entity/label' import { ILike, In } from 'typeorm' import { getRepository, setClaims } from '../../entity/utils' -import { deleteLabelInPages, getPageById, updatePage } from '../../elastic' import { createPubSubClient } from '../../datalayer/pubsub' import { AppDataSource } from '../../server' +import { getPageById, updatePage } from '../../elastic/pages' +import { deleteLabelInPages } from '../../elastic/labels' export const labelsResolver = authorized( async (_obj, _params, { claims: { uid }, log }) => { diff --git a/packages/api/src/resolvers/links/index.ts b/packages/api/src/resolvers/links/index.ts index 0f9705908..48554b69c 100644 --- a/packages/api/src/resolvers/links/index.ts +++ b/packages/api/src/resolvers/links/index.ts @@ -14,7 +14,7 @@ import { import { authorized } from '../../utils/helpers' import { analytics } from '../../utils/analytics' import { env } from '../../env' -import { updatePage } from '../../elastic' +import { updatePage } from '../../elastic/pages' export const updateLinkShareInfoResolver = authorized< UpdateLinkShareInfoSuccess, diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index ec89afb85..efde65410 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -6,8 +6,8 @@ import { createPubSubClient, readPushSubscription, } from '../../datalayer/pubsub' -import { getPageByParam, updatePage } from '../../elastic' import { Page } from '../../elastic/types' +import { getPageByParam, updatePage } from '../../elastic/pages' interface UpdateContentMessage { fileId: string diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index ba672a5b2..f2ff1a5cb 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -14,9 +14,9 @@ import { analytics } from '../../utils/analytics' import { getNewsletterEmail } from '../../services/newsletters' import { setClaims } from '../../datalayer/helpers' import { generateSlug } from '../../utils/helpers' -import { createPage } from '../../elastic' import { createPubSubClient } from '../../datalayer/pubsub' import { Page } from '../../elastic/types' +import { createPage } from '../../elastic/pages' export function pdfAttachmentsRouter() { const router = express.Router() diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index f18a3786b..2e4ecd4f3 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -286,6 +286,7 @@ const schema = gql` FILE PROFILE WEBSITE + HIGHLIGHTS UNKNOWN } @@ -569,7 +570,6 @@ const schema = gql` # used for simplified url format shortId: String! user: User! - article: Article! quote: String! # piece of content before the quote prefix: String @@ -1393,6 +1393,56 @@ const schema = gql` BAD_REQUEST } + # Query: search + union SearchResult = SearchSuccess | SearchError + + type SearchItem { + # used for pages + id: ID! + title: String! + slug: String! + # for uploaded file articles (PDFs), the URL here is the saved omnivore link in GCS + url: String! + pageType: PageType! + contentReader: ContentReader! + createdAt: Date! + isArchived: Boolean! + readingProgressPercent: Float + readingProgressAnchorIndex: Int + author: String + image: String + description: String + publishedAt: Date + ownedByViewer: Boolean + # for uploaded file articles (PDFs), we track the original article URL separately! + originalArticleUrl: String + uploadFileId: ID + # used for highlights + pageId: ID + shortId: String + quote: String + annotation: String + labels: [Label!] + } + + type SearchItemEdge { + cursor: String! + node: SearchItem! + } + + type SearchSuccess { + edges: [SearchItemEdge!]! + pageInfo: PageInfo! + } + + enum SearchErrorCode { + UNAUTHORIZED + } + + type SearchError { + errorCodes: [SearchErrorCode!]! + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -1490,6 +1540,7 @@ const schema = gql` newsletterEmails: NewsletterEmailsResult! reminder(linkId: ID!): ReminderResult! labels: LabelsResult! + search(after: String, first: Int, query: String): SearchResult! } ` diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index be4a81b16..60d82fc69 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -9,7 +9,7 @@ import { } from '../generated/graphql' import { articleSavingRequestDataToArticleSavingRequest } from '../utils/helpers' import * as privateIpLib from 'private-ip' -import { countByCreatedAt } from '../elastic' +import { countByCreatedAt } from '../elastic/pages' const isPrivateIP = privateIpLib.default diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 5b3be6915..7b908dd7f 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -2,7 +2,7 @@ import { Label } from '../entity/label' import { ILike, In } from 'typeorm' import { PageContext } from '../elastic/types' import { User } from '../entity/user' -import { addLabelInPage } from '../elastic' +import { addLabelInPage } from '../elastic/labels' import { getRepository } from '../entity/utils' import { Link } from '../entity/link' import DataLoader from 'dataloader' diff --git a/packages/api/src/services/reports.ts b/packages/api/src/services/reports.ts index ed545b075..5f39b669d 100644 --- a/packages/api/src/services/reports.ts +++ b/packages/api/src/services/reports.ts @@ -1,7 +1,7 @@ import { ReportItemInput, ReportType } from '../generated/graphql' import { ContentDisplayReport } from '../entity/reports/content_display_report' import { AbuseReport } from '../entity/reports/abuse_report' -import { getPageById } from '../elastic' +import { getPageById } from '../elastic/pages' import { getRepository } from '../entity/utils' export const saveContentDisplayReport = async ( diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index a733f84be..6f04b5d6c 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -7,7 +7,7 @@ import { import normalizeUrl from 'normalize-url' import { PubsubClient } from '../datalayer/pubsub' import { Page } from '../elastic/types' -import { createPage, getPageByParam, updatePage } from '../elastic' +import { createPage, getPageByParam, updatePage } from '../elastic/pages' export type SaveContext = { pubsub: PubsubClient diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index 592cdd064..dabf5f09a 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -13,7 +13,7 @@ import { DataModels } from '../resolvers/types' import { generateSlug } from '../utils/helpers' import { getStorageFileDetails, makeStorageFilePublic } from '../utils/uploads' import { createSavingRequest } from './save_page' -import { createPage, getPageByParam, updatePage } from '../elastic' +import { createPage, getPageByParam, updatePage } from '../elastic/pages' type SaveContext = { pubsub: PubsubClient diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index fb97d0fd6..9feecffc4 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -14,8 +14,8 @@ import normalizeUrl from 'normalize-url' import { createPageSaveRequest } from './create_page_save_request' import { kx } from '../datalayer/knex_config' import { setClaims } from '../datalayer/helpers' -import { createPage, getPageByParam, updatePage } from '../elastic' import { Page } from '../elastic/types' +import { createPage, getPageByParam, updatePage } from '../elastic/pages' type SaveContext = { pubsub: PubsubClient diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index b0f914607..5ffbc44b8 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -9,7 +9,7 @@ import { SearchParserKeyWordOffset, SearchParserTextOffset, } from 'search-query-parser' -import { PageType, SortBy, SortOrder, SortParams } from '../generated/graphql' +import { PageType } from '../generated/graphql' export enum ReadFilter { ALL, @@ -27,9 +27,12 @@ export type SearchFilter = { query: string | undefined inFilter: InFilter readFilter: ReadFilter - typeFilter?: PageType | undefined + typeFilter?: PageType labelFilters: LabelFilter[] sortParams?: SortParams + hasFilters: HasFilter[] + savedDateFilter?: DateRangeFilter + publishedDateFilter?: DateRangeFilter } export enum LabelFilterType { @@ -42,6 +45,32 @@ export type LabelFilter = { labels: string[] } +export enum HasFilter { + HIGHLIGHTS, + SHARED_AT, +} + +export type DateRangeFilter = { + startDate?: Date + endDate?: Date +} + +export enum SortBy { + SAVED = 'savedAt', + UPDATED = 'updatedAt', + SCORE = '_score', +} + +export enum SortOrder { + ASCENDING = 'asc', + DESCENDING = 'desc', +} + +export type SortParams = { + by: SortBy + order?: SortOrder +} + const parseIsFilter = (str: string | undefined): ReadFilter => { switch (str?.toUpperCase()) { case 'READ': @@ -72,7 +101,7 @@ const parseTypeFilter = (str: string | undefined): PageType | undefined => { return undefined } - switch (str) { + switch (str.toLowerCase()) { case 'article': return PageType.Article case 'book': @@ -86,6 +115,8 @@ const parseTypeFilter = (str: string | undefined): PageType | undefined => { return PageType.Website case 'unknown': return PageType.Unknown + case 'highlights': + return PageType.Highlights } return undefined } @@ -115,25 +146,55 @@ const parseSortParams = (str?: string): SortParams | undefined => { return undefined } - const [sort, order] = str.split(':') + const [sort, order] = str.split('-') const sortOrder = - order?.toUpperCase() === 'ASC' ? SortOrder.Ascending : SortOrder.Descending + order?.toUpperCase() === 'ASC' ? SortOrder.ASCENDING : SortOrder.DESCENDING switch (sort.toUpperCase()) { - case 'UPDATED_TIME': - case 'SAVED_AT': + case 'UPDATED': return { - by: SortBy.SavedAt, + by: SortBy.UPDATED, + order: sortOrder, + } + case 'SAVED': + return { + by: SortBy.SAVED, order: sortOrder, } case 'SCORE': // sort by score does not need an order return { - by: SortBy.Score, + by: SortBy.SCORE, } } } +const parseHasFilter = (str?: string): HasFilter | undefined => { + if (str === undefined) { + return undefined + } + + switch (str.toUpperCase()) { + case 'HIGHLIGHTS': + return HasFilter.HIGHLIGHTS + } +} + +const parseDateRangeFilter = (str?: string): DateRangeFilter | undefined => { + if (str === undefined) { + return undefined + } + + const [start, end] = str.split('..') + const startDate = start && start !== '*' ? new Date(start) : undefined + const endDate = end && end !== '*' ? new Date(end) : undefined + + return { + startDate, + endDate, + } +} + export const parseSearchQuery = (query: string | undefined): SearchFilter => { const searchQuery = query ? query.replace(/\W\s":/g, '') : undefined const result: SearchFilter = { @@ -141,6 +202,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { readFilter: ReadFilter.ALL, inFilter: searchQuery ? InFilter.ALL : InFilter.INBOX, labelFilters: [], + hasFilters: [], } if (!searchQuery) { @@ -149,11 +211,21 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { inFilter: InFilter.INBOX, readFilter: ReadFilter.ALL, labelFilters: [], + hasFilters: [], } } const parsed = parse(searchQuery, { - keywords: ['in', 'is', 'type', 'label', 'sort'], + keywords: [ + 'in', + 'is', + 'type', + 'label', + 'sort', + 'has', + 'saved', + 'published', + ], tokenize: true, }) if (parsed.offsets) { @@ -197,10 +269,20 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { labelFilter && result.labelFilters.push(labelFilter) break } - case 'sort': { + case 'sort': result.sortParams = parseSortParams(keyword.value) break + case 'has': { + const hasFilter = parseHasFilter(keyword.value) + hasFilter !== undefined && result.hasFilters.push(hasFilter) + break } + case 'saved': + result.savedDateFilter = parseDateRangeFilter(keyword.value) + break + case 'published': + result.publishedDateFilter = parseDateRangeFilter(keyword.value) + break } } } diff --git a/packages/api/test/elastic/index.test.ts b/packages/api/test/elastic/index.test.ts index 29e86d07d..d3d6e73eb 100644 --- a/packages/api/test/elastic/index.test.ts +++ b/packages/api/test/elastic/index.test.ts @@ -1,6 +1,9 @@ import 'mocha' +import { expect } from 'chai' +import { InFilter, ReadFilter } from '../../src/utils/search' +import { Highlight, Page, PageContext, PageType } from '../../src/elastic/types' +import { createPubSubClient } from '../../src/datalayer/pubsub' import { - addLabelInPage, countByCreatedAt, createPage, deletePage, @@ -8,12 +11,12 @@ import { getPageByParam, searchPages, updatePage, -} from '../../src/elastic' -import { PageType } from '../../src/generated/graphql' -import { expect } from 'chai' -import { InFilter, ReadFilter } from '../../src/utils/search' -import { Page, PageContext } from '../../src/elastic/types' -import { createPubSubClient } from '../../src/datalayer/pubsub' +} from '../../src/elastic/pages' +import { addLabelInPage } from '../../src/elastic/labels' +import { + addHighlightToPage, + searchHighlights, +} from '../../src/elastic/highlights' describe('elastic api', () => { const userId = 'userId' @@ -204,4 +207,33 @@ describe('elastic api', () => { expect(count).to.eq(1) }) }) + + describe('searchHighlights', () => { + const highlightId = 'highlightId' + + before(async () => { + const highlightData: Highlight = { + patch: 'test patch', + quote: 'test content', + shortId: 'test shortId', + id: highlightId, + userId: page.userId, + createdAt: new Date(), + } + + await addHighlightToPage(page.id, highlightData, ctx) + }) + + it('searches highlights', async () => { + const [searchResults, count] = (await searchHighlights( + { + query: 'test', + }, + page.userId + )) || [[], 0] + + expect(count).to.eq(1) + expect(searchResults[0].id).to.eq(highlightId) + }) + }) }) diff --git a/packages/api/test/global-setup.ts b/packages/api/test/global-setup.ts index f96afabb6..2025c97f7 100644 --- a/packages/api/test/global-setup.ts +++ b/packages/api/test/global-setup.ts @@ -1,5 +1,6 @@ import { createTestConnection } from './db' import { initElasticsearch } from '../src/elastic' +import { startApolloServer } from './util' export const mochaGlobalSetup = async () => { await createTestConnection() @@ -7,4 +8,7 @@ export const mochaGlobalSetup = async () => { await initElasticsearch() console.log('elasticsearch initialized') + + await startApolloServer() + console.log('apollo server started') } diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts index 259f354a6..ff2b9e6a0 100644 --- a/packages/api/test/global-teardown.ts +++ b/packages/api/test/global-teardown.ts @@ -1,6 +1,12 @@ import { AppDataSource } from '../src/server' +import { stopApolloServer } from './util' +import { kx } from '../src/datalayer/knex_config' export const mochaGlobalTeardown = async () => { await AppDataSource.destroy() + await kx.destroy() console.log('db connection closed') + + await stopApolloServer() + console.log('apollo server stopped') } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 3f0e1a6b5..901329bbe 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -11,17 +11,18 @@ import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' import { Label } from '../../src/entity/label' +import { UploadFileStatus } from '../../src/generated/graphql' +import { Highlight, Page, PageContext, PageType } from '../../src/elastic/types' +import { UploadFile } from '../../src/entity/upload_file' +import { createPubSubClient } from '../../src/datalayer/pubsub' +import { getRepository } from '../../src/entity/utils' import { createPage, deletePage, getPageById, updatePage, -} from '../../src/elastic' -import { PageType, UploadFileStatus } from '../../src/generated/graphql' -import { Page, PageContext } from '../../src/elastic/types' -import { UploadFile } from '../../src/entity/upload_file' -import { createPubSubClient } from '../../src/datalayer/pubsub' -import { getRepository } from '../../src/entity/utils' +} from '../../src/elastic/pages' +import { addHighlightToPage } from '../../src/elastic/highlights' chai.use(chaiString) @@ -85,15 +86,11 @@ const createArticleQuery = ( ` } -const articlesQuery = (after = '', order = 'ASCENDING') => { +const articlesQuery = (after = '') => { return ` query { articles( sharedOnly: ${false} - sort: { - order: ${order} - by: UPDATED_TIME - } after: "${after}" first: 5 query: "") { @@ -136,6 +133,17 @@ const getArticleQuery = (slug: string) => { article { id slug + highlights { + id + shortId + quote + prefix + suffix + patch + annotation + sharedAt + createdAt + } } } ... on ArticleError { @@ -146,6 +154,37 @@ const getArticleQuery = (slug: string) => { ` } +const searchQuery = (keyword = '') => { + return ` + query { + search( + after: "" + first: 5 + query: "${keyword}") { + ... on SearchSuccess { + edges { + cursor + node { + id + url + } + } + pageInfo { + hasNextPage + hasPreviousPage + startCursor + endCursor + totalCount + } + } + ... on SearchError { + errorCodes + } + } + } + ` +} + const savePageQuery = (url: string, title: string, originalContent: string) => { return ` mutation { @@ -302,6 +341,7 @@ describe('Article API', () => { describe('GetArticle', () => { const realSlug = 'testing-is-really-fun-with-omnivore' + let query = '' let slug = '' let pageId: string | undefined @@ -321,6 +361,15 @@ describe('Article API', () => { readingProgressAnchorIndex: 0, url: 'https://blog.omnivore.app/test-with-omnivore', savedAt: new Date(), + highlights: [ + { + id: 'test id', + shortId: 'test short id', + createdAt: new Date(), + patch: 'test patch', + quote: 'test quote', + }, + ], } as Page pageId = await createPage(page, ctx) }) @@ -335,19 +384,25 @@ describe('Article API', () => { query = getArticleQuery(slug) }) - context('when article exists', () => { + context('when page exists', () => { before(() => { slug = realSlug }) - it('should return the article', async () => { + it('should return the page', async () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.article.article.slug).to.eql(slug) }) + + it('should return highlights', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.article.article.highlights).to.length(1) + }) }) - context('when article does not exist', () => { + context('when page does not exist', () => { before(() => { slug = 'not-a-real-slug' }) @@ -418,11 +473,6 @@ describe('Article API', () => { }) context('when there are pages with labels', () => { - before(() => { - // get the last page - after = '14' - }) - it('should return labels', async () => { const res = await graphqlRequest(query, authToken).expect(200) @@ -437,15 +487,15 @@ describe('Article API', () => { after = '' }) - it('should return the first five items', async () => { + it('should return the first five items in desc order', async () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.articles.edges.length).to.eql(5) - expect(res.body.data.articles.edges[0].node.id).to.eql(pages[0].id) - expect(res.body.data.articles.edges[1].node.id).to.eql(pages[1].id) - expect(res.body.data.articles.edges[2].node.id).to.eql(pages[2].id) - expect(res.body.data.articles.edges[3].node.id).to.eql(pages[3].id) - expect(res.body.data.articles.edges[4].node.id).to.eql(pages[4].id) + expect(res.body.data.articles.edges[0].node.id).to.eql(pages[14].id) + expect(res.body.data.articles.edges[1].node.id).to.eql(pages[13].id) + expect(res.body.data.articles.edges[2].node.id).to.eql(pages[12].id) + expect(res.body.data.articles.edges[3].node.id).to.eql(pages[11].id) + expect(res.body.data.articles.edges[4].node.id).to.eql(pages[10].id) }) it('should set the pageInfo', async () => { @@ -471,11 +521,11 @@ describe('Article API', () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.articles.edges.length).to.eql(5) - expect(res.body.data.articles.edges[0].node.id).to.eql(pages[5].id) - expect(res.body.data.articles.edges[1].node.id).to.eql(pages[6].id) + expect(res.body.data.articles.edges[0].node.id).to.eql(pages[9].id) + expect(res.body.data.articles.edges[1].node.id).to.eql(pages[8].id) expect(res.body.data.articles.edges[2].node.id).to.eql(pages[7].id) - expect(res.body.data.articles.edges[3].node.id).to.eql(pages[8].id) - expect(res.body.data.articles.edges[4].node.id).to.eql(pages[9].id) + expect(res.body.data.articles.edges[3].node.id).to.eql(pages[6].id) + expect(res.body.data.articles.edges[4].node.id).to.eql(pages[5].id) }) it('should set the pageInfo', async () => { @@ -532,7 +582,7 @@ describe('Article API', () => { // set a slight delay to make sure the page is updated setTimeout(async () => { let allLinks = await graphqlRequest( - articlesQuery('', 'DESCENDING'), + articlesQuery(''), authToken ).expect(200) const justSavedId = allLinks.body.data.articles.edges[0].node.id @@ -541,10 +591,9 @@ describe('Article API', () => { // test the negative case, ensuring the archive link wasn't returned setTimeout(async () => { - allLinks = await graphqlRequest( - articlesQuery('', 'DESCENDING'), - authToken - ).expect(200) + allLinks = await graphqlRequest(articlesQuery(''), authToken).expect( + 200 + ) expect(allLinks.body.data.articles.edges[0].node.url).to.not.eq(url) }, 100) @@ -555,10 +604,9 @@ describe('Article API', () => { ).expect(200) setTimeout(async () => { - allLinks = await graphqlRequest( - articlesQuery('', 'DESCENDING'), - authToken - ).expect(200) + allLinks = await graphqlRequest(articlesQuery(''), authToken).expect( + 200 + ) expect(allLinks.body.data.articles.edges[0].node.url).to.eq(url) }, 100) }) @@ -729,4 +777,91 @@ describe('Article API', () => { }) }) }) + + describe('Search API', () => { + const url = 'https://blog.omnivore.app/p/getting-started-with-omnivore' + const pages: Page[] = [] + const highlights: Highlight[] = [] + + let query = '' + let keyword = '' + + before(async () => { + // Create some test pages + for (let i = 0; i < 5; i++) { + const page: Page = { + id: '', + hash: 'test hash', + userId: user.id, + pageType: PageType.Article, + title: 'test title', + content: '

search page

', + slug: 'test slug', + createdAt: new Date(), + updatedAt: new Date(), + readingProgressPercent: 0, + readingProgressAnchorIndex: 0, + url: url, + savedAt: new Date(), + } + const pageId = await createPage(page, ctx) + if (!pageId) { + expect.fail('Failed to create page') + } + page.id = pageId + pages.push(page) + + // Create some test highlights + const highlight: Highlight = { + id: `highlight-${i}`, + patch: 'test patch', + shortId: 'test shortId', + userId: user.id, + quote: '

search highlight

', + createdAt: new Date(), + updatedAt: new Date(), + } + await addHighlightToPage(pageId, highlight, ctx) + highlights.push(highlight) + } + }) + + beforeEach(async () => { + query = searchQuery(keyword) + }) + + context('when type:highlights is not in the query', () => { + before(() => { + keyword = 'search' + }) + + it('should return pages in descending order', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.edges.length).to.eql(5) + expect(res.body.data.search.edges[0].node.id).to.eq(pages[4].id) + expect(res.body.data.search.edges[1].node.id).to.eq(pages[3].id) + expect(res.body.data.search.edges[2].node.id).to.eq(pages[2].id) + expect(res.body.data.search.edges[3].node.id).to.eq(pages[1].id) + expect(res.body.data.search.edges[4].node.id).to.eq(pages[0].id) + }) + }) + + context('when type:highlights is in the query', () => { + before(() => { + keyword = 'search type:highlights' + }) + + it('should return highlights in descending order', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.edges.length).to.eq(5) + expect(res.body.data.search.edges[0].node.id).to.eq(highlights[4].id) + expect(res.body.data.search.edges[1].node.id).to.eq(highlights[3].id) + expect(res.body.data.search.edges[2].node.id).to.eq(highlights[2].id) + expect(res.body.data.search.edges[3].node.id).to.eq(highlights[1].id) + expect(res.body.data.search.edges[4].node.id).to.eq(highlights[0].id) + }) + }) + }) }) diff --git a/packages/api/test/resolvers/highlight.test.ts b/packages/api/test/resolvers/highlight.test.ts index f76898897..fbd3158e5 100644 --- a/packages/api/test/resolvers/highlight.test.ts +++ b/packages/api/test/resolvers/highlight.test.ts @@ -10,9 +10,9 @@ import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { deletePage } from '../../src/elastic' import { createPubSubClient } from '../../src/datalayer/pubsub' import { PageContext } from '../../src/elastic/types' +import { deletePage } from '../../src/elastic/pages' chai.use(chaiString) diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts index 9ef4fb7d7..e8f4594a0 100644 --- a/packages/api/test/resolvers/labels.test.ts +++ b/packages/api/test/resolvers/labels.test.ts @@ -10,8 +10,8 @@ import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import { Page } from '../../src/elastic/types' -import { getPageById } from '../../src/elastic' import { getRepository } from '../../src/entity/utils' +import { getPageById } from '../../src/elastic/pages' describe('Labels API', () => { const username = 'fakeUser' diff --git a/packages/api/test/routers/pdf_attachments.test.ts b/packages/api/test/routers/pdf_attachments.test.ts index 9e0648fe0..36e606b81 100644 --- a/packages/api/test/routers/pdf_attachments.test.ts +++ b/packages/api/test/routers/pdf_attachments.test.ts @@ -8,7 +8,7 @@ import { User } from '../../src/entity/user' import 'mocha' import * as jwt from 'jsonwebtoken' import { expect } from 'chai' -import { getPageById } from '../../src/elastic' +import { getPageById } from '../../src/elastic/pages' describe('PDF attachments Router', () => { const username = 'fakeUser' diff --git a/packages/api/test/services/save_email.test.ts b/packages/api/test/services/save_email.test.ts index cd4724636..fb06c4f95 100644 --- a/packages/api/test/services/save_email.test.ts +++ b/packages/api/test/services/save_email.test.ts @@ -4,7 +4,7 @@ import 'chai/register-should' import { createTestUser, deleteTestUser } from '../db' import { SaveContext, saveEmail } from '../../src/services/save_email' import { createPubSubClient } from '../../src/datalayer/pubsub' -import { getPageByParam } from '../../src/elastic' +import { getPageByParam } from '../../src/elastic/pages' describe('saveEmail', () => { const username = 'fakeUser' diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index 5fda82add..46d09daa6 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -4,11 +4,11 @@ import 'chai/register-should' import { createTestUser, deleteTestUser } from '../db' import { createNewsletterEmail } from '../../src/services/newsletters' import { saveNewsletterEmail } from '../../src/services/save_newsletter_email' -import { getPageByParam } from '../../src/elastic' import { User } from '../../src/entity/user' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { SaveContext } from '../../src/services/save_email' import { createPubSubClient } from '../../src/datalayer/pubsub' +import { getPageByParam } from '../../src/elastic/pages' describe('saveNewsletterEmail', () => { const username = 'fakeUser' diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index b53342fde..1d09d798c 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -4,18 +4,22 @@ import { v4 } from 'uuid' import { corsConfig } from '../src/utils/corsConfig' import { Page } from '../src/elastic/types' import { PageType } from '../src/generated/graphql' -import { createPage, getPageById } from '../src/elastic' import { User } from '../src/entity/user' import { Label } from '../src/entity/label' import { createPubSubClient } from '../src/datalayer/pubsub' +import { createPage, getPageById } from '../src/elastic/pages' const { app, apollo } = createApp() export const request = supertest(app) -before(async () => { +export const startApolloServer = async () => { await apollo.start() apollo.applyMiddleware({ app, path: '/api/graphql', cors: corsConfig }) -}) +} + +export const stopApolloServer = async () => { + await apollo.stop() +} export const graphqlRequest = ( query: string,