diff --git a/packages/api/package.json b/packages/api/package.json index 53618a156..5bf2ca441 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -71,8 +71,6 @@ "intercom-client": "^3.1.4", "jsonwebtoken": "^8.5.1", "jwks-rsa": "^2.0.3", - "knex": "2.4.2", - "knex-stringcase": "^1.4.2", "linkedom": "^0.14.9", "lodash": "^4.17.21", "luxon": "^3.2.1", diff --git a/packages/api/src/datalayer/article/index.ts b/packages/api/src/datalayer/article/index.ts deleted file mode 100644 index f6235265a..000000000 --- a/packages/api/src/datalayer/article/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { ArticleData, CreateSet, keys as modelKeys, UpdateSet } from './model' -import DataModel from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { logMethod } from '../helpers' - -class ArticleModel extends DataModel { - public tableName = Table.PAGES - protected modelKeys = modelKeys - constructor(kx: Knex, cache = true) { - super(kx, cache) - } - - @logMethod - async getByUrlAndHash( - params: { url: ArticleData['url']; hash: ArticleData['hash'] }, - tx = this.kx - ): Promise { - const row: ArticleData | null = await tx(this.tableName) - .select(this.modelKeys) - .where(params) - .first() - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getByUploadFileId( - uploadFileId: string, - tx = this.kx - ): Promise { - const row: ArticleData | null = await tx(this.tableName) - .select(this.modelKeys) - .where({ uploadFileId }) - .first() - if (!row) return null - this.loader.prime(row.id, row) - return row - } -} - -export default ArticleModel diff --git a/packages/api/src/datalayer/article/model.ts b/packages/api/src/datalayer/article/model.ts deleted file mode 100644 index ceace76b4..000000000 --- a/packages/api/src/datalayer/article/model.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { ContentReader, PageType } from '../../generated/graphql' -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * title | text | | not null | - * description | text | | | - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * published_at | timestamp with time zone | | | - * url | text | | not null | - * hash | text | | not null | - * original_html | text | | | - * content | text | | not null | - * author | text | | | - * image | text | | | - * upload_file_id| uuid reference | | | - * ``` - * */ - -export interface ArticleData { - id: string - title: string - description?: string | null - createdAt: Date - publishedAt?: Date | null - url: string - hash: string - originalHtml?: string | null - content: string - pageType: PageType - author?: string | null - image?: string | null - uploadFileId?: string | null - contentReader: ContentReader -} - -export const keys = [ - 'id', - 'title', - 'description', - 'createdAt', - 'publishedAt', - 'url', - 'hash', - 'originalHtml', - 'content', - 'pageType', - 'author', - 'image', - 'uploadFileId', -] as const - -export const defaultedKeys = ['id', 'createdAt'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [] as const - -export type UpdateSet = Partialize> diff --git a/packages/api/src/datalayer/article_saving_request/index.ts b/packages/api/src/datalayer/article_saving_request/index.ts deleted file mode 100644 index c9d625622..000000000 --- a/packages/api/src/datalayer/article_saving_request/index.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - ArticleSavingRequestData, - CreateSet, - keys as modelKeys, - UpdateSet, -} from './model' -import DataModel from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { logMethod } from '../helpers' - -class ArticleSavingRequestModel extends DataModel< - ArticleSavingRequestData, - CreateSet, - UpdateSet -> { - public tableName = Table.ARTICLE_SAVING_REQUEST - protected modelKeys = modelKeys - constructor(kx: Knex, cache = true) { - super(kx, cache) - } - - @logMethod - async getByUserId( - userId: ArticleSavingRequestData['userId'], - tx = this.kx - ): Promise { - const row: ArticleSavingRequestData | null = await tx(this.tableName) - .select() - .where({ userId }) - .first(this.modelKeys) - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getByUserIdAndArticleId( - userId: ArticleSavingRequestData['userId'], - articleId: ArticleSavingRequestData['articleId'], - tx = this.kx - ): Promise { - const row: ArticleSavingRequestData | null = await tx(this.tableName) - .select() - .where({ userId, articleId }) - .first(this.modelKeys) - if (!row) return null - this.loader.prime(row.id, row) - return row - } -} - -export default ArticleSavingRequestModel diff --git a/packages/api/src/datalayer/article_saving_request/model.ts b/packages/api/src/datalayer/article_saving_request/model.ts deleted file mode 100644 index 91bc8e4dd..000000000 --- a/packages/api/src/datalayer/article_saving_request/model.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * article_id | uuid | | | - * status | text | | | 'PROCESSING'::text - * error_code | text | | | - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * task_name | text | | | - * ``` - * */ -export interface ArticleSavingRequestData { - id: string - userId: string - articleId?: string | null - status?: string | null - errorCode?: string | null - createdAt: Date - updatedAt: Date - taskName?: string - elasticPageId?: string -} - -export const keys = [ - 'id', - 'userId', - 'articleId', - 'status', - 'errorCode', - 'createdAt', - 'updatedAt', - 'taskName', - 'elasticPageId', -] as const - -export const defaultedKeys = ['id', 'createdAt', 'updatedAt', 'status'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [ - 'articleId', - 'status', - 'errorCode', - 'taskName', - 'elasticPageId', -] as const - -export type UpdateSet = PickTuple - -export const getByParametersKeys = exclude(keys, ['id'] as const) - -export type ParametersSet = PickTuple< - ArticleSavingRequestData, - typeof getByParametersKeys -> diff --git a/packages/api/src/datalayer/helpers.ts b/packages/api/src/datalayer/helpers.ts deleted file mode 100644 index f357f3c6c..000000000 --- a/packages/api/src/datalayer/helpers.ts +++ /dev/null @@ -1,156 +0,0 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import DataModel from './model' -import { Knex } from 'knex' -import DataLoader from 'dataloader' -import { snakeCase } from 'snake-case' -import { buildLogger } from '../utils/logger' -import { SetClaimsRole } from '../utils/dictionary' - -export const logger = buildLogger('datalayer') - -export const setClaims = async ( - tx: Knex.Transaction, - uuid?: string, - userRole?: string -): Promise => { - const uid = uuid || '00000000-0000-0000-0000-000000000000' - const dbRole = - userRole === SetClaimsRole.ADMIN ? 'omnivore_admin' : 'omnivore_user' - return tx.raw('SELECT * from omnivore.set_claims(?, ?)', [uid, dbRole]) -} - -export type Optional = Pick, K> & Omit - -/** - * Set to true to enable DB request statistics collection - * When on, every 5 seconds CallCounter will be printing an object that counts DB calls by types. - * **Must be off on prod**. - * */ -export const ENABLE_DB_REQUEST_LOGGING = false - -/** Doesnt preserve function context */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function debounce( - func: (...args: A) => void, - wait: number, - immediate: boolean -): (...args: A) => void { - let timeout: NodeJS.Timeout | null - return function (...args: A): void { - if (timeout) { - clearTimeout(timeout) - } - timeout = global.setTimeout(function () { - timeout = null - if (!immediate) func(...args) - }, wait) - if (immediate && !timeout) func(...args) - } -} - -class CallCounter { - private counter: Record = {} - - log(tableName: string, methodName: string, params: string): void { - const key = `${tableName}.${methodName}` - if (key in this.counter) { - this.counter[key]++ - } else { - this.counter[key] = 1 - } - const count = this.counter[key] - // display in console the function call details - logger.info(`Call (${count}): ${key}(${params})`, { - labels: { - source: 'callCounter', - }, - }) - this.printCounts() - } - - printCounts = debounce(() => logger.info(this.counter), 5000, false) -} - -export const globalCounter = new CallCounter() - -export function logMethod( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - target: Record, - propertyName: string, - propertyDesciptor: PropertyDescriptor -): PropertyDescriptor { - const method = propertyDesciptor.value - // eslint-disable-next-line @typescript-eslint/no-explicit-any - propertyDesciptor.value = async function ( - this: DataModel, - ...args: any[] - ) { - // invoke wrapped function and get its return value - const result = await method.apply(this, args) - - if (ENABLE_DB_REQUEST_LOGGING) { - const params = args.map((a) => JSON.stringify(a)).join() - globalCounter.log(this.tableName, propertyName, params) - } - - return result - } - return propertyDesciptor -} - -/** - * Creates a non-caching loader that fetches model data by a foreign key - * @example Fetching replies by question ID - * */ -export const edgeLoader = < - ModelData extends { id: string }, - ForeignKey extends keyof ModelData ->( - kx: Knex, - tableName: string, - foreignKey: ForeignKey, - modelKeys: readonly string[] -): DataLoader => - new DataLoader( - async (keys: readonly string[]) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log( - tableName, - `load_by_${foreignKey}`, - JSON.stringify(keys) - ) - } - const columnName = snakeCase(foreignKey as string) - try { - const rows: ModelData[] = await kx(tableName) - .select(modelKeys) - .whereIn(columnName, keys) - - const keyMap: Record = {} - for (const row of rows) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const keyValue = row[foreignKey] as any as string - if (keyValue in keyMap) { - keyMap[keyValue].push(row) - } else keyMap[keyValue] = [row] - } - const result = [] - for (const key of keys) { - result.push(keyMap[key] || []) - } - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - } catch (e) { - logger.error('DataModel error: ', e) - throw e - } - }, - { cache: false } - ) diff --git a/packages/api/src/datalayer/highlight/index.ts b/packages/api/src/datalayer/highlight/index.ts deleted file mode 100644 index 6ef94805a..000000000 --- a/packages/api/src/datalayer/highlight/index.ts +++ /dev/null @@ -1,179 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import DataModel, { DataModelError, MAX_RECORDS_LIMIT } from '../model' -import { CreateSet, HighlightData, keys as modelKeys, UpdateSet } from './model' -import { Table } from '../../utils/dictionary' -import { Knex } from 'knex' -import { ENABLE_DB_REQUEST_LOGGING, globalCounter, logger } from '../helpers' -import DataLoader from 'dataloader' - -class HighlightModel extends DataModel { - protected batchLoader: DataLoader - public tableName = Table.HIGHLIGHT - protected modelKeys = modelKeys - - batchGet: DataLoader['load'] - - constructor(kx: Knex, cache = true) { - super(kx, cache) - - // override dataloader to skip rows where 'deleted = true' - this.loader = new DataLoader( - async (keys) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log(this.tableName, 'load', JSON.stringify(keys)) - } - - try { - const rows: HighlightData[] = await kx - .table(this.tableName) - .select(this.modelKeys) - .whereIn('id', keys) - .andWhere('deleted', false) - .limit(MAX_RECORDS_LIMIT) - - const keyMap: Record = {} - for (const row of rows) { - if (row.id in keyMap) continue - keyMap[row.id] = row - } - const result = keys.map((key) => keyMap[key]) - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - } catch (e) { - logger.error('DataModel error ', e) - throw e - } - }, - { cache } - ) - - // separate dataloader for fetching grouped highlights - this.batchLoader = new DataLoader(async (articleIds) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log( - this.tableName, - 'batchLoad', - JSON.stringify(articleIds) - ) - } - - const result = await this.kx - .table(Table.HIGHLIGHT) - .select(modelKeys) - .whereIn('elasticPageId', articleIds) - .andWhere('deleted', false) - .orderBy(`${Table.HIGHLIGHT}.created_at`, 'desc') - .limit(MAX_RECORDS_LIMIT) - .then((highlights: HighlightData[]) => { - // group highlights so that each article has its own array of highlights - const result: HighlightData[][] = Array.from( - Array(articleIds.length), - () => [] - ) - // keep track of nested array indices to preserve the order - const positions = articleIds.reduce( - (res, cur, i) => ({ ...res, [cur]: i }), - {} as { [key: string]: number } - ) - - highlights.forEach((highlight) => { - const index = positions[highlight.elasticPageId] - result[index].push({ - ...highlight, - updatedAt: highlight.updatedAt || highlight.createdAt, - }) - this.loader.prime(highlight.id, highlight) - }) - - return result - }) - - if (!result.length) { - return new Array(articleIds.length).fill([]) - } - return result - }) - - this.get = this.loader.load.bind(this.loader) - this.getMany = this.loader.loadMany.bind(this.loader) - this.batchGet = this.batchLoader.load.bind(this.batchLoader) - } - - async unshareAllHighlights( - articleId: string, - userId: string, - tx: Knex.Transaction - ): Promise { - const rows: HighlightData[] = await tx(this.tableName) - .update({ sharedAt: null }) - .where({ elasticPageId: articleId, userId }) - .andWhere(tx.raw(`shared_at is not null`)) - .returning(this.modelKeys) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - - return rows - } - - async delete( - id: string, - tx: Knex.Transaction - ): Promise { - const [row]: HighlightData[] = await tx(this.tableName) - .update({ deleted: true }) - .where({ id }) - .returning(this.modelKeys) - - if (!row) return { error: DataModelError.notFound } - - this.loader.clear(id) - return row - } - - async deleteMany( - idList: string[], - tx: Knex.Transaction - ): Promise { - const rows: HighlightData[] = await tx(this.tableName) - .update({ deleted: true }) - .whereIn('id', idList) - .returning(this.modelKeys) - - if (!rows.length) return { error: DataModelError.notFound } - - idList.forEach((id) => this.loader.clear(id)) - return rows - } - - async getForUserArticle( - userId: string, - articleId: string - ): Promise { - const highlights: HighlightData[] = await this.kx - .table(Table.HIGHLIGHT) - .select(modelKeys) - .where('user_id', userId) - .andWhere('elastic_page_id', articleId) - .andWhere('deleted', false) - .orderBy(`${Table.HIGHLIGHT}.created_at`, 'desc') - .limit(MAX_RECORDS_LIMIT) - - const result = highlights.map((highlight) => { - if (!highlight.updatedAt) { - highlight.updatedAt = highlight.createdAt - } - this.loader.prime(highlight.id, highlight) - return highlight - }) - - return result - } -} - -export default HighlightModel diff --git a/packages/api/src/datalayer/highlight/model.ts b/packages/api/src/datalayer/highlight/model.ts deleted file mode 100644 index 97436ce57..000000000 --- a/packages/api/src/datalayer/highlight/model.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * short_id | varchar(14) | | not null | - * user_id | uuid | | not null | - * article_id | uuid | | not null | - * quote | text | | not null | - * prefix | varchar(5000) | | | - * suffix | varchar(5000) | | | - * patch | text | | not null | - * annotation | text | | | - * deleted | boolean | | not null | false - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | | - * shared_at | timestamp with time zone | | | - * ``` - * */ - -export interface HighlightData { - id: string - shortId: string - userId: string - articleId?: string - quote: string - prefix?: string | null - suffix?: string | null - patch: string - annotation?: string | null - deleted: boolean - createdAt: Date - updatedAt?: Date | null - sharedAt?: Date | null - elasticPageId: string -} - -export const keys = [ - 'id', - 'shortId', - 'userId', - 'articleId', - 'quote', - 'prefix', - 'suffix', - 'patch', - 'annotation', - 'deleted', - 'createdAt', - 'updatedAt', - 'sharedAt', - 'elasticPageId', -] as const - -export const defaultedKeys = [ - 'createdAt', - 'updatedAt', - 'sharedAt', - 'deleted', -] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = ['annotation', 'sharedAt'] as const - -export type UpdateSet = Partialize> diff --git a/packages/api/src/datalayer/knex_config.ts b/packages/api/src/datalayer/knex_config.ts deleted file mode 100644 index 06ff66783..000000000 --- a/packages/api/src/datalayer/knex_config.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { env } from '../env' -import knex from 'knex' -import knexStringcase from 'knex-stringcase' - -export const kx = knex( - knexStringcase({ - client: 'pg', - connection: { - host: env.pg.host, - port: env.pg.port, - user: env.pg.userName, - password: env.pg.password, - database: env.pg.dbName, - }, - pool: { - max: env.pg.pool.max, - acquireTimeoutMillis: 40000, - }, - }) -) diff --git a/packages/api/src/datalayer/links/index.ts b/packages/api/src/datalayer/links/index.ts deleted file mode 100644 index 7bf69f56e..000000000 --- a/packages/api/src/datalayer/links/index.ts +++ /dev/null @@ -1,685 +0,0 @@ -/* eslint-disable @typescript-eslint/no-floating-promises */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -import { - CreateSet, - keys as modelKeys, - ParametersSet, - UpdateSet, - UserArticleData, - UserFeedArticleData, -} from './model' -import DataModel, { MAX_RECORDS_LIMIT } from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { - Article, - PageType, - SortOrder, - SortParams, -} from '../../generated/graphql' -import { - ENABLE_DB_REQUEST_LOGGING, - globalCounter, - logMethod, - logger, -} from '../helpers' -import DataLoader from 'dataloader' -import { ArticleData } from '../article/model' -import { InFilter, LabelFilter, ReadFilter } from '../../utils/search' - -type PartialArticle = Omit< - Article, - 'updatedAt' | 'readingProgressPercent' | 'readingProgressAnchorIndex' -> - -type UserArticleStats = { - highlightsCount: string - annotationsCount: string -} - -const LINK_COLS = [ - 'omnivore.links.id as linkId', - 'omnivore.links.userId', - 'omnivore.links.slug', - 'omnivore.links.article_url as url', - 'omnivore.links.createdAt', - 'omnivore.links.sharedAt', - 'omnivore.links.savedAt', - 'omnivore.links.sharedComment', - 'omnivore.links.articleReadingProgress', - 'omnivore.links.articleReadingProgressAnchorIndex', - 'omnivore.pages.id', - 'omnivore.pages.pageType', - 'omnivore.pages.url as originalArticleUrl', - 'omnivore.pages.title', - 'omnivore.pages.description', - 'omnivore.pages.hash', - 'omnivore.pages.author', - 'omnivore.pages.image', - 'omnivore.pages.pageType', - 'omnivore.pages.publishedAt', -] - -// When fetching the library list we don't need to -// pull all the content out of the database into -// memory just to discard it later -const linkColsWithoutContent = (tx: Knex) => { - return [ - tx.raw(` - CASE - WHEN omnivore.links.article_reading_progress < 98 THEN 'UNREAD' - ELSE 'READ' - END - as read_status - `), - tx.raw(` - CASE - WHEN omnivore.links.archived_at is null THEN false - ELSE true - END - as is_archived - `), - ...LINK_COLS, - ] -} - -const linkCols = (tx: Knex) => { - return [ - 'omnivore.pages.content', - 'omnivore.pages.originalHtml', - ...linkColsWithoutContent(tx), - ] -} - -const readFilterQuery = (filter: ReadFilter) => { - switch (filter) { - case ReadFilter.ALL: - return 'true' - case ReadFilter.UNREAD: - return 'omnivore.links.article_reading_progress < 98' - case ReadFilter.READ: - return 'omnivore.links.article_reading_progress >= 98' - } -} - -class UserArticleModel extends DataModel< - UserArticleData, - CreateSet, - UpdateSet -> { - public tableName = Table.LINKS - protected modelKeys = modelKeys - protected userAndArticleLoader: DataLoader< - { userId: string; articleId: string }, - UserArticleData - > - protected userArticleStatsLoader: DataLoader - getStats: DataLoader['load'] - - constructor(kx: Knex, cache = true) { - super(kx, cache) - this.userAndArticleLoader = new DataLoader( - async (keys) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log( - this.tableName, - 'userId_articleId_load', - JSON.stringify(keys) - ) - } - try { - const rows: UserArticleData[] = await this.kx(this.tableName) - .select(this.modelKeys) - .whereIn( - ['articleId', 'userId'], - keys.map((key) => [key.articleId, key.userId]) - ) - .limit(MAX_RECORDS_LIMIT) - - const keyMap: Map = new Map() - for (const row of rows) { - const hash = `${row.userId}.${row.articleId}` - if (keyMap.has(hash)) continue - keyMap.set(hash, row) - } - - const result = keys.map(({ userId, articleId }) => - keyMap.get(`${userId}.${articleId}`) - ) as UserArticleData[] - // logger.debug('\n\n\n\n\nResult for userId_articleId_load', { keys, result }); - - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - } catch (e) { - logger.error('DataModel error: ', e) - throw e - } - }, - { cache } - ) - - this.userArticleStatsLoader = new DataLoader( - async (keys) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log( - this.tableName, - 'userArticleId_stats_load', - JSON.stringify(keys) - ) - } - try { - const rows: ({ id: string } & UserArticleStats)[] = await this.kx( - `${this.tableName} as ua` - ) - .select([ - 'ua.id', - this.kx.raw('count(h2.id) as highlights_count'), - this.kx.raw('count(h2.annotation) as annotations_count'), - ]) - .leftJoin(`${Table.HIGHLIGHT} as h2`, function () { - this.on('h2.article_id', '=', 'ua.article_id') - this.andOn('h2.user_id', '=', 'ua.user_id') - this.andOn('h2.deleted', '=', kx.raw('FALSE')) - }) - .whereIn('ua.id', keys) - .groupBy(['ua.id']) - .limit(MAX_RECORDS_LIMIT) - - const keyMap: Map = new Map() - for (const row of rows) { - if (keyMap.has(row.id)) continue - keyMap.set(row.id, row) - } - - const result = keys.map((id) => { - const stats = keyMap.get(id) - if (!stats) - throw new Error('User article stats data loader state missmatch!') - return stats - }) - - // logger.debug('\n\n\n\n\nResult for userArticleId_stats_load', { keys, result }); - - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - } catch (e) { - logger.error('DataModel error: ', e) - throw e - } - }, - { cache } - ) - - this.getStats = this.userArticleStatsLoader.load.bind( - this.userArticleStatsLoader - ) - } - - @logMethod - async getByParameters( - userId: UserArticleData['userId'], - params: Record, - tx = this.kx - ): Promise { - const row: UserArticleData | null = await tx(this.tableName) - .select(this.modelKeys) - .where({ userId }) - .andWhere(params) - .orderBy('created_at', 'desc') - .first() - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async articlesForUser( - userId: UserArticleData['userId'], - tx = this.kx - ): Promise { - const rows: UserArticleData[] | null = await tx(this.tableName) - .select(this.modelKeys) - .where({ userId }) - .orderBy('saved_at', 'desc') - .limit(MAX_RECORDS_LIMIT) - if (!rows || !rows.length) { - return null - } - rows.forEach((r) => this.loader.prime(r.id, r)) - return rows - } - - // @logMethod - async getByArticleId( - userId: UserArticleData['userId'], - articleId: UserArticleData['articleId'], - _tx = this.kx - ): Promise { - return this.userAndArticleLoader.load({ userId, articleId }) - } - - /* TODO: move to separate dataloader for checking list of articles have been saved or not */ - @logMethod - async getCountByParameters( - userId: UserArticleData['userId'], - params: Record, - tx = this.kx - ): Promise { - const [{ rowCount }] = await tx(this.tableName) - .count('id as rowCount') - .where({ userId }) - .andWhere(params) - return rowCount as number - } - - @logMethod - async updateByArticleId( - userId: UserArticleData['userId'], - articleId: UserArticleData['articleId'], - params: Record, - tx = this.kx - ): Promise { - const rows: UserArticleData[] | null = await tx(this.tableName) - .update(params) - .where({ articleId, userId }) - .returning(this.modelKeys) - - if (!rows) return null - - for (const row of rows) { - this.loader.prime(row.id, row) - } - - return rows[0] || null - } - - @logMethod - async updateByIds( - ids: UserArticleData['id'][], - params: Record, - tx = this.kx - ): Promise { - const rows: UserArticleData[] | null = await tx(this.tableName) - .update(params) - .whereIn('id', ids) - .returning(this.modelKeys) - - if (!rows) return null - - for (const row of rows) { - this.loader.prime(row.id, row) - } - - return rows[0] || null - } - - /** - * @deprecated - */ - async getUserFeedArticlesLegacy( - userId: string, - tx = this.kx - ): Promise { - const rows = await tx(this.tableName) - .select([ - 'omnivore.links.id', - 'omnivore.links.user_id', - 'omnivore.links.article_id', - 'omnivore.links.shared_at', - 'omnivore.links.saved_at', - 'omnivore.links.shared_comment', - ]) - .leftJoin('omnivore.user_friends', function () { - this.on( - tx.raw('omnivore.user_friends.user_id::text = ?', [userId]) - ).andOn( - 'omnivore.user_friends.friend_user_id', - '=', - 'omnivore.links.user_id' - ) - }) - .whereNotNull('omnivore.links.shared_at') - .andWhere(function () { - this.whereRaw('omnivore.links.user_id::text = ?', [ - userId, - ]).orWhereNotNull('omnivore.user_friends.id') - }) - .orderBy('omnivore.links.shared_at', 'DESC') - .limit(MAX_RECORDS_LIMIT) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - @logMethod - async getSharedArticlesCount( - userId: string, - tx = this.kx - ): Promise { - const rows = await tx(this.tableName) - .select([tx.raw('count(omnivore.links.id) as shared_articles_count')]) - .whereNotNull('omnivore.links.shared_at') - .andWhere('omnivore.links.user_id', userId) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows[0]?.sharedArticlesCount || 0 - } - - async getUserSharedArticles( - userId: string, - tx = this.kx - ): Promise { - const rows = await tx(this.tableName) - .select([ - 'omnivore.links.id', - 'omnivore.links.user_id', - 'omnivore.links.article_id', - 'omnivore.links.shared_at', - 'omnivore.links.saved_at', - 'omnivore.links.shared_comment', - ]) - .whereNotNull('omnivore.links.shared_at') - .andWhere('omnivore.links.user_id', userId) - .orderBy('omnivore.links.shared_at', 'DESC') - .limit(MAX_RECORDS_LIMIT) - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - @logMethod - async getPaginated( - args: { - cursor: string - first: number - sort?: SortParams - query?: string - inFilter: InFilter - readFilter: ReadFilter - typeFilter: PageType | undefined - labelFilters: LabelFilter[] - }, - userId: string, - tx = this.kx, - notNullField: string | null = null - ): Promise<[PartialArticle[], number] | null> { - const { cursor, first, sort, query, readFilter } = args - - const sortOrder = sort?.order === SortOrder.Ascending ? 'ASC' : 'DESC' - const whereOperator = sort?.order === SortOrder.Ascending ? '>=' : '<=' - - const queryPromise = tx(this.tableName) - .select(linkColsWithoutContent(tx)) - .innerJoin(Table.PAGES, 'pages.id', 'links.article_id') - .where({ 'links.user_id': userId }) - .where(tx.raw(readFilterQuery(readFilter))) - - if (query) { - const searchQuery = tx.raw(`tsv @@ websearch_to_tsquery(?)`, query) - queryPromise.where(searchQuery) - } - - if (args.typeFilter) { - queryPromise.where( - tx.raw(`omnivore.pages.page_type = ?`, args.typeFilter) - ) - } - - if (args.inFilter !== InFilter.ALL) { - switch (args.inFilter) { - case InFilter.INBOX: - queryPromise.whereNull('links.archivedAt') - break - case InFilter.ARCHIVE: - queryPromise.whereNotNull('links.archivedAt') - break - } - } - - if (notNullField) { - queryPromise.whereNotNull(notNullField) - } - - const [{ count: totalCount }] = - (await tx(queryPromise.clone().as('subq')).count()) || '0' - - // This is a temp hack as we move from time based cursors to - // using offset, this will be replaced when we change the - // storage backend. - if (cursor && Number(cursor) > 1546300800000) { - queryPromise.where( - `omnivore.links.saved_at`, - whereOperator, - new Date(parseInt(cursor) + (sortOrder === 'ASC' ? -1 : 1)) //Time Comparison Bias - ) - } else if (cursor && Number(cursor) <= 1546300800000) { - queryPromise.offset(Number(cursor)) - } - - // If first is greater than 100 set it to 100 - const limit = first > 100 ? 100 : first - queryPromise - .orderBy('omnivore.links.saved_at', sortOrder) - .orderBy('omnivore.links.created_at', sortOrder) - .orderBy('omnivore.links.id', sortOrder) - .limit(limit) - - const rows = await queryPromise - for (const row of rows) { - this.loader.prime(row.id, row) - } - - return [rows, parseInt(totalCount as string)] - } - - @logMethod - async getUserFeedArticlesPaginatedWithHighlights( - args: { cursor: string; first: number; sharedByUser?: string | null }, - userId: string, - tx = this.kx - ): Promise< - | (UserFeedArticleData & { - highlightsCount: number - annotationsCount: number - })[] - | null - > { - const { cursor, first, sharedByUser } = args - - // let userArticlesListQuery; - - // Getting the list of friends user ids - const friendsListQuery = sharedByUser - ? ([] as never) - : tx(Table.USER_FRIEND).select('friendUserId').where('userId', userId) - - // Getting the links ids list that applies to the "My feed" page - const userArticlesListQuery = tx(this.tableName) - .select('id') - .where(function () { - this.whereIn('userId', friendsListQuery) - this.orWhere('userId', sharedByUser || userId) - }) - .whereNotNull('sharedAt') - - // Collecting the highlights and annotations stats for the links records - const userArticlesStatsQuery = tx(`${this.tableName} as ua`) - .select([ - 'ua.article_id', - 'ua.user_id', - tx.raw('count(h2.id) as highlights_count'), - tx.raw( - `count(case when h2.annotation is not null and h2.annotation <> '' then 1 else null end) as annotations_count` - ), - ]) - .leftJoin(`${Table.HIGHLIGHT} as h2`, function () { - this.on('h2.article_id', '=', 'ua.article_id') - this.andOn('h2.user_id', '=', 'ua.user_id') - this.andOn('h2.deleted', '=', tx.raw('FALSE')) - }) - // TODO: Check if using the join isntead could be more efficient approach here - // (https://github.com/omnivore-app/omnivore/pull/1053#discussion_r604914773) - .whereIn('ua.id', userArticlesListQuery) - .where('ua.shared_with_highlights', 'TRUE') - .groupBy(['ua.article_id', 'ua.user_id']) - - // Combining required link columns with the stats calculated - const userArticlesQuery = tx(`${this.tableName} as ua`) - .select([ - 'ua.id', - 'ua.article_id', - tx.raw('null as highlight_id'), - 'ua.user_id', - 'uas.highlights_count', - 'uas.annotations_count', - 'ua.shared_at', - 'ua.saved_at', - 'ua.shared_comment', - 'ua.shared_with_highlights', - ]) - .leftJoin(userArticlesStatsQuery.as('uas'), function () { - this.on('uas.article_id', '=', 'ua.article_id') - this.andOn('uas.user_id', '=', 'ua.user_id') - }) - .whereIn('ua.id', userArticlesListQuery) - - // Getting the shared highlights - const highlightsQuery = tx(`${Table.HIGHLIGHT} as hi`) - .select([ - 'id', - 'article_id', - 'id as highlight_id', - 'user_id', - tx.raw('0 as highlights_count'), - tx.raw('0 as annotations_count'), - 'shared_at', - tx.raw('null as saved_at'), - tx.raw('null as shared_comment'), - tx.raw('null as shared_with_highlights'), - ]) - .where(function () { - this.whereIn('userId', friendsListQuery) - this.orWhere('userId', sharedByUser || userId) - }) - .where(tx.raw('deleted is not true')) - .andWhere(tx.raw('shared_at is not null')) - - if (sharedByUser) { - highlightsQuery.andWhere('user_id', sharedByUser) - } - - // Merging links record with the shared highlights - // NOTE: Number of columns and order should match in both queries - const feedItemsQuery = tx - .union([userArticlesQuery, highlightsQuery]) - .orderBy('shared_at', 'DESC') - - // Appending resulting query with a cursor if provided - const resultQuery = cursor - ? tx - .select('*') - .from(feedItemsQuery.as('r')) - .where( - `shared_at`, - '<=', - new Date(parseInt(cursor) + 1) //Time Comparison Bias - ) - : feedItemsQuery - - resultQuery.limit(first) - - const rows = await resultQuery - for (const row of rows) { - this.loader.prime(row.id, row) - this.userArticleStatsLoader.prime(row.id, { - highlightsCount: row.highlightsCount, - annotationsCount: row.annotationsCount, - }) - } - - return rows - } - - @logMethod - async getForUser( - userId: string, - articleId: string, - tx = this.kx - ): Promise<(ArticleData & UserArticleData) | null> { - const row = await tx(Table.LINKS) - .select(linkCols(tx)) - .innerJoin(Table.PAGES, 'links.article_id', 'pages.id') - .where('links.user_id', userId) - .where('links.article_id', articleId) - .first() - - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getBySlug( - username: string, - slug: string, - tx = this.kx - ): Promise<(ArticleData & UserArticleData) | null> { - const row = await tx(Table.LINKS) - .select(linkCols(tx)) - .innerJoin(Table.PAGES, 'links.article_id', 'pages.id') - .innerJoin(Table.USER_PROFILE, 'links.user_id', 'user_profile.user_id') - .where('user_profile.username', username) - .where('links.slug', slug) - .first() - - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - async getByUserIdAndSlug( - uid: string, - slug: string, - tx = this.kx - ): Promise<(ArticleData & UserArticleData) | null> { - const row = await tx(Table.LINKS) - .select(linkCols(tx)) - .innerJoin(Table.PAGES, 'links.article_id', 'pages.id') - .where('links.user_id', uid) - .where('links.slug', slug) - .first() - - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async setBookmarkOnMultiple( - userId: string, - articles: CreateSet[], - tx = this.kx - ): Promise { - const rows = await tx(Table.LINKS) - .insert(articles) - .returning(this.modelKeys) - - if (!rows) return null - rows.forEach((r) => this.loader.prime(r.id, r)) - return rows - } -} - -export default UserArticleModel diff --git a/packages/api/src/datalayer/links/model.ts b/packages/api/src/datalayer/links/model.ts deleted file mode 100644 index b6a226eba..000000000 --- a/packages/api/src/datalayer/links/model.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------------------------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * article_id | uuid | | not null | - * article_url | text | | not null | - * slug | text | | not null | - * article_hash | text | | not null | - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * shared_at | timestamp with time zone | | | - * shared_comment | text | | | - * article_reading_progress | real | | not null | 0 - * article_reading_progress_anchor_index | integer | | not null | 0 - * saved_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * shared_with_highlights | boolean | | | false - * ``` - * */ -export interface UserArticleData { - id: string - userId: string - articleId: string - slug: string - articleUrl: string - articleHash: string - createdAt: Date - updatedAt: Date - savedAt: Date - sharedAt?: Date | null - archivedAt?: Date | null - sharedComment?: string | null - articleReadingProgress: number - articleReadingProgressAnchorIndex: number - sharedWithHighlights?: boolean - isArchived: boolean -} - -export interface UserFeedArticleData { - id: string - userId: string - articleId: string - sharedAt: Date - createdAt: Date - sharedComment?: string | null - sharedWithHighlights?: boolean -} - -export const keys = [ - 'id', - 'userId', - 'articleId', - 'slug', - 'articleUrl', - 'articleHash', - 'createdAt', - 'updatedAt', - 'savedAt', - 'sharedAt', - 'archivedAt', - 'sharedComment', - 'articleReadingProgress', - 'articleReadingProgressAnchorIndex', - 'sharedWithHighlights', -] as const - -export const defaultedKeys = [ - 'id', - 'createdAt', - 'updatedAt', - 'savedAt', - 'articleReadingProgress', - 'articleReadingProgressAnchorIndex', - 'articleDeleted', - 'sharedWithHighlights', -] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [ - 'slug', - 'savedAt', - 'articleId', - 'articleUrl', - 'articleHash', - 'sharedAt', - 'archivedAt', - 'sharedComment', - 'articleReadingProgress', - 'articleReadingProgressAnchorIndex', - 'sharedWithHighlights', -] as const - -export type UpdateSet = Partialize< - PickTuple -> - -export const getByParametersKeys = exclude(keys, ['id'] as const) - -export type ParametersSet = PickTuple< - UserArticleData, - typeof getByParametersKeys -> diff --git a/packages/api/src/datalayer/links/share_info.ts b/packages/api/src/datalayer/links/share_info.ts deleted file mode 100644 index 13d72451b..000000000 --- a/packages/api/src/datalayer/links/share_info.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { Knex } from 'knex' -import { LinkShareInfo } from '../../generated/graphql' -import { getPageByParam } from '../../elastic/pages' - -// once we have links setup properly in the API we will remove this method -// and have a getShareInfoForLink method -export const getShareInfoForArticle = async ( - kx: Knex, - userId: string, - articleId: string -): Promise => { - // TEMP: because the old API uses articles instead of Links, we are actually - // getting an article ID here and need to map it to a link ID. When the API - // is updated to use Links instead of Articles this will be removed. - const page = await getPageByParam({ userId, _id: articleId }) - - if (!page) { - return undefined - } - - const result = await kx('omnivore.link_share_info') - .select('*') - .where({ elastic_page_id: page.id }) - .first() - - return result -} - -export const createOrUpdateLinkShareInfo = async ( - tx: Knex, - linkId: string, - title: string, - description: string -): Promise => { - const item = { linkId, title, description } - const [result]: LinkShareInfo[] = await tx('omnivore.link_share_info') - .insert(item) - .onConflict('link_id') - .merge() - .returning('*') - - if (!result) { - return Promise.reject(new Error('No result')) - } - - return result -} diff --git a/packages/api/src/datalayer/model.ts b/packages/api/src/datalayer/model.ts deleted file mode 100644 index ab17480f6..000000000 --- a/packages/api/src/datalayer/model.ts +++ /dev/null @@ -1,183 +0,0 @@ -/* eslint-disable @typescript-eslint/no-floating-promises */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import DataLoader from 'dataloader' -import { Knex } from 'knex' -import { ENABLE_DB_REQUEST_LOGGING, globalCounter, logger } from './helpers' - -export enum DataModelError { - notFound = 'NOT_FOUND', -} - -export const MAX_RECORDS_LIMIT = 1000 - -abstract class DataModel< - ModelData extends { id: string }, - CreateSet, - UpdateSet -> { - protected loader: DataLoader - public tableName!: string - protected modelKeys!: readonly (keyof ModelData)[] - kx: Knex - get: DataLoader['load'] - getMany: DataLoader['loadMany'] - - /** - * @param kx - DB connection - * @param userId - user id to use when executing data loader queries - * */ - constructor(kx: Knex, cache = true) { - this.kx = kx - this.loader = new DataLoader( - async (keys) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log(this.tableName, 'load', JSON.stringify(keys)) - } - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const rows: ModelData[] = await this.kx(this.tableName) - .select(this.modelKeys) - .whereIn('id', keys) - - const keyMap: Record = {} - for (const row of rows) { - if (row.id in keyMap) continue - keyMap[row.id] = row - } - const result = keys.map((key) => keyMap[key]) - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - } catch (e) { - logger.error('DataModel error: ', e) - throw e - } - }, - { cache } - ) - this.get = this.loader.load.bind(this.loader) - this.getMany = this.loader.loadMany.bind(this.loader) - } - - /** - * Gets entity that accomplish "whereIn" condition - * @param field - string specifying a field to use in "whereIn" condition - * @param values - an array of values to compare in "whereIn" condition - * @param tx - DB transaction - * @example - * // Return entity that match id comparison - * const collaboratorsIds = ['1', '2']; - * return models.user.getWhereIn('id', collaboratorsIds); - */ - async getWhereIn( - field: K, - values: ModelData[K][], - tx = this.kx, - notNullField: string | null = null - ): Promise { - let queryPromise = tx(this.tableName) - .select(this.modelKeys) - .whereIn(field, values) - .orderBy('created_at', 'desc') - if (notNullField) { - queryPromise = queryPromise.whereNotNull(notNullField) - } - const rows: ModelData[] = await queryPromise - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - async getAll(tx = this.kx): Promise { - const rows: ModelData[] = await tx(this.tableName) - .select(this.modelKeys) - .orderBy('created_at', 'desc') - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - async create(data: CreateSet, tx = this.kx): Promise { - const [row]: ModelData[] = await tx(this.tableName) - .insert(data) - .returning('*') - this.loader.prime(row.id, row) - return row - } - - async createMany( - data: CreateSet[], - tx: Knex.Transaction - ): Promise { - const rows: ModelData[] = await tx - .batchInsert(this.tableName, data as never) - .returning('*') - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - async update(id: string, data: UpdateSet, tx = this.kx): Promise { - const [row]: ModelData[] = await tx(this.tableName) - .update(data) - .where({ id }) - .returning('*') - this.loader.prime(id, row) - return row - } - - async delete( - id: string, - tx: Knex.Transaction - ): Promise { - const [row]: ModelData[] = await tx(this.tableName) - .where({ id }) - .delete() - .returning('*') - - if (!row) return { error: DataModelError.notFound } - - this.loader.clear(id) - return row - } - - async deleteWhere( - params: Record, - tx: Knex.Transaction - ): Promise { - const rows: ModelData[] = await tx(this.tableName) - .andWhere(params) - .delete() - .returning('*') - - for (const row of rows) { - this.loader.clear(row.id) - } - return rows - } - - async deleteWhereIn( - params: Record, - tx: Knex.Transaction - ): Promise { - const rows: ModelData[] = await tx(this.tableName) - .where((builder) => { - for (const field in params) { - builder.whereIn(field, params[field]) - } - }) - .delete() - .returning('*') - - for (const row of rows) { - this.loader.clear(row.id) - } - return rows - } -} - -export default DataModel diff --git a/packages/api/src/datalayer/reaction/index.ts b/packages/api/src/datalayer/reaction/index.ts deleted file mode 100644 index 6c77544f1..000000000 --- a/packages/api/src/datalayer/reaction/index.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { - ENABLE_DB_REQUEST_LOGGING, - globalCounter, - logMethod, -} from './../helpers' -import { CreateSet, keys as modelKeys, ReactionData, UpdateSet } from './model' -import DataModel, { DataModelError, MAX_RECORDS_LIMIT } from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' - -import DataLoader from 'dataloader' - -class ReactionModel extends DataModel { - public tableName = Table.REACTION - protected modelKeys = modelKeys - - protected getBatchLoader: ( - type: 'highlightId' | 'userArticleId' - ) => DataLoader - - batchGetFromArticle: DataLoader['load'] - batchGetFromHighlight: DataLoader['load'] - - constructor(kx: Knex, cache = true) { - super(kx, cache) - - // override dataloader to skip rows where 'deleted = true' - - // separate dataloader for fetching grouped highlights - this.getBatchLoader = (type: 'highlightId' | 'userArticleId') => - new DataLoader(async (ids) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log(this.tableName, 'batchLoad', JSON.stringify(ids)) - } - - const result: ReactionData[][] = await this.kx(Table.REACTION) - .select(modelKeys) - .whereIn(type, ids) - .andWhere('deleted', false) - .orderBy(`${Table.REACTION}.created_at`, 'desc') - .limit(MAX_RECORDS_LIMIT) - .then((reactions: ReactionData[]) => { - // group highlights so that each article has its own array of highlights - const arr: ReactionData[][] = Array.from( - Array(ids.length), - () => [] - ) - // keep track of nested array indices to preserve the order - const positions = ids.reduce( - (acc, cur, i) => ({ ...acc, [cur]: i }), - {} as { [key: string]: number } - ) - - reactions.forEach((re) => { - const pos = re[type] - if (!pos) { - return - } - const index = positions[pos] - arr[index].push({ - ...re, - updatedAt: re.updatedAt || re.createdAt, - }) - this.loader.prime(re.id, re) - }) - - return arr - }) - - return result - }) - - this.get = this.loader.load.bind(this.loader) - this.getMany = this.loader.loadMany.bind(this.loader) - - const ba = this.getBatchLoader('userArticleId') - const bh = this.getBatchLoader('highlightId') - - this.batchGetFromArticle = ba.load.bind(ba) - this.batchGetFromHighlight = bh.load.bind(bh) - } - - @logMethod - async getByUserAndParam( - userId: ReactionData['userId'], - params: Record, - tx = this.kx - ): Promise { - const row: ReactionData | null = await tx(this.tableName) - .select() - .where({ userId }) - .andWhere(params) - .andWhere('deleted', false) - .first(this.modelKeys) - - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - async delete( - id: string, - tx: Knex.Transaction - ): Promise { - const [row]: ReactionData[] = await tx(this.tableName) - .update({ deleted: true }) - .where({ id }) - .returning(this.modelKeys) - - if (!row) return { error: DataModelError.notFound } - - this.loader.clear(id) - return row - } -} - -export default ReactionModel diff --git a/packages/api/src/datalayer/reaction/model.ts b/packages/api/src/datalayer/reaction/model.ts deleted file mode 100644 index 2cef009ba..000000000 --- a/packages/api/src/datalayer/reaction/model.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * --------------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * user_article_id | uuid | | | - * highlight_id | uuid | | | - * highlight_reply_id | uuid | | | - * code | varchar(50) | | not null | - * deleted | boolean | | not null | false - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | | - * ``` - * */ - -export interface ReactionData { - id: string - userId: string - userArticleId?: string | null - highlightId?: string | null - highlightReplyId?: string | null - code: string - deleted: boolean - createdAt: Date - updatedAt?: Date | null -} - -export const keys = [ - 'id', - 'userId', - 'userArticleId', - 'highlightId', - 'highlightReplyId', - 'code', - 'deleted', - 'createdAt', - 'updatedAt', -] as const - -export const defaultedKeys = [ - 'id', - 'createdAt', - 'updatedAt', - 'deleted', -] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = ['code'] as const - -export type UpdateSet = Partialize> diff --git a/packages/api/src/datalayer/reminders/index.ts b/packages/api/src/datalayer/reminders/index.ts deleted file mode 100644 index 2b6e74728..000000000 --- a/packages/api/src/datalayer/reminders/index.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { - CreateSet, - keys as modelKeys, - ParametersSet, - ReminderData, - UpdateSet, -} from './model' -import DataModel, { DataModelError } from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { logMethod } from '../helpers' -import { ArticleData } from '../article/model' -import { UserArticleData } from '../links/model' - -const JOIN_COLS = [ - 'links2.id', - 'links2.slug', - 'pages.title', - 'pages.description', - 'pages.author', - 'pages.image', - 'reminders.send_notification', -] - -class ReminderModel extends DataModel { - public tableName = Table.REMINDER - protected modelKeys = modelKeys - - constructor(kx: Knex, cache = true) { - super(kx, cache) - } - - @logMethod - async setRemindersComplete( - userId: ReminderData['userId'], - remindAt: ReminderData['remindAt'], - tx = this.kx - ): Promise { - const [row]: ReminderData[] = await tx(this.tableName) - .where({ userId, remindAt }) - .update({ status: 'COMPLETED' }) - .returning(this.modelKeys) - - if (!row) return null - - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getCreated( - id: ReminderData['id'], - tx = this.kx - ): Promise { - const row: ReminderData = await tx(this.tableName) - .select() - .where({ id, status: 'CREATED' }) - .first(this.modelKeys) - - if (!row) return null - - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getCreatedByParameters( - userId: ReminderData['userId'], - params: Record, - tx = this.kx - ): Promise { - const row: ReminderData = await tx(this.tableName) - .select() - .where({ userId: userId, status: 'CREATED' }) - .andWhere(params) - .first(this.modelKeys) - - if (!row) return null - - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getByRequestId( - userId: string, - requestId: string, - tx = this.kx - ): Promise { - const row: ReminderData = await tx(this.tableName) - .select() - .where({ userId, elasticPageId: requestId }) - .first(this.modelKeys) - - if (!row) return null - - this.loader.prime(row.id, row) - return row - } - - @logMethod - async existByUserAndRemindAt( - userId: ReminderData['userId'], - remindAt: ReminderData['remindAt'], - tx = this.kx - ): Promise { - const row: ReminderData | null = await tx(this.tableName) - .select() - .where({ userId: userId, status: 'CREATED', remindAt: remindAt }) - .first('id') - - return !!row?.id - } - - @logMethod - async getByUserAndRemindAt( - userId: ReminderData['userId'], - remindAt: ReminderData['remindAt'], - tx = this.kx - ): Promise<(ReminderData & ArticleData & UserArticleData)[] | null> { - const rows: (ReminderData & ArticleData & UserArticleData)[] | null = - await tx(this.tableName) - .select(JOIN_COLS) - .leftJoin(Table.LINKS, 'links.id', 'reminders.link_id') - .leftJoin( - Table.ARTICLE_SAVING_REQUEST, - 'article_saving_request.id', - 'reminders.article_saving_request_id' - ) - .leftJoin(Table.PAGES, function () { - this.on('pages.id ', '=', 'links.article_id') - this.orOn('pages.id', '=', 'article_saving_request.article_id') - }) - .leftJoin(`${Table.LINKS} as links2`, 'links2.article_id', 'pages.id') - .where({ - 'reminders.user_id': userId, - 'reminders.status': 'CREATED', - 'reminders.remind_at': remindAt, - }) - - if (rows.length == 0) return null - - return rows - } - - @logMethod - async delete( - id: ReminderData['id'], - tx = this.kx - ): Promise { - const [row]: ReminderData[] = await tx(this.tableName) - .where({ id }) - .update({ status: 'DELETED' }) - .returning(this.modelKeys) - - if (!row) return { error: DataModelError.notFound } - - this.loader.clear(id) - return row - } -} - -export default ReminderModel diff --git a/packages/api/src/datalayer/reminders/model.ts b/packages/api/src/datalayer/reminders/model.ts deleted file mode 100644 index 0bfad531d..000000000 --- a/packages/api/src/datalayer/reminders/model.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------------------+--------------------------+-----------+----------+---------------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * article_saving_request_id | uuid | | | - * link_id | uuid | | | - * task_name | text | | | - * type | reminder_type | | not null | - * status | reminder_status | | not null | 'CREATED'::reminder_status - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | | - * remind_at | timestamp with time zone | | not null | - * archive_until | boolean | | not null | false - * send_notification | boolean | | not null | true - * ``` - * */ -export interface ReminderData { - id: string - userId: string - articleSavingRequestId?: string - linkId?: string - archiveUntil?: boolean - sendNotification?: boolean - taskName?: string - type: string - status?: string - createdAt: Date - updatedAt?: Date - remindAt: Date - elasticPageId?: string -} - -export const keys = [ - 'id', - 'userId', - 'articleSavingRequestId', - 'linkId', - 'archiveUntil', - 'sendNotification', - 'taskName', - 'remindAt', - 'status', - 'createdAt', - 'updatedAt', - 'elasticPageId', -] as const - -export const defaultedKeys = ['id', 'updatedAt', 'status'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [ - 'taskName', - 'remindAt', - 'status', - 'sendNotification', - 'archiveUntil', -] as const - -export type UpdateSet = PickTuple - -export const getByParametersKeys = exclude(keys, ['id'] as const) - -export type ParametersSet = PickTuple diff --git a/packages/api/src/datalayer/report/index.ts b/packages/api/src/datalayer/report/index.ts deleted file mode 100644 index 0906e281e..000000000 --- a/packages/api/src/datalayer/report/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Knex } from 'knex' -import { ReportType } from '../../generated/graphql' - -interface ReportItem { - pageId: string - itemUrl: string - - sharedBy: string - reportedBy: string | undefined - - reportTypes: ReportType[] - reportComment: string -} - -// Returns the ID of the report -export const createAbuseReport = async ( - tx: Knex, - reportedBy: string | undefined, - input: ReportItem -): Promise => { - const report = { ...input, reportedBy } - const result: string[] = await tx( - 'omnivore.abuse_reports' - ).insert(report, ['id']) - - if (!result || result.length !== 1) { - throw new Error('Unable to create abuse report.') - } - - return result[0] -} diff --git a/packages/api/src/datalayer/task/index.ts b/packages/api/src/datalayer/task/index.ts deleted file mode 100644 index b0f31624d..000000000 --- a/packages/api/src/datalayer/task/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { CreateSet, keys as modelKeys, TaskData, UpdateSet } from './model' -import DataModel from '../model' -import { Knex } from 'knex' -import { logMethod } from '../helpers' - -class TaskModel extends DataModel { - public tableName = 'omnivore.task' - protected modelKeys = modelKeys - constructor(kx: Knex, cache = true) { - super(kx, cache) - } - - @logMethod - async create(set: CreateSet, tx?: Knex.Transaction): Promise { - if (tx) { - return super.create(set, tx) - } - return this.kx.transaction((tx) => super.create(set, tx)) - } -} - -export default TaskModel diff --git a/packages/api/src/datalayer/task/model.ts b/packages/api/src/datalayer/task/model.ts deleted file mode 100644 index 11007c219..000000000 --- a/packages/api/src/datalayer/task/model.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------------+--------------------------+------------+-------------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * title | text | | not null | - * created_by | uuid | | not null | - * created_at | timestamp with time zone | | not null | - * ``` - * */ -export interface TaskData { - id: string - title: string - createdBy: string - createdAt: Date -} - -export const keys = ['id', 'title', 'createdBy', 'createdAt'] as const - -export const defaultedKeys = ['id'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = ['title'] as const - -export type UpdateSet = PickTuple diff --git a/packages/api/src/datalayer/upload_files/index.ts b/packages/api/src/datalayer/upload_files/index.ts deleted file mode 100644 index 6899d6309..000000000 --- a/packages/api/src/datalayer/upload_files/index.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { - CreateSet, - keys as modelKeys, - UpdateSet, - UploadFileData, -} from './model' -import DataModel from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { logMethod } from '../helpers' - -class UploadFileDataModel extends DataModel< - UploadFileData, - CreateSet, - UpdateSet -> { - public tableName = Table.UPLOAD_FILES - protected modelKeys = modelKeys - constructor(kx: Knex, cache = true) { - super(kx, cache) - } - - @logMethod - async create(set: CreateSet, tx?: Knex.Transaction): Promise { - if (tx) { - return super.create(set, tx) - } - return this.kx.transaction((tx) => super.create(set, tx)) - } - - @logMethod - async getWhere( - params: { - id?: UploadFileData['id'] - userId?: UploadFileData['userId'] - url?: UploadFileData['url'] - }, - tx = this.kx - ): Promise { - const row: UploadFileData | null = await tx(this.tableName) - .select(this.modelKeys) - .where(params) - .first() - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async setFileUploadComplete( - id: UploadFileData['id'], - tx = this.kx - ): Promise { - const row: UploadFileData | null = await tx(this.tableName) - .select() - .where({ id }) - .update({ status: 'COMPLETED' }) - .returning(this.modelKeys) - .limit(2) - .then((result) => { - if (result.length == 1) { - return result[0] - } - throw Error('Should never return multiple rows') - }) - if (row?.id) { - this.loader.prime(row.id, row) - } - return row - } - - async uploadFileForUserAndArticle( - userId: string, - articleId: string - ): Promise { - const row: UploadFileData | null = await this.kx(Table.PAGES) - .select(this.modelKeys.map((k) => `${Table.UPLOAD_FILES}.${k}`)) - .join(Table.UPLOAD_FILES, 'upload_files.id', '=', 'pages.upload_file_id') - .where({ 'upload_files.userId': userId, 'pages.id': articleId }) - .limit(1) - .first() - return row - } - - async uploadFileForArticle( - articleId: string - ): Promise { - const row: UploadFileData | null = await this.kx(Table.PAGES) - .select(this.modelKeys.map((k) => `${Table.UPLOAD_FILES}.${k}`)) - .join(Table.UPLOAD_FILES, 'upload_files.id', '=', 'pages.upload_file_id') - .where({ 'pages.id': articleId }) - .limit(1) - .first() - return row - } -} - -export default UploadFileDataModel diff --git a/packages/api/src/datalayer/upload_files/model.ts b/packages/api/src/datalayer/upload_files/model.ts deleted file mode 100644 index 49db669e2..000000000 --- a/packages/api/src/datalayer/upload_files/model.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * -------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * url | text | | not null | - * file_name | text | | not null | - * content_type| text | | not null | - * status | upload_status_type (text)| | not null | - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * ``` - * */ -export interface UploadFileData { - id: string - userId: string - url: string - fileName: string - contentType: string - status: string - createdAt: Date - updatedAt: Date -} - -export const keys = [ - 'id', - 'userId', - 'url', - 'fileName', - 'contentType', - 'status', - 'createdAt', - 'updatedAt', -] as const - -export const defaultedKeys = ['id', 'createdAt', 'updatedAt'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = ['url', 'status'] as const - -export type UpdateSet = PickTuple - -export const getByParametersKeys = exclude(keys, ['id'] as const) - -export type ParametersSet = PickTuple< - UploadFileData, - typeof getByParametersKeys -> diff --git a/packages/api/src/datalayer/user/index.ts b/packages/api/src/datalayer/user/index.ts deleted file mode 100644 index 4fa382ba0..000000000 --- a/packages/api/src/datalayer/user/index.ts +++ /dev/null @@ -1,349 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { - CreateSet, - keys as modelKeys, - ProfileCreateSet, - profileKeys, - ProfileUpdateSet, - UpdateSet, - UserData, -} from './model' -import DataModel, { DataModelError, MAX_RECORDS_LIMIT } from '../model' -import { Knex } from 'knex' -import { - ENABLE_DB_REQUEST_LOGGING, - globalCounter, - logMethod, - logger, -} from '../helpers' -import { Table } from '../../utils/dictionary' -import DataLoader from 'dataloader' -import { Partialize } from '../../util' -import { Profile } from '../../generated/graphql' -import { kx as knexConfig } from './../knex_config' - -const TOP_USERS = [ - 'jacksonh', - 'nat', - 'luis', - 'satindar', - 'malandrina', - 'patrick', - 'alexgutjahr', -] - -class UserModel extends DataModel { - public tableName = Table.USER - protected modelKeys = modelKeys - constructor(kx: Knex = knexConfig, cache = true) { - super(kx, cache) - // override DataModel's base class dataloader to include profile join in user query - this.loader = new DataLoader( - async (keys) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log(this.tableName, 'load', JSON.stringify(keys)) - } - try { - const rows: UserData[] = await this.kx({ u: this.tableName }) - .select([ - ...this.modelKeys.map((k) => `u.${k}`), - this.kx.raw('to_jsonb(p) as profile'), - ]) - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .whereIn('u.id', keys) - .limit(MAX_RECORDS_LIMIT) - - const keyMap: Record = {} - for (const row of rows) { - if (row.id in keyMap) continue - keyMap[row.id] = row - } - const result = keys.map((key) => keyMap[key]) - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - } catch (e) { - logger.error('DataModel error: ', e) - throw e - } - }, - { cache } - ) - - this.get = this.loader.load.bind(this.loader) - this.getMany = this.loader.loadMany.bind(this.loader) - } - - @logMethod - async getWhere( - params: { - email?: UserData['email'] - source?: UserData['source'] - username?: UserData['profile']['username'] - sourceUserId?: UserData['sourceUserId'] - 'p.username'?: UserData['profile']['username'] - }, - tx = this.kx - ): Promise { - // rewrite username key if it exists to use it with joined profile table - if ('username' in params) { - params['p.username'] = params.username - delete params.username - } - - const row: UserData | null = await tx({ u: this.tableName }) - .select([ - ...this.modelKeys.map((k) => `u.${k}`), - this.kx.raw('to_jsonb(p) as profile'), - ]) - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .where(params) - .first() - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async exists( - params: { - email?: UserData['email'] - username?: UserData['profile']['username'] - }, - tx = this.kx - ): Promise { - const row: UserData | null = await tx({ u: this.tableName }) - .select('u.id') - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .where('email', params.email || '') - .orWhere('p.username', params.username || '') - .first() - - return !!row?.id - } - - @logMethod - async getSharedHighlightsStats( - userId: string, - tx = this.kx - ): Promise<{ sharedHighlightsCount: number; sharedNotesCount: number }> { - const row = await tx('omnivore.highlight as h2') - .select([ - tx.raw('count(h2.id) as shared_highlights_count'), - tx.raw('count(h2.annotation) as shared_notes_count'), - ]) - .whereRaw('h2.user_id::text = ?', [userId]) - .andWhere('h2.deleted', '=', tx.raw('FALSE')) - .whereNotNull('h2.shared_at') - .first() - - if (!row) { - return { sharedHighlightsCount: 0, sharedNotesCount: 0 } - } - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getUserDetails( - viewerId: string | undefined, - userId: string, - tx = this.kx - ): Promise< - | (UserData & { - followersCount: number - friendsCount: number - viewerIsFollowing: boolean - }) - | null - > { - /* - * We join the user table with the user_friends table twice, once to fetch - * the list of friends, and another join to query list of followers. We - * group the results by user ID, aggregate the friends and followers - * results using postgres array operators. We use coalesce() to return 0 - * when the results array is null. - */ - const row = await tx({ u: this.tableName }) - .select([ - 'u.*', - this.kx.raw( - 'coalesce(array_length(array_remove(array_agg(DISTINCT omnivore.user_friends.friend_user_id), null), 1), 0) as friends_count' - ), - this.kx.raw( - 'coalesce(array_length(array_remove(array_agg(DISTINCT user_followers.user_id), null), 1), 0) as followers_count' - ), - this.kx.raw( - viewerId - ? 'coalesce(? = ANY(array_agg(DISTINCT user_followers.user_id)), false) as viewer_is_following' - : '? as _unused', - [viewerId || ''] - ), - this.kx.raw('to_jsonb(p) as profile'), - ]) - .leftJoin('omnivore.user_friends', function () { - this.on( - tx.raw('omnivore.user_friends.user_id::text = ?', [userId]) - ).andOn('omnivore.user_friends.user_id', '=', 'u.id') - }) - .leftJoin('omnivore.user_friends as user_followers', function () { - this.on(tx.raw('user_followers.friend_user_id::text = ?', [userId])) - }) - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .whereRaw('u.id::text = ?', [userId]) - .groupBy('u.id') - .groupBy('p.id') - .first() - if (!row) { - return null - } - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getTopUsers( - userId: string, - tx = this.kx - ): Promise< - | (UserData & { - followersCount: number - friendsCount: number - isFriend: boolean - })[] - | null - > { - const rows = await tx({ u: this.tableName }) - .select(['u.*', this.kx.raw('to_jsonb(p) as profile')]) - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .whereIn('p.username', TOP_USERS) - .limit(MAX_RECORDS_LIMIT) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - @logMethod - async getUserFollowersList( - userId: string, - tx = this.kx - ): Promise { - const rows: UserData[] = await tx({ u: this.tableName }) - .select([`u.*`, this.kx.raw('to_jsonb(p) as profile')]) - .leftJoin(Table.USER_FRIEND, `user_id`, '=', `u.id`) - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .where(`friend_user_id`, '=', userId) - .limit(MAX_RECORDS_LIMIT) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - @logMethod - async getUserFollowingList( - userId: string, - tx = this.kx - ): Promise { - const rows: UserData[] = await tx({ u: this.tableName }) - .select([`u.*`, this.kx.raw('to_jsonb(p) as profile')]) - .leftJoin(Table.USER_FRIEND, `friend_user_id`, '=', `u.id`) - .leftJoin({ p: Table.USER_PROFILE }, 'u.id', 'p.user_id') - .where(`user_friends.user_id`, '=', userId) - .limit(MAX_RECORDS_LIMIT) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - @logMethod - async createUserWithProfile( - createSet: CreateSet, - username: string, - bio?: string - ): Promise { - return this.kx.transaction(async (tx) => { - const userData = await this.create(createSet, tx) - await this.createProfile({ username, userId: userData.id, bio }, tx) - return userData - }) - } - - @logMethod - async create(set: CreateSet, tx?: Knex.Transaction): Promise { - if (tx) { - return super.create(set, tx) - } - return this.kx.transaction((tx) => super.create(set, tx)) - } - - @logMethod - async update( - userId: string, - set: UpdateSet, - tx?: Knex.Transaction - ): Promise { - if (tx) { - return super.update(userId, set, tx) - } - return this.kx.transaction((tx) => super.update(userId, set, tx)) - } - - @logMethod - async createProfile( - set: ProfileCreateSet, - tx?: Knex.Transaction - ): Promise { - if (tx) { - const [profile] = (await tx(Table.USER_PROFILE) - .insert(set) - .returning(profileKeys)) as Profile[] - return profile - } - - return this.kx.transaction(async (tx) => { - const [profile] = (await tx(Table.USER_PROFILE) - .insert(set) - .returning(profileKeys)) as Profile[] - return profile - }) - } - - @logMethod - async updateProfile( - userId: string, - set: Partialize, - tx?: Knex.Transaction - ): Promise { - if (tx) { - const [profile]: Profile[] = await tx(Table.USER_PROFILE) - .update(set) - .where({ userId }) - .returning(profileKeys) - return profile - } - return this.kx.transaction((tx) => this.updateProfile(userId, set, tx)) - } - - @logMethod - async delete( - userId: string, - tx?: Knex.Transaction - ): Promise { - if (tx) { - return super.delete(userId, tx) - } - - return this.kx.transaction((tx) => super.delete(userId, tx)) - } -} - -export default UserModel diff --git a/packages/api/src/datalayer/user/model.ts b/packages/api/src/datalayer/user/model.ts deleted file mode 100644 index f8b182cf6..000000000 --- a/packages/api/src/datalayer/user/model.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { exclude, Partialize, PickTuple } from '../../util' - -// Table "omnivore.user" -// Column | Type | Collation | Nullable | Default -// ----------------+--------------------------+-----------+----------+--------------------------------------- -// id | uuid | | not null | uuid_generate_v1mc() -// first_name | text | | | -// last_name | text | | | -// source | registration_type | | not null | -// email | text | | | -// phone | text | | | -// source_user_id | text | | not null | -// created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP -// updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - -// Table "omnivore.user_profile" -// Column | Type | Collation | Nullable | Default -// -------------+--------------------------+-----------+----------+---------------------- -// id | uuid | | not null | uuid_generate_v1mc() -// username | text | | not null | -// private | boolean | | not null | false -// bio | text | | | -// picture_url | text | | | -// user_id | uuid | | not null | -// created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP -// updated_at | timestamp with time zone | | | - -export interface UserData { - id: string - name: string - source: string - email?: string | null - phone?: string | null - sourceUserId: string - createdAt: Date - // snake_case here because our knex case transformation doesn't support nested objects - profile: { - id: string - username: string - bio?: string | null - picture_url?: string | null - private: boolean - } - password?: string | null - status?: StatusType -} - -export enum RegistrationType { - Google = 'GOOGLE', - Apple = 'APPLE', - Email = 'EMAIL', -} - -export enum StatusType { - Active = 'ACTIVE', - Pending = 'PENDING', -} - -export const keys = [ - 'id', - 'name', - 'source', - 'email', - 'phone', - 'sourceUserId', - 'createdAt', - 'password', - 'status', -] as const - -export const defaultedKeys = ['id', 'createdAt'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [ - 'name', - 'sourceUserId', - 'source', - 'password', -] as const - -export type UpdateSet = PickTuple - -// Profile-related types -export const profileKeys = [ - 'id', - 'username', - 'bio', - 'pictureUrl', - 'private', - 'userId', -] as const - -export const createProfileKeys = exclude(profileKeys, ['id']) - -export type ProfileCreateSet = PickTuple< - UserData['profile'], - typeof createProfileKeys -> - -export const profileUpdateKeys = [ - 'username', - 'bio', - 'picture_url', - 'private', -] as const - -export type ProfileUpdateSet = PickTuple< - UserData['profile'], - typeof profileUpdateKeys -> diff --git a/packages/api/src/datalayer/user_friends/index.ts b/packages/api/src/datalayer/user_friends/index.ts deleted file mode 100644 index f568f15b7..000000000 --- a/packages/api/src/datalayer/user_friends/index.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { - CreateSet, - keys as modelKeys, - UpdateSet, - UserFriendData, -} from './model' -import DataModel, { MAX_RECORDS_LIMIT } from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { - ENABLE_DB_REQUEST_LOGGING, - globalCounter, - logMethod, - logger, -} from '../helpers' -import DataLoader from 'dataloader' - -class UserFriendModel extends DataModel { - public tableName = Table.USER_FRIEND - protected modelKeys = modelKeys - - protected getFriendsLoader: DataLoader - getFriends: DataLoader['load'] - - constructor(kx: Knex, cache = true) { - super(kx, cache) - this.getFriendsLoader = new DataLoader( - async (keys) => { - if (ENABLE_DB_REQUEST_LOGGING) { - globalCounter.log( - this.tableName, - 'user_friends_loader', - JSON.stringify(keys) - ) - } - const rows: UserFriendData[] = await this.kx(this.tableName) - .select(this.modelKeys) - .whereIn('userId', keys) - .limit(MAX_RECORDS_LIMIT) - - const keyMap: Record = {} - for (const row of rows) { - keyMap[row.userId] = [...(keyMap[row.userId] || []), row.friendUserId] - } - - const result = keys.map((userId) => keyMap[userId] || []) - // logger.debug('\n\n\n\n\nResult for userId_articleId_load', { keys, result }); - - if (result.length !== keys.length) { - logger.error('DataModel error: count mismatch ', keys, result) - } - return result - }, - { cache } - ) - - this.getFriends = this.getFriendsLoader.load.bind(this.getFriendsLoader) - } - - @logMethod - async getByUserFriendId( - userId: UserFriendData['userId'], - friendUserId: UserFriendData['friendUserId'], - tx = this.kx - ): Promise { - const row: UserFriendData | null = await tx(this.tableName) - .select() - .where({ userId, friendUserId }) - .first(this.modelKeys) - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async getFollowers( - userId: UserFriendData['userId'], - tx = this.kx - ): Promise { - const rows: UserFriendData[] = await tx(this.tableName) - .where({ friendUserId: userId }) - .select(this.modelKeys) - .limit(MAX_RECORDS_LIMIT) - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } - - @logMethod - async getByFriendIds( - userId: string, - followersIds: string[], - tx = this.kx - ): Promise { - const rows: UserFriendData[] = await tx(this.tableName) - .select(this.modelKeys) - .whereIn('friend_user_id', followersIds) - .andWhere({ userId }) - .limit(MAX_RECORDS_LIMIT) - - for (const row of rows) { - this.loader.prime(row.id, row) - } - return rows - } -} - -export default UserFriendModel diff --git a/packages/api/src/datalayer/user_friends/model.ts b/packages/api/src/datalayer/user_friends/model.ts deleted file mode 100644 index 601c78d24..000000000 --- a/packages/api/src/datalayer/user_friends/model.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * friend_user_id| uuid | | not null | - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * ``` - * */ -export interface UserFriendData { - id: string - userId: string - friendUserId: string - createdAt: Date -} - -export const keys = ['id', 'userId', 'friendUserId', 'createdAt'] as const - -export const defaultedKeys = ['id', 'createdAt'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [] as const - -export type UpdateSet = PickTuple - -export const getByParametersKeys = exclude(keys, ['id'] as const) - -export type ParametersSet = PickTuple< - UserFriendData, - typeof getByParametersKeys -> diff --git a/packages/api/src/datalayer/user_personalization/index.ts b/packages/api/src/datalayer/user_personalization/index.ts deleted file mode 100644 index 424de749d..000000000 --- a/packages/api/src/datalayer/user_personalization/index.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { - CreateSet, - keys as modelKeys, - UpdateSet, - UserPersonalizationData, -} from './model' -import DataModel from '../model' -import { Knex } from 'knex' -import { Table } from '../../utils/dictionary' -import { camelCase } from 'voca' -import { logMethod } from '../helpers' - -class UserPersonalizationModel extends DataModel< - UserPersonalizationData, - CreateSet, - UpdateSet -> { - public tableName = Table.USER_PERSONALIZATION - protected modelKeys = modelKeys - constructor(kx: Knex, cache = true) { - super(kx, cache) - } - - @logMethod - async getByUserId( - userId: UserPersonalizationData['userId'], - tx = this.kx - ): Promise { - const row: UserPersonalizationData | null = await tx(this.tableName) - .select() - .where({ userId }) - .first(this.modelKeys) - if (!row) return null - this.loader.prime(row.id, row) - return row - } - - @logMethod - async upsert( - data: CreateSet, - tx = this.kx - ): Promise { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { userId, id, ...updateSet } = data - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const row = {} as any - const result = ( - await tx.raw( - `? ON CONFLICT (user_id) - DO ?`, - [ - tx(this.tableName).insert(data), - tx.update(updateSet).returning(this.modelKeys), - ] - ) - ).rows[0] - for (const [key, value] of Object.entries(result)) { - row[camelCase(key)] = value - } - this.loader.prime(row.id, row) - return row - } -} - -export default UserPersonalizationModel diff --git a/packages/api/src/datalayer/user_personalization/model.ts b/packages/api/src/datalayer/user_personalization/model.ts deleted file mode 100644 index dffb818ec..000000000 --- a/packages/api/src/datalayer/user_personalization/model.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { exclude, Partialize, PickTuple } from '../../util' - -/** - * ``` - * Column | Type | Collation | Nullable | Default - * ---------------------+--------------------------+-----------+----------+---------------------- - * id | uuid | | not null | uuid_generate_v1mc() - * user_id | uuid | | not null | - * font_size | integer | | | - * font_family | text | | | - * theme | text | | | - * created_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * updated_at | timestamp with time zone | | not null | CURRENT_TIMESTAMP - * margin | integer | | | - * library_layout_type | text | | | - * library_sort_order | text | | | - * ``` - * */ -export interface UserPersonalizationData { - id: string - userId: string - fontFamily?: string | null - fontSize?: number | null - margin?: number | null - theme?: string | null - libraryLayoutType?: string | null - librarySortOrder?: string | null - createdAt: Date - updatedAt: Date -} - -export const keys = [ - 'id', - 'userId', - 'fontSize', - 'fontFamily', - 'margin', - 'theme', - 'libraryLayoutType', - 'librarySortOrder', - 'createdAt', - 'updatedAt', -] as const - -export const defaultedKeys = ['id', 'createdAt', 'updatedAt'] as const - -type DefaultedSet = PickTuple - -export const createKeys = exclude(keys, defaultedKeys) - -export type CreateSet = PickTuple & - Partialize - -export const updateKeys = [ - 'fontSize', - 'fontFamily', - 'margin', - 'theme', - 'libraryLayoutType', - 'librarySortOrder', -] as const - -export type UpdateSet = PickTuple - -export const getByParametersKeys = exclude(keys, ['id'] as const) - -export type ParametersSet = PickTuple< - UserPersonalizationData, - typeof getByParametersKeys -> diff --git a/packages/api/src/elastic/highlights.ts b/packages/api/src/elastic/highlights.ts deleted file mode 100644 index 431568c7d..000000000 --- a/packages/api/src/elastic/highlights.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { errors } from '@elastic/elasticsearch' -import { EntityType } from '../pubsub' -import { SortBy, SortOrder, SortParams } from '../utils/search' -import { client, INDEX_ALIAS, logger } from './index' -import { - Highlight, - Page, - PageContext, - PageType, - SearchItem, - SearchResponse, -} from './types' - -export const addHighlightToPage = async ( - id: string, - highlight: Highlight, - ctx: PageContext -): Promise => { - try { - const { body } = await client.update({ - index: INDEX_ALIAS, - id, - body: { - script: { - source: `if (ctx._source.highlights == null) { - ctx._source.highlights = [params.highlight] - } else { - ctx._source.highlights.add(params.highlight) - } - ctx._source.updatedAt = params.highlight.updatedAt`, - lang: 'painless', - params: { - highlight, - }, - }, - }, - refresh: ctx.refresh, - retry_on_conflict: 3, - }) - - if (body.result !== 'updated') return false - - await ctx.pubsub.entityCreated( - EntityType.HIGHLIGHT, - highlight, - ctx.uid - ) - - return true - } catch (e) { - if ( - e instanceof errors.ResponseError && - e.message === 'document_missing_exception' - ) { - logger.info('page has been deleted', id) - return false - } - logger.error('failed to add highlight to a page in elastic', e) - return false - } -} - -export const getHighlightById = async ( - id: string -): Promise => { - try { - const { body } = await client.search({ - index: INDEX_ALIAS, - body: { - query: { - nested: { - path: 'highlights', - query: { - term: { - 'highlights.id': id, - }, - }, - inner_hits: {}, - }, - }, - _source: false, - }, - }) - - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - if (body.hits.total.value === 0) { - return undefined - } - - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-return - return body.hits.hits[0].inner_hits.highlights.hits.hits[0]._source - } catch (e) { - logger.error('failed to get highlight from a page in elastic', e) - return undefined - } -} - -export const deleteHighlight = async ( - highlightId: string, - ctx: PageContext -): Promise => { - try { - const { body } = await client.updateByQuery({ - index: INDEX_ALIAS, - body: { - script: { - source: `ctx._source.highlights.removeIf(h -> h.id == params.highlightId); - ctx._source.updatedAt = params.updatedAt`, - lang: 'painless', - params: { - highlightId: highlightId, - updatedAt: new Date(), - }, - }, - query: { - bool: { - filter: [ - { - term: { - userId: ctx.uid, - }, - }, - { - nested: { - path: 'highlights', - query: { - term: { - 'highlights.id': highlightId, - }, - }, - }, - }, - ], - }, - }, - }, - refresh: ctx.refresh, - }) - - body.updated > 0 && - (await ctx.pubsub.entityDeleted( - EntityType.HIGHLIGHT, - highlightId, - ctx.uid - )) - - return true - } catch (e) { - logger.error('failed to delete a highlight in elastic', e) - - return false - } -} - -export const searchHighlights = async ( - args: { - from?: number - size?: number - sort?: SortParams - query?: string - }, - userId: string -): Promise<[SearchItem[], number] | undefined> => { - try { - const { from = 0, size = 10, sort, query } = args - const sortOrder = sort?.order || SortOrder.DESCENDING - // default sort by updatedAt - const sortField = - sort?.by === SortBy.SCORE ? SortBy.SCORE : 'highlights.updatedAt' - - const searchBody = { - query: { - bool: { - filter: [ - { - nested: { - path: 'highlights', - query: { - term: { - 'highlights.userId': userId, - }, - }, - }, - }, - ], - should: [ - { - multi_match: { - query: query || '', - fields: [ - 'highlights.quote^5', - 'title^3', - 'description^2', - 'content', - ], - }, - }, - ], - minimum_should_match: query ? 1 : 0, - }, - }, - sort: [ - '_score', - { - [sortField]: { - order: sortOrder, - nested: { - path: 'highlights', - }, - }, - }, - ], - from, - size, - _source: [ - 'title', - 'slug', - 'url', - 'savedAt', - 'highlights', - 'readingProgressPercent', - 'readingProgressAnchorIndex', - ], - } - - logger.info('searching highlights in elastic', searchBody) - - const response = await client.search>({ - index: INDEX_ALIAS, - body: searchBody, - }) - - if (response.body.hits.total.value === 0) { - return [[], 0] - } - - const results: SearchItem[] = [] - response.body.hits.hits.forEach((hit) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-member-access - hit._source.highlights?.forEach((highlight) => { - results.push({ - ...highlight, - ...hit._source, - pageId: hit._id, - pageType: PageType.Highlights, - }) - }) - }) - - return [results, response.body.hits.total.value] - } catch (e) { - logger.error('failed to search highlights in elastic', e) - return undefined - } -} - -export const updateHighlight = async ( - highlight: Highlight, - ctx: PageContext -): Promise => { - try { - const { body } = await client.updateByQuery({ - index: INDEX_ALIAS, - body: { - script: { - source: `ctx._source.highlights.removeIf(h -> h.id == params.highlight.id); - ctx._source.highlights.add(params.highlight); - ctx._source.updatedAt = params.highlight.updatedAt`, - lang: 'painless', - params: { - highlight, - }, - }, - query: { - bool: { - filter: [ - { - term: { - userId: ctx.uid, - }, - }, - { - nested: { - path: 'highlights', - query: { - term: { - 'highlights.id': highlight.id, - }, - }, - }, - }, - ], - }, - }, - }, - refresh: ctx.refresh, - conflicts: 'proceed', - }) - - body.updated > 0 && - (await ctx.pubsub.entityUpdated( - EntityType.HIGHLIGHT, - highlight, - ctx.uid - )) - - return true - } catch (e) { - logger.error('failed to update highlight in elastic', e) - return false - } -} diff --git a/packages/api/src/elastic/index.ts b/packages/api/src/elastic/index.ts deleted file mode 100644 index 298ef8b40..000000000 --- a/packages/api/src/elastic/index.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Client } from '@elastic/elasticsearch' -import { readFileSync } from 'fs' -import { env } from '../env' -import { buildLogger } from '../utils/logger' - -export const INDEX_ALIAS = 'pages_alias' -export const client = new Client({ - node: env.elastic.url, - maxRetries: 3, - requestTimeout: 50000, - auth: { - username: env.elastic.username, - password: env.elastic.password, - }, -}) -const INDEX_NAME = 'pages' -export const logger = buildLogger('elasticsearch') - -const createIndex = async (): Promise => { - // read index settings from file - const indexSettings = readFileSync( - __dirname + '/../../../db/elastic_migrations/index_settings.json', - 'utf8' - ) - - // create index - await client.indices.create({ - index: INDEX_NAME, - body: indexSettings, - }) -} - -export const initElasticsearch = async (): Promise => { - try { - const response = await client.info() - logger.info('elastic info: ', response) - - // check if index exists - const { body: indexExists } = await client.indices.exists({ - index: INDEX_ALIAS, - }) - if (!indexExists) { - logger.info('creating index...') - await createIndex() - - logger.info('refreshing index...') - await refreshIndex() - } - logger.info('elastic client is ready') - } catch (e) { - logger.error('failed to init elasticsearch', e) - throw e - } -} - -export const refreshIndex = async (): Promise => { - try { - const { body } = await client.indices.refresh({ - index: INDEX_ALIAS, - }) - logger.info('elastic refresh: ', body) - } catch (e) { - logger.error('failed to refresh elastic index', e) - throw e - } -} diff --git a/packages/api/src/elastic/labels.ts b/packages/api/src/elastic/labels.ts deleted file mode 100644 index d66cdc85f..000000000 --- a/packages/api/src/elastic/labels.ts +++ /dev/null @@ -1,333 +0,0 @@ -import { errors } from '@elastic/elasticsearch' -import { EntityType } from '../pubsub' -import { client, INDEX_ALIAS, logger } from './index' -import { Label, PageContext } from './types' - -export const addLabelInPage = async ( - pageId: string, - label: Label, - ctx: PageContext -): Promise => { - try { - const { body } = await client.update({ - index: INDEX_ALIAS, - id: pageId, - body: { - script: { - source: `if (ctx._source.labels == null) { - ctx._source.labels = [params.label]; - ctx._source.updatedAt = params.updatedAt - } else if (!ctx._source.labels.any(label -> label.name == params.label.name)) { - ctx._source.labels.add(params.label); - ctx._source.updatedAt = params.updatedAt - } else { ctx.op = 'none' }`, - lang: 'painless', - params: { - label: label, - updatedAt: new Date(), - }, - }, - }, - refresh: ctx.refresh, - retry_on_conflict: 3, - }) - - if (body.result !== 'updated') return false - - await ctx.pubsub.entityCreated