From 2b70d480d257482c96fa7e14e3d0745f4b20e5be Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 29 Apr 2022 13:41:06 +0800 Subject: [PATCH] Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper --- packages/api/Dockerfile | 1 - packages/api/src/elastic/index.ts | 20 +- packages/api/src/elastic/pages.ts | 18 +- packages/api/src/elastic/types.ts | 12 +- packages/api/src/generated/graphql.ts | 8 + packages/api/src/generated/schema.graphql | 7 +- packages/api/src/resolvers/article/index.ts | 135 +++++---- .../resolvers/article_saving_request/index.ts | 50 ++-- packages/api/src/routers/article_router.ts | 6 +- .../api/src/routers/svc/pdf_attachments.ts | 3 +- packages/api/src/schema.ts | 6 +- .../src/services/create_page_save_request.ts | 57 +++- packages/api/src/services/save_email.ts | 3 +- packages/api/src/services/save_file.ts | 33 +-- packages/api/src/services/save_page.ts | 51 +--- packages/api/src/services/save_url.ts | 22 +- packages/api/src/utils/helpers.ts | 57 ++-- packages/api/test/elastic/index.test.ts | 11 +- packages/api/test/resolvers/article.test.ts | 117 +++++++- .../resolvers/article_saving_request.test.ts | 149 ++++++++++ packages/api/test/util.ts | 3 +- packages/db/Dockerfile | 1 + .../add_highlight_to_elastic.py | 0 .../elastic_migrations}/elastic_migrate.py | 0 .../elastic_migrations}/index_settings.json | 6 + packages/db/migrate.ts | 108 +++++--- packages/db/package.json | 1 + .../components/templates/SavingRequest.tsx | 32 ++- .../templates/article/ArticleActionsMenu.tsx | 45 +-- .../article/SkeletonArticleContainer.tsx | 69 +++++ .../templates/homeFeed/HomeFeedContainer.tsx | 7 +- packages/web/lib/hooks/useReaderSettings.tsx | 92 +++++++ .../networking/fragments/articleFragment.ts | 8 + .../networking/queries/useGetArticleQuery.tsx | 11 +- .../queries/useGetArticleSavingStatus.tsx | 13 +- .../queries/useGetLibraryItemsQuery.tsx | 9 +- packages/web/package.json | 1 - .../web/pages/[username]/[slug]/index.tsx | 257 ++++++++---------- packages/web/pages/[username]/links/[id].tsx | 74 ++++- .../app/[username]/link-request/[id].tsx | 16 +- packages/web/pages/article/sr/[id].tsx | 74 ++++- yarn.lock | 5 - 42 files changed, 1101 insertions(+), 497 deletions(-) create mode 100644 packages/api/test/resolvers/article_saving_request.test.ts rename packages/{api => db/elastic_migrations}/add_highlight_to_elastic.py (100%) rename packages/{api => db/elastic_migrations}/elastic_migrate.py (100%) rename packages/{api => db/elastic_migrations}/index_settings.json (95%) create mode 100644 packages/web/components/templates/article/SkeletonArticleContainer.tsx create mode 100644 packages/web/lib/hooks/useReaderSettings.tsx diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index 377397b16..6b0d2282f 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -40,7 +40,6 @@ COPY --from=builder /app/packages/api/package.json /app/packages/api/package.jso COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules COPY --from=builder /app/node_modules /app/node_modules COPY --from=builder /app/package.json /app/package.json -COPY --from=builder /app/packages/api/index_settings.json /app/packages/api/index_settings.json EXPOSE 8080 CMD ["yarn", "workspace", "@omnivore/api", "start"] diff --git a/packages/api/src/elastic/index.ts b/packages/api/src/elastic/index.ts index 118132069..2b0d8a383 100644 --- a/packages/api/src/elastic/index.ts +++ b/packages/api/src/elastic/index.ts @@ -1,7 +1,5 @@ import { env } from '../env' import { Client } from '@elastic/elasticsearch' -import { readFileSync } from 'fs' -import { join } from 'path' export const INDEX_NAME = 'pages' export const INDEX_ALIAS = 'pages_alias' @@ -15,19 +13,6 @@ export const client = new Client({ }, }) -const ingest = async (): Promise => { - // read index settings from file - const indexSettings = readFileSync( - join(__dirname, '..', '..', 'index_settings.json'), - 'utf8' - ) - // create index - await client.indices.create({ - index: INDEX_NAME, - body: indexSettings, - }) -} - export const initElasticsearch = async (): Promise => { try { const response = await client.info() @@ -38,10 +23,7 @@ export const initElasticsearch = async (): Promise => { index: INDEX_ALIAS, }) if (!indexExists) { - console.log('ingesting index...') - await ingest() - - await client.indices.refresh({ index: INDEX_ALIAS }) + throw new Error('elastic index does not exist') } console.log('elastic client is ready') } catch (e) { diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 7ed529420..f0a19434a 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -1,4 +1,5 @@ import { + ArticleSavingRequestStatus, Page, PageContext, PageType, @@ -335,6 +336,7 @@ export const searchPages = async ( savedDateFilter?: DateRangeFilter publishedDateFilter?: DateRangeFilter subscriptionFilter?: SubscriptionFilter + includePending?: boolean | null }, userId: string ): Promise<[Page[], number] | undefined> => { @@ -375,7 +377,13 @@ export const searchPages = async ( }, ], should: [], - must_not: [], + must_not: [ + { + term: { + state: ArticleSavingRequestStatus.Failed, + }, + }, + ], }, }, sort: [ @@ -424,6 +432,14 @@ export const searchPages = async ( appendSubscriptionFilter(body, subscriptionFilter) } + if (!args.includePending) { + body.query.bool.must_not.push({ + term: { + state: ArticleSavingRequestStatus.Processing, + }, + }) + } + console.log('searching pages in elastic', JSON.stringify(body)) const response = await client.search, SearchBody>({ diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 69b2af47d..c6909eb97 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -69,6 +69,7 @@ export interface SearchBody { }[] minimum_should_match?: number must_not: ( + | { term: { state: ArticleSavingRequestStatus } } | { exists: { field: string @@ -147,6 +148,12 @@ export enum PageType { Highlights = 'HIGHLIGHTS', } +export enum ArticleSavingRequestStatus { + Failed = 'FAILED', + Processing = 'PROCESSING', + Succeeded = 'SUCCEEDED', +} + export interface Label { id: string name: string @@ -199,6 +206,8 @@ export interface Page { subscription?: string unsubMailTo?: string unsubHttpUrl?: string + state: ArticleSavingRequestStatus + taskName?: string } export interface SearchItem { @@ -221,9 +230,10 @@ export interface SearchItem { readingProgressPercent?: number readingProgressAnchorIndex?: number userId: string + state?: ArticleSavingRequestStatus } -const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId'] as const +const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const export type ParamSet = PickTuple diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 779435430..e70bf2dd9 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -69,6 +69,7 @@ export type Article = { siteIcon?: Maybe; siteName?: Maybe; slug: Scalars['String']; + state?: Maybe; subscription?: Maybe; title: Scalars['String']; unsubHttpUrl?: Maybe; @@ -107,10 +108,12 @@ export type ArticleResult = ArticleError | ArticleSuccess; export type ArticleSavingRequest = { __typename?: 'ArticleSavingRequest'; + /** @deprecated article has been replaced with slug */ article?: Maybe
; createdAt: Scalars['Date']; errorCode?: Maybe; id: Scalars['ID']; + slug: Scalars['String']; status: ArticleSavingRequestStatus; updatedAt: Scalars['Date']; user: User; @@ -1152,6 +1155,7 @@ export type QueryArticleArgs = { export type QueryArticlesArgs = { after?: InputMaybe; first?: InputMaybe; + includePending?: InputMaybe; query?: InputMaybe; sharedOnly?: InputMaybe; sort?: InputMaybe; @@ -1381,6 +1385,7 @@ export type SearchItem = { readingProgressPercent?: Maybe; shortId?: Maybe; slug: Scalars['String']; + state?: Maybe; subscription?: Maybe; title: Scalars['String']; unsubHttpUrl?: Maybe; @@ -2649,6 +2654,7 @@ export type ArticleResolvers, ParentType, ContextType>; siteName?: Resolver, ParentType, ContextType>; slug?: Resolver; + state?: Resolver, ParentType, ContextType>; subscription?: Resolver, ParentType, ContextType>; title?: Resolver; unsubHttpUrl?: Resolver, ParentType, ContextType>; @@ -2678,6 +2684,7 @@ export type ArticleSavingRequestResolvers; errorCode?: Resolver, ParentType, ContextType>; id?: Resolver; + slug?: Resolver; status?: Resolver; updatedAt?: Resolver; user?: Resolver; @@ -3378,6 +3385,7 @@ export type SearchItemResolvers, ParentType, ContextType>; shortId?: Resolver, ParentType, ContextType>; slug?: Resolver; + state?: Resolver, ParentType, ContextType>; subscription?: Resolver, ParentType, ContextType>; title?: Resolver; unsubHttpUrl?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 703d79014..f6dfe4e86 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -50,6 +50,7 @@ type Article { siteIcon: String siteName: String slug: String! + state: ArticleSavingRequestStatus subscription: String title: String! unsubHttpUrl: String @@ -80,10 +81,11 @@ input ArticleHighlightsInput { union ArticleResult = ArticleError | ArticleSuccess type ArticleSavingRequest { - article: Article + article: Article @deprecated(reason: "article has been replaced with slug") createdAt: Date! errorCode: CreateArticleErrorCode id: ID! + slug: String! status: ArticleSavingRequestStatus! updatedAt: Date! user: User! @@ -817,7 +819,7 @@ type Profile { type Query { article(slug: String!, username: String!): ArticleResult! - articles(after: String, first: Int, query: String, sharedOnly: Boolean, sort: SortParams): ArticlesResult! + articles(after: String, first: Int, includePending: Boolean, query: String, sharedOnly: Boolean, sort: SortParams): ArticlesResult! articleSavingRequest(id: ID!): ArticleSavingRequestResult! feedArticles(after: String, first: Int, sharedByUser: ID, sort: SortParams): FeedArticlesResult! getFollowers(userId: ID): GetFollowersResult! @@ -990,6 +992,7 @@ type SearchItem { readingProgressPercent: Float shortId: String slug: String! + state: ArticleSavingRequestStatus subscription: String title: String! unsubHttpUrl: String diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 21221ca0e..4aa1ac171 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -19,7 +19,6 @@ import { MutationSetBookmarkArticleArgs, MutationSetShareArticleArgs, PageInfo, - PageType, QueryArticleArgs, QueryArticlesArgs, QuerySearchArgs, @@ -45,10 +44,9 @@ import { } from '../../utils/uploads' import { ContentParseError } from '../../utils/errors' import { - articleSavingRequestError, - articleSavingRequestPopulate, authorized, generateSlug, + pageError, stringToHash, userDataToUser, validatedDate, @@ -72,7 +70,12 @@ import { createIntercomEvent } from '../../utils/intercom' import { analytics } from '../../utils/analytics' import { env } from '../../env' -import { Page, SearchItem as SearchItemData } from '../../elastic/types' +import { + ArticleSavingRequestStatus, + Page, + PageType, + SearchItem as SearchItemData, +} from '../../elastic/types' import { createPage, deletePage, @@ -100,6 +103,7 @@ const FORCE_PUPPETEER_URLS = [ /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/, /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/, ] +const UNPARSEABLE_CONTENT = 'We were unable to parse this page.' export type CreateArticlesSuccessPartial = Merge< CreateArticleSuccess, @@ -116,7 +120,7 @@ export const createArticleResolver = authorized< input: { url, preparedDocument, - articleSavingRequestId, + articleSavingRequestId: pageId, uploadFileId, skipParsing, source, @@ -142,25 +146,15 @@ export const createArticleResolver = authorized< }) await createIntercomEvent('link-saved', uid) - const articleSavingRequest = articleSavingRequestId - ? (await models.articleSavingRequest.get(articleSavingRequestId)) || - (await authTrx((tx) => - models.articleSavingRequest.create( - { userId: uid, id: articleSavingRequestId }, - tx - ) - )) - : undefined - const user = userDataToUser(await models.user.get(uid)) try { if (isSiteBlockedForParse(url)) { - return articleSavingRequestError( + return pageError( { errorCodes: [CreateArticleErrorCode.NotAllowedToParse], }, ctx, - articleSavingRequest + pageId ) } @@ -213,10 +207,10 @@ export const createArticleResolver = authorized< userId: uid, }) if (!uploadFile) { - return articleSavingRequestError( + return pageError( { errorCodes: [CreateArticleErrorCode.UploadFileMissing] }, ctx, - articleSavingRequest + pageId ) } const uploadFileDetails = await getStorageFileDetails( @@ -281,14 +275,12 @@ export const createArticleResolver = authorized< siteIcon: parsedContent?.siteIcon, readingProgressPercent: 0, readingProgressAnchorIndex: 0, + state: ArticleSavingRequestStatus.Succeeded, } let archive = false - if (articleSavingRequestId) { - const reminder = await models.reminder.getByRequestId( - uid, - articleSavingRequestId - ) + if (pageId) { + const reminder = await models.reminder.getByRequestId(uid, pageId) if (reminder) { archive = reminder.archiveUntil || false } @@ -313,12 +305,12 @@ export const createArticleResolver = authorized< return await models.uploadFile.setFileUploadComplete(uploadFileId, tx) }) if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { - return articleSavingRequestError( + return pageError( { errorCodes: [CreateArticleErrorCode.UploadFileMissing], }, ctx, - articleSavingRequest + pageId ) } uploadFileUrlOverride = await makeStorageFilePublic( @@ -330,6 +322,7 @@ export const createArticleResolver = authorized< const existingPage = await getPageByParam({ userId: uid, url: articleToSave.url, + state: ArticleSavingRequestStatus.Succeeded, }) if (existingPage) { // update existing page in elastic @@ -345,17 +338,34 @@ export const createArticleResolver = authorized< articleToSave = existingPage } else { // create new page in elastic - const pageId = await createPage(articleToSave, { ...ctx, uid }) - if (!pageId) { - return articleSavingRequestError( - { - errorCodes: [CreateArticleErrorCode.ElasticError], - }, - ctx, - articleSavingRequest - ) + pageId = await createPage(articleToSave, { ...ctx, uid }) + if (!pageId) { + return pageError( + { + errorCodes: [CreateArticleErrorCode.ElasticError], + }, + ctx, + pageId + ) + } + } else { + const updated = await updatePage(pageId, articleToSave, { + ...ctx, + uid, + }) + + if (!updated) { + return pageError( + { + errorCodes: [CreateArticleErrorCode.ElasticError], + }, + ctx, + pageId + ) + } } + log.info( 'page created in elastic', pageId, @@ -370,25 +380,20 @@ export const createArticleResolver = authorized< ...articleToSave, isArchived: !!articleToSave.archivedAt, } - return articleSavingRequestPopulate( - { - user, - created: false, - createdArticle: createdArticle, - }, - ctx, - articleSavingRequest?.id, - createdArticle.id || undefined - ) + return { + user, + created: false, + createdArticle: createdArticle, + } } catch (error) { if ( error instanceof ContentParseError && error.message === 'UNABLE_TO_PARSE' ) { - return articleSavingRequestError( + return pageError( { errorCodes: [CreateArticleErrorCode.UnableToParse] }, ctx, - articleSavingRequest + pageId ) } throw error @@ -426,10 +431,19 @@ export const getArticleResolver: ResolverFn< return { errorCodes: [ArticleErrorCode.NotFound] } } + if ( + page.state === ArticleSavingRequestStatus.Processing && + new Date(page.createdAt).getTime() < new Date().getTime() - 1000 * 60 + ) { + page.content = UNPARSEABLE_CONTENT + page.description = UNPARSEABLE_CONTENT + } + return { article: { ...page, isArchived: !!page.archivedAt, linkId: page.id }, } } catch (error) { + console.log(error) return { errorCodes: [ArticleErrorCode.BadData] } } } @@ -483,6 +497,7 @@ export const getArticlesResolver = authorized< savedDateFilter: searchQuery.savedDateFilter, publishedDateFilter: searchQuery.publishedDateFilter, subscriptionFilter: searchQuery.subscriptionFilter, + includePending: params.includePending, }, claims.uid )) || [[], 0] @@ -618,29 +633,29 @@ export const setBookmarkArticleResolver = authorized< { input: { articleID, bookmark } }, { models, authTrx, claims: { uid }, log, pubsub } ) => { - const article = await getPageById(articleID) - if (!article) { + const page = await getPageById(articleID) + if (!page) { return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] } } if (!bookmark) { - const userArticleRemoved = await getPageByParam({ + const pageRemoved = await getPageByParam({ userId: uid, _id: articleID, }) - if (!userArticleRemoved) { + if (!pageRemoved) { return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] } } - await deletePage(userArticleRemoved.id, { pubsub, uid }) + await deletePage(pageRemoved.id, { pubsub, uid }) const highlightsUnshared = await authTrx(async (tx) => { return models.highlight.unshareAllHighlights(articleID, uid, tx) }) log.info('Article unbookmarked', { - article: Object.assign({}, article, { + page: Object.assign({}, page, { content: undefined, originalHtml: undefined, }), @@ -655,7 +670,7 @@ export const setBookmarkArticleResolver = authorized< // Make sure article.id instead of userArticle.id has passed. We use it for cache updates return { bookmarkedArticle: { - ...userArticleRemoved, + ...pageRemoved, isArchived: false, savedByViewer: false, postedByViewer: false, @@ -663,14 +678,14 @@ export const setBookmarkArticleResolver = authorized< } } else { try { - const userArticle: Partial = { + const pageUpdated: Partial = { userId: uid, - slug: generateSlug(article.title), + slug: generateSlug(page.title), } - await updatePage(articleID, userArticle, { pubsub, uid }) + await updatePage(articleID, pageUpdated, { pubsub, uid }) log.info('Article bookmarked', { - article: Object.assign({}, article, { + page: Object.assign({}, page, { content: undefined, originalHtml: undefined, }), @@ -684,8 +699,8 @@ export const setBookmarkArticleResolver = authorized< // Make sure article.id instead of userArticle.id has passed. We use it for cache updates return { bookmarkedArticle: { - ...userArticle, - ...article, + ...pageUpdated, + ...page, isArchived: false, savedByViewer: true, postedByViewer: false, diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 68acd39a6..44b2112ba 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -1,29 +1,39 @@ /* eslint-disable prefer-const */ import { - ArticleSavingRequestSuccess, ArticleSavingRequestError, - QueryArticleSavingRequestArgs, ArticleSavingRequestErrorCode, - CreateArticleSavingRequestSuccess, + ArticleSavingRequestSuccess, CreateArticleSavingRequestError, + CreateArticleSavingRequestErrorCode, + CreateArticleSavingRequestSuccess, MutationCreateArticleSavingRequestArgs, + QueryArticleSavingRequestArgs, } from '../../generated/graphql' -import { - authorized, - articleSavingRequestDataToArticleSavingRequest, -} from '../../utils/helpers' +import { authorized, pageToArticleSavingRequest } from '../../utils/helpers' import { createPageSaveRequest } from '../../services/create_page_save_request' import { createIntercomEvent } from '../../utils/intercom' +import { getPageById } from '../../elastic/pages' +import { isErrorWithCode } from '../user' export const createArticleSavingRequestResolver = authorized< CreateArticleSavingRequestSuccess, CreateArticleSavingRequestError, MutationCreateArticleSavingRequestArgs ->(async (_, { input: { url } }, { models, claims }) => { +>(async (_, { input: { url } }, { models, claims, pubsub }) => { await createIntercomEvent('link-save-request', claims.uid) - const request = await createPageSaveRequest(claims.uid, url, models) - return { - articleSavingRequest: request, + try { + const request = await createPageSaveRequest(claims.uid, url, models, pubsub) + return { + articleSavingRequest: request, + } + } catch (err) { + console.log('error', err) + if (isErrorWithCode(err)) { + return { + errorCodes: [err.errorCode as CreateArticleSavingRequestErrorCode], + } + } + return { errorCodes: [CreateArticleSavingRequestErrorCode.BadData] } } }) @@ -32,20 +42,18 @@ export const articleSavingRequestResolver = authorized< ArticleSavingRequestError, QueryArticleSavingRequestArgs >(async (_, { id }, { models }) => { - let articleSavingRequest + let page let user try { - articleSavingRequest = await models.articleSavingRequest.get(id) - user = await models.user.get(articleSavingRequest.userId) + page = await getPageById(id) + if (!page) { + return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] } + } + user = await models.user.get(page.userId) // eslint-disable-next-line no-empty } catch (error) {} - if (user && articleSavingRequest) - return { - articleSavingRequest: articleSavingRequestDataToArticleSavingRequest( - user, - articleSavingRequest - ), - } + if (user && page) + return { articleSavingRequest: pageToArticleSavingRequest(user, page) } return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] } }) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 396319222..930a577ba 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -10,7 +10,6 @@ import { env } from './../env' import { buildLogger } from './../utils/logger' import * as jwt from 'jsonwebtoken' import { corsConfig } from '../utils/corsConfig' -import { v4 as uuidv4 } from 'uuid' import { createPageSaveRequest } from '../services/create_page_save_request' import { initModels } from '../server' import { kx } from '../datalayer/knex_config' @@ -45,9 +44,8 @@ export function articleRouter() { return res.status(400).send({ errorCode: 'BAD_DATA' }) } - const requestId = uuidv4() const models = initModels(kx, false) - const result = await createPageSaveRequest(uid, url, models, requestId) + const result = await createPageSaveRequest(uid, url, models) if (isSiteBlockedForParse(url)) { return res @@ -60,7 +58,7 @@ export function articleRouter() { } return res.send({ - articleSavingRequestId: requestId, + articleSavingRequestId: result.id, }) }) return router diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index c57253632..4fcba0176 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -15,7 +15,7 @@ import { getNewsletterEmail } from '../../services/newsletters' import { setClaims } from '../../datalayer/helpers' import { generateSlug } from '../../utils/helpers' import { createPubSubClient } from '../../datalayer/pubsub' -import { Page } from '../../elastic/types' +import { ArticleSavingRequestStatus, Page } from '../../elastic/types' import { createPage } from '../../elastic/pages' export function pdfAttachmentsRouter() { @@ -157,6 +157,7 @@ export function pdfAttachmentsRouter() { createdAt: new Date(), readingProgressPercent: 0, readingProgressAnchorIndex: 0, + state: ArticleSavingRequestStatus.Succeeded, } const pageId = await createPage(articleToSave, { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index f8c27e5b8..836606f1b 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -345,6 +345,7 @@ const schema = gql` subscription: String unsubMailTo: String unsubHttpUrl: String + state: ArticleSavingRequestStatus } # Query: article @@ -954,7 +955,8 @@ const schema = gql` id: ID! userId: ID! @deprecated(reason: "userId has been replaced with user") user: User! - article: Article + article: Article @deprecated(reason: "article has been replaced with slug") + slug: String! status: ArticleSavingRequestStatus! errorCode: CreateArticleErrorCode createdAt: Date! @@ -1435,6 +1437,7 @@ const schema = gql` subscription: String unsubMailTo: String unsubHttpUrl: String + state: ArticleSavingRequestStatus } type SearchItemEdge { @@ -1583,6 +1586,7 @@ const schema = gql` after: String first: Int query: String + includePending: Boolean ): ArticlesResult! article(username: String!, slug: String!): ArticleResult! sharedArticle( diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 60d82fc69..4c0170012 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -7,9 +7,14 @@ import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, } from '../generated/graphql' -import { articleSavingRequestDataToArticleSavingRequest } from '../utils/helpers' +import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers' import * as privateIpLib from 'private-ip' -import { countByCreatedAt } from '../elastic/pages' +import { countByCreatedAt, createPage, getPageByParam } from '../elastic/pages' +import { ArticleSavingRequestStatus, Page, PageType } from '../elastic/types' +import { createPubSubClient, PubsubClient } from '../datalayer/pubsub' +import normalizeUrl from 'normalize-url' + +const SAVING_DESCRIPTION = 'Your link is being saved...' const isPrivateIP = privateIpLib.default @@ -53,6 +58,7 @@ export const createPageSaveRequest = async ( userId: string, url: string, models: DataModels, + pubsub: PubsubClient = createPubSubClient(), articleSavingRequestId = uuidv4(), priority?: 'low' | 'high' ): Promise => { @@ -76,6 +82,10 @@ export const createPageSaveRequest = async ( // get priority by checking rate limit if not specified priority = priority || (await getPriorityByRateLimit(userId)) + url = normalizeUrl(url, { + stripHash: true, + stripWWW: false, + }) const createdTaskName = await enqueueParseRequest( url, userId, @@ -83,14 +93,41 @@ export const createPageSaveRequest = async ( priority ) - const articleSavingRequestData = await models.articleSavingRequest.create({ - userId: userId, - taskName: createdTaskName, - id: articleSavingRequestId, + const existingPage = await getPageByParam({ + userId, + url, + state: ArticleSavingRequestStatus.Succeeded, }) + if (existingPage) { + console.log('Page already exists', url) + existingPage.taskName = createdTaskName + return pageToArticleSavingRequest(user, existingPage) + } - return articleSavingRequestDataToArticleSavingRequest( - user, - articleSavingRequestData - ) + const page: Page = { + id: articleSavingRequestId, + userId, + content: SAVING_DESCRIPTION, + createdAt: new Date(), + hash: '', + pageType: PageType.Unknown, + readingProgressAnchorIndex: 0, + readingProgressPercent: 0, + slug: generateSlug(url), + title: url, + url, + taskName: createdTaskName, + state: ArticleSavingRequestStatus.Processing, + description: SAVING_DESCRIPTION, + } + + const pageId = await createPage(page, { pubsub, uid: userId }) + if (!pageId) { + console.log('Failed to create page', page) + return Promise.reject({ + errorCode: CreateArticleSavingRequestErrorCode.BadData, + }) + } + + return pageToArticleSavingRequest(user, page) } diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index acfd34e64..9088fa0e8 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -6,7 +6,7 @@ import { } from '../utils/parser' import normalizeUrl from 'normalize-url' import { PubsubClient } from '../datalayer/pubsub' -import { Page } from '../elastic/types' +import { ArticleSavingRequestStatus, Page } from '../elastic/types' import { createPage, getPageByParam, updatePage } from '../elastic/pages' export type SaveContext = { @@ -69,6 +69,7 @@ export const saveEmail = async ( readingProgressAnchorIndex: 0, readingProgressPercent: 0, subscription: input.author, + state: ArticleSavingRequestStatus.Succeeded, } const page = await getPageByParam({ userId: ctx.uid, url: articleToSave.url }) diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index dabf5f09a..b8cf39307 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -3,7 +3,6 @@ import { PubsubClient } from '../datalayer/pubsub' import { UserData } from '../datalayer/user/model' import { homePageURL } from '../env' import { - ArticleSavingRequestStatus, PageType, SaveErrorCode, SaveFileInput, @@ -12,8 +11,8 @@ import { import { DataModels } from '../resolvers/types' import { generateSlug } from '../utils/helpers' import { getStorageFileDetails, makeStorageFilePublic } from '../utils/uploads' -import { createSavingRequest } from './save_page' import { createPage, getPageByParam, updatePage } from '../elastic/pages' +import { ArticleSavingRequestStatus } from '../elastic/types' type SaveContext = { pubsub: PubsubClient @@ -45,8 +44,6 @@ export const saveFile = async ( } } - const savingRequest = await createSavingRequest(ctx, input.clientRequestId) - const uploadFileDetails = await getStorageFileDetails( input.uploadFileId, uploadFile.fileName @@ -71,6 +68,7 @@ export const saveFile = async ( const matchedUserArticleRecord = await getPageByParam({ userId: saver.id, url: uploadFileUrlOverride, + state: ArticleSavingRequestStatus.Succeeded, }) if (matchedUserArticleRecord) { @@ -82,17 +80,7 @@ export const saveFile = async ( }, ctx ) - - await ctx.authTrx(async (tx) => { - await ctx.models.articleSavingRequest.update( - savingRequest.id, - { - elasticPageId: matchedUserArticleRecord.id, - status: ArticleSavingRequestStatus.Succeeded, - }, - tx - ) - }) + input.clientRequestId = matchedUserArticleRecord.id } else { const pageId = await createPage( { @@ -104,10 +92,11 @@ export const saveFile = async ( uploadFileId: input.uploadFileId, slug: generateSlug(uploadFile.fileName), userId: saver.id, - id: '', + id: input.clientRequestId, createdAt: new Date(), readingProgressPercent: 0, readingProgressAnchorIndex: 0, + state: ArticleSavingRequestStatus.Succeeded, }, ctx ) @@ -118,17 +107,7 @@ export const saveFile = async ( errorCodes: [SaveErrorCode.Unknown], } } - - await ctx.authTrx(async (tx) => { - await ctx.models.articleSavingRequest.update( - savingRequest.id, - { - elasticPageId: pageId, - status: ArticleSavingRequestStatus.Succeeded, - }, - tx - ) - }) + input.clientRequestId = pageId } return { diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 9feecffc4..f54cc351e 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -1,20 +1,13 @@ import { PubsubClient } from '../datalayer/pubsub' import { homePageURL } from '../env' -import { - ArticleSavingRequestStatus, - Maybe, - SavePageInput, - SaveResult, -} from '../generated/graphql' +import { Maybe, SavePageInput, SaveResult } from '../generated/graphql' import { DataModels } from '../resolvers/types' import { generateSlug, stringToHash, validatedDate } from '../utils/helpers' import { parseOriginalContent, parsePreparedContent } from '../utils/parser' import normalizeUrl from 'normalize-url' import { createPageSaveRequest } from './create_page_save_request' -import { kx } from '../datalayer/knex_config' -import { setClaims } from '../datalayer/helpers' -import { Page } from '../elastic/types' +import { ArticleSavingRequestStatus, Page } from '../elastic/types' import { createPage, getPageByParam, updatePage } from '../elastic/pages' type SaveContext = { @@ -70,8 +63,6 @@ export const savePage = async ( saver: SaverUserData, input: SavePageInput ): Promise => { - const savingRequest = await createSavingRequest(ctx, input.clientRequestId) - const [slug, croppedPathname] = createSlug(input.url, input.title) const parseResult = await parsePreparedContent(input.url, { document: input.originalContent, @@ -84,7 +75,7 @@ export const savePage = async ( const pageType = parseOriginalContent(input.url, input.originalContent) const articleToSave: Page = { - id: '', + id: input.clientRequestId, slug, userId: saver.userId, originalHtml: parseResult.domContent, @@ -103,11 +94,13 @@ export const savePage = async ( createdAt: new Date(), readingProgressPercent: 0, readingProgressAnchorIndex: 0, + state: ArticleSavingRequestStatus.Succeeded, } const existingPage = await getPageByParam({ userId: saver.userId, url: articleToSave.url, + state: ArticleSavingRequestStatus.Succeeded, }) if (existingPage) { await updatePage( @@ -118,33 +111,17 @@ export const savePage = async ( }, ctx ) - await kx.transaction(async (tx) => { - await setClaims(tx, saver.userId) - await ctx.models.articleSavingRequest.update( - savingRequest.id, - { - status: ArticleSavingRequestStatus.Succeeded, - elasticPageId: existingPage.id, - }, - tx - ) - }) + input.clientRequestId = existingPage.id } else if (shouldParseInBackend(input)) { - await createPageSaveRequest(saver.userId, input.url, ctx.models) + await createPageSaveRequest( + saver.userId, + input.url, + ctx.models, + ctx.pubsub, + input.clientRequestId + ) } else { - const pageId = await createPage(articleToSave, ctx) - - await kx.transaction(async (tx) => { - await setClaims(tx, saver.userId) - await ctx.models.articleSavingRequest.update( - savingRequest.id, - { - status: ArticleSavingRequestStatus.Succeeded, - elasticPageId: pageId, - }, - tx - ) - }) + await createPage(articleToSave, ctx) } return { diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 2df37cc97..ed26080dc 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -1,7 +1,7 @@ import { PubsubClient } from '../datalayer/pubsub' import { UserData } from '../datalayer/user/model' import { homePageURL } from '../env' -import { SaveResult, SaveUrlInput } from '../generated/graphql' +import { SaveErrorCode, SaveResult, SaveUrlInput } from '../generated/graphql' import { DataModels } from '../resolvers/types' import { createPageSaveRequest } from './create_page_save_request' @@ -16,20 +16,24 @@ export const saveUrl = async ( input: SaveUrlInput ): Promise => { try { - await createPageSaveRequest( + const pageSaveRequest = await createPageSaveRequest( saver.id, input.url, ctx.models, + ctx.pubsub, input.clientRequestId ) + + return { + clientRequestId: pageSaveRequest.id, + url: `${homePageURL()}/${saver.profile.username}/links/${ + pageSaveRequest.id + }`, + } } catch (error) { console.log('error enqueuing request', error) - } - - return { - clientRequestId: input.clientRequestId, - url: `${homePageURL()}/${saver.profile.username}/links/${ - input.clientRequestId - }`, + return { + errorCodes: [SaveErrorCode.Unknown], + } } } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 52ba466ad..f5499fd56 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -1,9 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { ArticleSavingRequest, - ArticleSavingRequestStatus, CreateArticleError, - CreateArticleErrorCode, FeedArticle, Profile, ResolverFn, @@ -17,8 +15,9 @@ import { import crypto from 'crypto' import slugify from 'voca/slugify' import { Merge } from '../util' -import { ArticleSavingRequestData } from '../datalayer/article_saving_request/model' import { CreateArticlesSuccessPartial } from '../resolvers' +import { ArticleSavingRequestStatus, Page } from '../elastic/types' +import { updatePage } from '../elastic/pages' interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -175,56 +174,32 @@ export const generateSlug = (title: string): string => { export const MAX_CONTENT_LENGTH = 5e7 //50MB -export const articleSavingRequestError = async ( +export const pageError = async ( result: CreateArticleError, ctx: WithDataSourcesContext, - articleSavingReqest?: ArticleSavingRequestData + pageId?: string | null ): Promise => { - if (!articleSavingReqest) return result + if (!pageId) return result - await ctx.authTrx((tx) => - ctx.models.articleSavingRequest.update( - articleSavingReqest.id, - { - status: ArticleSavingRequestStatus.Failed, - errorCode: result.errorCodes[0], - }, - tx - ) + await updatePage( + pageId, + { + state: ArticleSavingRequestStatus.Failed, + }, + ctx ) return result } -export const articleSavingRequestPopulate = async ( - result: CreateArticlesSuccessPartial, - ctx: WithDataSourcesContext, - articleSavingReqestId: string | undefined, - articleId: string | undefined -): Promise => { - if (!articleSavingReqestId) return result - await ctx.authTrx((tx) => - ctx.models.articleSavingRequest.update( - articleSavingReqestId, - { - status: ArticleSavingRequestStatus.Succeeded, - elasticPageId: articleId, - }, - tx - ) - ) - - return result -} - -export const articleSavingRequestDataToArticleSavingRequest = ( +export const pageToArticleSavingRequest = ( user: UserData, - articleSavingRequest: ArticleSavingRequestData + page: Page ): ArticleSavingRequest => ({ - ...articleSavingRequest, + ...page, user: userDataToUser(user), - status: articleSavingRequest.status as ArticleSavingRequestStatus, - errorCode: articleSavingRequest.errorCode as CreateArticleErrorCode, + status: page.state, + updatedAt: page.updatedAt || new Date(), }) export const validatedDate = ( diff --git a/packages/api/test/elastic/index.test.ts b/packages/api/test/elastic/index.test.ts index d3d6e73eb..7815db81b 100644 --- a/packages/api/test/elastic/index.test.ts +++ b/packages/api/test/elastic/index.test.ts @@ -1,7 +1,13 @@ import 'mocha' import { expect } from 'chai' import { InFilter, ReadFilter } from '../../src/utils/search' -import { Highlight, Page, PageContext, PageType } from '../../src/elastic/types' +import { + ArticleSavingRequestStatus, + Highlight, + Page, + PageContext, + PageType, +} from '../../src/elastic/types' import { createPubSubClient } from '../../src/datalayer/pubsub' import { countByCreatedAt, @@ -58,6 +64,7 @@ describe('elastic api', () => { createdAt: new Date(), }, ], + state: ArticleSavingRequestStatus.Succeeded, } const pageId = await createPage(page, ctx) if (!pageId) { @@ -94,6 +101,7 @@ describe('elastic api', () => { readingProgressPercent: 0, readingProgressAnchorIndex: 0, url: 'https://blog.omnivore.app/testUrl', + state: ArticleSavingRequestStatus.Succeeded, } newPageId = await createPage(newPageData, ctx) @@ -197,6 +205,7 @@ describe('elastic api', () => { readingProgressPercent: 0, readingProgressAnchorIndex: 0, url: 'https://blog.omnivore.app/testCount', + state: ArticleSavingRequestStatus.Succeeded, } await createPage(newPageData, ctx) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 901329bbe..c6b4e10fe 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -12,7 +12,13 @@ import { User } from '../../src/entity/user' import chaiString from 'chai-string' import { Label } from '../../src/entity/label' import { UploadFileStatus } from '../../src/generated/graphql' -import { Highlight, Page, PageContext, PageType } from '../../src/elastic/types' +import { + ArticleSavingRequestStatus, + Highlight, + Page, + PageContext, + PageType, +} from '../../src/elastic/types' import { UploadFile } from '../../src/entity/upload_file' import { createPubSubClient } from '../../src/datalayer/pubsub' import { getRepository } from '../../src/entity/utils' @@ -133,6 +139,7 @@ const getArticleQuery = (slug: string) => { article { id slug + content highlights { id shortId @@ -230,6 +237,27 @@ const saveFileQuery = (url: string, uploadFileId: string) => { ` } +const saveUrlQuery = (url: string) => { + return ` + mutation { + saveUrl( + input: { + url: "${url}", + source: "test", + clientRequestId: "${generateFakeUuid()}", + } + ) { + ... on SaveSuccess { + url + } + ... on SaveError { + errorCodes + } + } + } + ` +} + const setBookmarkQuery = (articleId: string, bookmark: boolean) => { return ` mutation { @@ -344,7 +372,7 @@ describe('Article API', () => { let query = '' let slug = '' - let pageId: string | undefined + let pageId: string before(async () => { const page = { @@ -371,13 +399,12 @@ describe('Article API', () => { }, ], } as Page - pageId = await createPage(page, ctx) + const id = await createPage(page, ctx) + id && (pageId = id) }) after(async () => { - if (pageId) { - await deletePage(pageId, ctx) - } + await deletePage(pageId, ctx) }) beforeEach(async () => { @@ -400,6 +427,27 @@ describe('Article API', () => { expect(res.body.data.article.article.highlights).to.length(1) }) + + context('when page is failed to process', () => { + before(async () => { + await updatePage( + pageId, + { + state: ArticleSavingRequestStatus.Processing, + createdAt: new Date(Date.now() - 1000 * 60), + }, + ctx + ) + }) + + it('should return unable to parse', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.article.article.content).to.eql( + 'We were unable to parse this page.' + ) + }) + }) }) context('when page does not exist', () => { @@ -613,6 +661,61 @@ describe('Article API', () => { }) }) + describe('SaveUrl', () => { + let query = '' + let url = 'https://example.com/new-url-1' + + beforeEach(() => { + query = saveUrlQuery(url) + }) + + context('when we save a new url', () => { + it('should return a slugged url', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + expect(res.body.data.saveUrl.url).to.startsWith( + 'http://localhost:3000/fakeUser/links/' + ) + }) + }) + + context('when we save a url that is already archived', () => { + it('it should return that page in the GetArticles Query', async () => { + url = 'https://example.com/new-url' + await graphqlRequest(saveUrlQuery(url), authToken).expect(200) + + let allLinks + // Save a link, then archive it + // set a slight delay to make sure the page is updated + setTimeout(async () => { + let allLinks = await graphqlRequest( + articlesQuery(''), + authToken + ).expect(200) + const justSavedId = allLinks.body.data.articles.edges[0].node.id + await archiveLink(authToken, justSavedId) + }, 100) + + // test the negative case, ensuring the archive link wasn't returned + setTimeout(async () => { + allLinks = await graphqlRequest(articlesQuery(''), authToken).expect( + 200 + ) + expect(allLinks.body.data.articles.edges[0].node.url).to.not.eq(url) + }, 100) + + // Now save the link again, and ensure it is returned + await graphqlRequest(saveUrlQuery(url), authToken).expect(200) + + setTimeout(async () => { + allLinks = await graphqlRequest(articlesQuery(''), authToken).expect( + 200 + ) + expect(allLinks.body.data.articles.edges[0].node.url).to.eq(url) + }, 100) + }) + }) + }) + describe('setBookmarkArticle', () => { let query = '' let articleId = '' @@ -632,6 +735,7 @@ describe('Article API', () => { slug: 'test-with-omnivore', readingProgressPercent: 0, readingProgressAnchorIndex: 0, + state: ArticleSavingRequestStatus.Succeeded, } const newPageId = await createPage(page, ctx) if (newPageId) { @@ -803,6 +907,7 @@ describe('Article API', () => { readingProgressAnchorIndex: 0, url: url, savedAt: new Date(), + state: ArticleSavingRequestStatus.Succeeded, } const pageId = await createPage(page, ctx) if (!pageId) { diff --git a/packages/api/test/resolvers/article_saving_request.test.ts b/packages/api/test/resolvers/article_saving_request.test.ts new file mode 100644 index 000000000..43cb5cfa8 --- /dev/null +++ b/packages/api/test/resolvers/article_saving_request.test.ts @@ -0,0 +1,149 @@ +import { User } from '../../src/entity/user' +import { + ArticleSavingRequestStatus, + PageContext, +} from '../../src/elastic/types' +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' +import { createPubSubClient } from '../../src/datalayer/pubsub' +import { expect } from 'chai' +import { describe } from 'mocha' +import { getPageById } from '../../src/elastic/pages' +import { + ArticleSavingRequestErrorCode, + CreateArticleSavingRequestErrorCode, +} from '../../src/generated/graphql' + +const articleSavingRequestQuery = (id: string) => ` + query { + articleSavingRequest(id: "${id}") { + ... on ArticleSavingRequestSuccess { + articleSavingRequest { + id + status + } + } + ... on ArticleSavingRequestError { + errorCodes + } + } + } +` + +const createArticleSavingRequestMutation = (url: string) => ` + mutation { + createArticleSavingRequest(input: { + url: "${url}" + }) { + ... on CreateArticleSavingRequestSuccess { + articleSavingRequest { + id + status + } + } + ... on CreateArticleSavingRequestError { + errorCodes + } + } + } +` + +describe('ArticleSavingRequest API', () => { + const username = 'fakeUser' + let authToken: string + let user: User + let ctx: PageContext + + before(async () => { + // create test user and login + user = await createTestUser(username) + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + authToken = res.body.authToken + + ctx = { + pubsub: createPubSubClient(), + refresh: true, + uid: user.id, + } + }) + + after(async () => { + // clean up + await deleteTestUser(username) + }) + + describe('createArticleSavingRequest', () => { + it('returns the article saving request', async () => { + const res = await graphqlRequest( + createArticleSavingRequestMutation('https://example.com'), + authToken + ).expect(200) + + expect( + res.body.data.createArticleSavingRequest.articleSavingRequest.status + ).to.eql(ArticleSavingRequestStatus.Processing) + }) + + it('creates a page in elastic', async () => { + const res = await graphqlRequest( + createArticleSavingRequestMutation('https://example.com/1'), + authToken + ).expect(200) + + const page = await getPageById( + res.body.data.createArticleSavingRequest.articleSavingRequest.id + ) + expect(page?.description).to.eq('Your link is being saved...') + }) + + it('returns an error if the url is invalid', async () => { + const res = await graphqlRequest( + createArticleSavingRequestMutation('invalid url'), + authToken + ).expect(200) + + expect(res.body.data.createArticleSavingRequest.errorCodes).to.eql([ + CreateArticleSavingRequestErrorCode.BadData, + ]) + }) + }) + + describe('articleSavingRequest', () => { + let articleSavingRequestId: string + + before(async () => { + // create article saving request + const res = await graphqlRequest( + createArticleSavingRequestMutation('https://example.com/2'), + authToken + ).expect(200) + articleSavingRequestId = + res.body.data.createArticleSavingRequest.articleSavingRequest.id + }) + + it('returns the article saving request if exists', async () => { + const res = await graphqlRequest( + articleSavingRequestQuery(articleSavingRequestId), + authToken + ).expect(200) + + expect(res.body.data.articleSavingRequest.articleSavingRequest.id).to.eql( + articleSavingRequestId + ) + }) + + it('returns not_found if not exists', async () => { + const res = await graphqlRequest( + articleSavingRequestQuery('invalid-id'), + authToken + ).expect(200) + + expect(res.body.data.articleSavingRequest.errorCodes).to.eql([ + ArticleSavingRequestErrorCode.NotFound, + ]) + }) + }) +}) diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index 1d09d798c..d235a2c66 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -2,7 +2,7 @@ import { createApp } from '../src/server' import supertest from 'supertest' import { v4 } from 'uuid' import { corsConfig } from '../src/utils/corsConfig' -import { Page } from '../src/elastic/types' +import { ArticleSavingRequestStatus, Page } from '../src/elastic/types' import { PageType } from '../src/generated/graphql' import { User } from '../src/entity/user' import { Label } from '../src/entity/label' @@ -56,6 +56,7 @@ export const createTestElasticPage = async ( labels: labels, readingProgressPercent: 0, readingProgressAnchorIndex: 0, + state: ArticleSavingRequestStatus.Succeeded, } const pageId = await createPage(page, { diff --git a/packages/db/Dockerfile b/packages/db/Dockerfile index 9b5fe3890..6bc8424a0 100644 --- a/packages/db/Dockerfile +++ b/packages/db/Dockerfile @@ -14,5 +14,6 @@ RUN yarn install ADD /packages/db ./packages/db ADD /packages/db/setup.sh ./packages/db/setup.sh +ADD /packages/db/elastic_migrations ./packages/db/elastic_migrations CMD ["yarn", "workspace", "@omnivore/db", "migrate"] diff --git a/packages/api/add_highlight_to_elastic.py b/packages/db/elastic_migrations/add_highlight_to_elastic.py similarity index 100% rename from packages/api/add_highlight_to_elastic.py rename to packages/db/elastic_migrations/add_highlight_to_elastic.py diff --git a/packages/api/elastic_migrate.py b/packages/db/elastic_migrations/elastic_migrate.py similarity index 100% rename from packages/api/elastic_migrate.py rename to packages/db/elastic_migrations/elastic_migrate.py diff --git a/packages/api/index_settings.json b/packages/db/elastic_migrations/index_settings.json similarity index 95% rename from packages/api/index_settings.json rename to packages/db/elastic_migrations/index_settings.json index bf7aebcfd..dc51766ee 100644 --- a/packages/api/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -101,6 +101,12 @@ "subscription": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "state": { + "type": "keyword" + }, + "taskName": { + "type": "keyword" } } } diff --git a/packages/db/migrate.ts b/packages/db/migrate.ts index 442822e8a..d31992ba7 100755 --- a/packages/db/migrate.ts +++ b/packages/db/migrate.ts @@ -1,19 +1,22 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import * as dotenv from 'dotenv'; -dotenv.config(); +import * as dotenv from 'dotenv' +import Postgrator from 'postgrator' +import chalk from 'chalk' +import { Client } from '@elastic/elasticsearch' +import { readFileSync } from 'fs' +import { join } from 'path' -import Postgrator from 'postgrator'; -import chalk from 'chalk'; +dotenv.config() const log = (text: string, style: typeof chalk.white = chalk.white): void => - console.log(`${chalk.cyanBright('>')} ${style(text)}`); + console.log(`${chalk.cyanBright('>')} ${style(text)}`) interface DBEnv { - host: string; - port: number; - database: string; - username: string; - password: string; + host: string + port: number + database: string + username: string + password: string } const getEnv = (): DBEnv => { @@ -23,14 +26,14 @@ const getEnv = (): DBEnv => { PG_DB: database, PG_USER: username, PG_PASSWORD: password, - } = process.env; + } = process.env if (typeof username !== 'string') { - throw new Error('No PG user passed in env'); + throw new Error('No PG user passed in env') } if (typeof password !== 'string') { - throw new Error('No PG password passed in env'); + throw new Error('No PG password passed in env') } const config = { @@ -39,10 +42,10 @@ const getEnv = (): DBEnv => { database: database || 'omnivore', username, password, - }; + } - return config; -}; + return config +} const postgrator = new Postgrator({ migrationDirectory: __dirname + '/migrations', @@ -52,37 +55,76 @@ const postgrator = new Postgrator({ schemaTable: 'schemaversion', // Validate migration md5 checksum to ensure the contents of the script have not changed validateChecksums: true, -}); +}) -log('Starting migration manager'); +log('Starting migration manager') -const targetMigration = process.argv[2]; +const targetMigration = process.argv[2] const targetMigrationLabel = targetMigration ? `'${chalk.blue(targetMigration)}'` - : chalk.blue('latest'); + : chalk.blue('latest') -log(`Migrating to ${targetMigrationLabel}.\n`); +log(`Migrating to ${targetMigrationLabel}.\n`) -const logAppliedMigrations = (appliedMigrations: Postgrator.Migration[]): void => { +const logAppliedMigrations = ( + appliedMigrations: Postgrator.Migration[] +): void => { if (appliedMigrations.length > 0) { - log(`Applied ${chalk.green(appliedMigrations.length.toString())} migrations successfully:`); + log( + `Applied ${chalk.green( + appliedMigrations.length.toString() + )} migrations successfully:` + ) for (const migration of appliedMigrations) { - const actionLabel = migration.action === 'do' ? chalk.green('+') : chalk.red('-'); - console.log(` ${actionLabel} ${migration.name}`); + const actionLabel = + migration.action === 'do' ? chalk.green('+') : chalk.red('-') + console.log(` ${actionLabel} ${migration.name}`) } } else { - log(`No migrations applied.`); + log(`No migrations applied.`) } -}; +} + +export const INDEX_ALIAS = 'pages_alias' +export const client = new Client({ + node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200', + auth: { + username: process.env.ELASTICSEARCH_USERNAME || '', + password: process.env.ELASTICSEARCH_PASSWORD || '', + }, +}) + +const updateMappings = async (): Promise => { + // read index settings from file + const indexSettings = readFileSync( + join(__dirname, 'elastic_migrations', 'index_settings.json'), + 'utf8' + ) + + // update mappings + await client.indices.putMapping({ + index: INDEX_ALIAS, + body: JSON.parse(indexSettings).mappings, + }) +} postgrator .migrate(targetMigration) .then(logAppliedMigrations) - .catch(error => { - log(`${chalk.red('Migration failed: ')}${error.message}`, chalk.red); - const { appliedMigrations } = error; - logAppliedMigrations(appliedMigrations); - process.exit(1); + .catch((error) => { + log(`${chalk.red('Migration failed: ')}${error.message}`, chalk.red) + const { appliedMigrations } = error + logAppliedMigrations(appliedMigrations) + process.exit(1) + }) + .then(() => console.log('\nExiting...')) + +log('Starting updating elasticsearch index mappings...') + +updateMappings() + .then(() => console.log('\nUpdating elastic completed.')) + .catch((error) => { + log(`${chalk.red('Updating failed: ')}${error.message}`, chalk.red) + process.exit(1) }) - .then(() => console.log('\nExiting...')); diff --git a/packages/db/package.json b/packages/db/package.json index fce36c4af..bf7cfd766 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -9,6 +9,7 @@ "author": "", "license": "ISC", "dependencies": { + "@elastic/elasticsearch": "~7.12.0", "dotenv": "^8.2.0", "pg": "^8.3.0", "postgrator": "^4.1.1", diff --git a/packages/web/components/templates/SavingRequest.tsx b/packages/web/components/templates/SavingRequest.tsx index 40c98bbc7..04604ad04 100644 --- a/packages/web/components/templates/SavingRequest.tsx +++ b/packages/web/components/templates/SavingRequest.tsx @@ -5,11 +5,11 @@ import { StyledText } from '../elements/StyledText' export function Loader(): JSX.Element { const breathe = keyframes({ - '0%': { transform: 'scale(0.8)' }, - '25%': { transform: 'scale(1)' }, - '50%': { transform: 'scale(0.8)' }, - '75%': { transform: 'scale(1)' }, - '100%': { transform: 'scale(0.8)' }, + '0%': { content: '' }, + '25%': { content: '.' }, + '50%': { content: '..' }, + '75%': { content: '...' }, + '100%': { content: '....' }, }) return ( @@ -18,22 +18,25 @@ export function Loader(): JSX.Element { distribution="center" css={{ pt: '$6', - bg: '$grayBase', width: '100%', }} > - + - Saving Link... + Saving Link ) } @@ -55,7 +58,6 @@ export function ErrorComponent(props: ErrorComponentProps): JSX.Element { css={{ pt: '$6', px: '$3', - bg: '$grayBase', width: '100%', }} > diff --git a/packages/web/components/templates/article/ArticleActionsMenu.tsx b/packages/web/components/templates/article/ArticleActionsMenu.tsx index a5121d9c8..1500b4105 100644 --- a/packages/web/components/templates/article/ArticleActionsMenu.tsx +++ b/packages/web/components/templates/article/ArticleActionsMenu.tsx @@ -12,7 +12,7 @@ import { ReaderSettingsControl } from "./ReaderSettingsControl" export type ArticleActionsMenuLayout = 'top' | 'side' type ArticleActionsMenuProps = { - article: ArticleAttributes + article?: ArticleAttributes layout: ArticleActionsMenuLayout lineHeight: number marginWidth: number @@ -97,24 +97,35 @@ export function ArticleActionsMenu(props: ArticleActionsMenuProps): JSX.Element display: 'none', }}} > - + + + } > + + + ) : ( + + )}