Merge pull request #3261 from omnivore-app/feature/add-state-for-non-fetch-content

feature/add state for non fetch content

packages/api/src/entity/library_item.ts

@@ -24,6 +24,7 @@ export enum LibraryItemState {
   Succeeded = 'SUCCEEDED',
   Deleted = 'DELETED',
   Archived = 'ARCHIVED',
+  ContentNotFetched = 'CONTENT_NOT_FETCHED',
 }
 
 export enum ContentReaderType {

packages/api/src/generated/graphql.ts

@@ -191,6 +191,7 @@ export type ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavi
 
 export enum ArticleSavingRequestStatus {
   Archived = 'ARCHIVED',
+  ContentNotFetched = 'CONTENT_NOT_FETCHED',
   Deleted = 'DELETED',
   Failed = 'FAILED',
   Processing = 'PROCESSING',
@@ -819,6 +820,23 @@ export type FeedsSuccess = {
   pageInfo: PageInfo;
 };
 
+export type FetchContentError = {
+  __typename?: 'FetchContentError';
+  errorCodes: Array<FetchContentErrorCode>;
+};
+
+export enum FetchContentErrorCode {
+  BadRequest = 'BAD_REQUEST',
+  Unauthorized = 'UNAUTHORIZED'
+}
+
+export type FetchContentResult = FetchContentError | FetchContentSuccess;
+
+export type FetchContentSuccess = {
+  __typename?: 'FetchContentSuccess';
+  success: Scalars['Boolean'];
+};
+
 export type Filter = {
   __typename?: 'Filter';
   category?: Maybe<Scalars['String']>;
@@ -1334,6 +1352,7 @@ export type Mutation = {
   deleteNewsletterEmail: DeleteNewsletterEmailResult;
   deleteRule: DeleteRuleResult;
   deleteWebhook: DeleteWebhookResult;
+  fetchContent: FetchContentResult;
   generateApiKey: GenerateApiKeyResult;
   googleLogin: LoginResult;
   googleSignup: GoogleSignupResult;
@@ -1466,6 +1485,11 @@ export type MutationDeleteWebhookArgs = {
 };
 
 
+export type MutationFetchContentArgs = {
+  id: Scalars['ID'];
+};
+
+
 export type MutationGenerateApiKeyArgs = {
   input: GenerateApiKeyInput;
 };
@@ -3623,6 +3647,10 @@ export type ResolversTypes = {
   FeedsInput: FeedsInput;
   FeedsResult: ResolversTypes['FeedsError'] | ResolversTypes['FeedsSuccess'];
   FeedsSuccess: ResolverTypeWrapper<FeedsSuccess>;
+  FetchContentError: ResolverTypeWrapper<FetchContentError>;
+  FetchContentErrorCode: FetchContentErrorCode;
+  FetchContentResult: ResolversTypes['FetchContentError'] | ResolversTypes['FetchContentSuccess'];
+  FetchContentSuccess: ResolverTypeWrapper<FetchContentSuccess>;
   Filter: ResolverTypeWrapper<Filter>;
   FiltersError: ResolverTypeWrapper<FiltersError>;
   FiltersErrorCode: FiltersErrorCode;
@@ -4118,6 +4146,9 @@ export type ResolversParentTypes = {
   FeedsInput: FeedsInput;
   FeedsResult: ResolversParentTypes['FeedsError'] | ResolversParentTypes['FeedsSuccess'];
   FeedsSuccess: FeedsSuccess;
+  FetchContentError: FetchContentError;
+  FetchContentResult: ResolversParentTypes['FetchContentError'] | ResolversParentTypes['FetchContentSuccess'];
+  FetchContentSuccess: FetchContentSuccess;
   Filter: Filter;
   FiltersError: FiltersError;
   FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess'];
@@ -4986,6 +5017,20 @@ export type FeedsSuccessResolvers<ContextType = ResolverContext, ParentType exte
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type FetchContentErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['FetchContentError'] = ResolversParentTypes['FetchContentError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['FetchContentErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type FetchContentResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['FetchContentResult'] = ResolversParentTypes['FetchContentResult']> = {
+  __resolveType: TypeResolveFn<'FetchContentError' | 'FetchContentSuccess', ParentType, ContextType>;
+};
+
+export type FetchContentSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['FetchContentSuccess'] = ResolversParentTypes['FetchContentSuccess']> = {
+  success?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type FilterResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Filter'] = ResolversParentTypes['Filter']> = {
   category?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
@@ -5376,6 +5421,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   deleteNewsletterEmail?: Resolver<ResolversTypes['DeleteNewsletterEmailResult'], ParentType, ContextType, RequireFields<MutationDeleteNewsletterEmailArgs, 'newsletterEmailId'>>;
   deleteRule?: Resolver<ResolversTypes['DeleteRuleResult'], ParentType, ContextType, RequireFields<MutationDeleteRuleArgs, 'id'>>;
   deleteWebhook?: Resolver<ResolversTypes['DeleteWebhookResult'], ParentType, ContextType, RequireFields<MutationDeleteWebhookArgs, 'id'>>;
+  fetchContent?: Resolver<ResolversTypes['FetchContentResult'], ParentType, ContextType, RequireFields<MutationFetchContentArgs, 'id'>>;
   generateApiKey?: Resolver<ResolversTypes['GenerateApiKeyResult'], ParentType, ContextType, RequireFields<MutationGenerateApiKeyArgs, 'input'>>;
   googleLogin?: Resolver<ResolversTypes['LoginResult'], ParentType, ContextType, RequireFields<MutationGoogleLoginArgs, 'input'>>;
   googleSignup?: Resolver<ResolversTypes['GoogleSignupResult'], ParentType, ContextType, RequireFields<MutationGoogleSignupArgs, 'input'>>;
@@ -6558,6 +6604,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   FeedsError?: FeedsErrorResolvers<ContextType>;
   FeedsResult?: FeedsResultResolvers<ContextType>;
   FeedsSuccess?: FeedsSuccessResolvers<ContextType>;
+  FetchContentError?: FetchContentErrorResolvers<ContextType>;
+  FetchContentResult?: FetchContentResultResolvers<ContextType>;
+  FetchContentSuccess?: FetchContentSuccessResolvers<ContextType>;
   Filter?: FilterResolvers<ContextType>;
   FiltersError?: FiltersErrorResolvers<ContextType>;
   FiltersResult?: FiltersResultResolvers<ContextType>;

packages/api/src/generated/schema.graphql

@@ -155,6 +155,7 @@ union ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequ
 
 enum ArticleSavingRequestStatus {
   ARCHIVED
+  CONTENT_NOT_FETCHED
   DELETED
   FAILED
   PROCESSING
@@ -727,6 +728,21 @@ type FeedsSuccess {
   pageInfo: PageInfo!
 }
 
+type FetchContentError {
+  errorCodes: [FetchContentErrorCode!]!
+}
+
+enum FetchContentErrorCode {
+  BAD_REQUEST
+  UNAUTHORIZED
+}
+
+union FetchContentResult = FetchContentError | FetchContentSuccess
+
+type FetchContentSuccess {
+  success: Boolean!
+}
+
 type Filter {
   category: String
   createdAt: Date!
@@ -1197,6 +1213,7 @@ type Mutation {
   deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult!
   deleteRule(id: ID!): DeleteRuleResult!
   deleteWebhook(id: ID!): DeleteWebhookResult!
+  fetchContent(id: ID!): FetchContentResult!
   generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult!
   googleLogin(input: GoogleLoginInput!): LoginResult!
   googleSignup(input: GoogleSignupInput!): GoogleSignupResult!

packages/api/src/resolvers/article/index.ts

@@ -21,11 +21,15 @@ import {
   CreateArticleError,
   CreateArticleErrorCode,
   CreateArticleSuccess,
+  FetchContentError,
+  FetchContentErrorCode,
+  FetchContentSuccess,
   MoveToFolderError,
   MoveToFolderErrorCode,
   MoveToFolderSuccess,
   MutationBulkActionArgs,
   MutationCreateArticleArgs,
+  MutationFetchContentArgs,
   MutationMoveToFolderArgs,
   MutationSaveArticleReadingProgressArgs,
   MutationSetBookmarkArticleArgs,
@@ -67,6 +71,7 @@ import {
 } from '../../services/labels'
 import {
   createLibraryItem,
+  findLibraryItemById,
   findLibraryItemByUrl,
   findLibraryItemsByPrefix,
   searchLibraryItems,
@@ -955,7 +960,7 @@ export const moveToFolderResolver = authorized<
   )
 
   // if the content is not fetched yet, create a page save request
-  if (!item.readableContent) {
+  if (item.state === LibraryItemState.ContentNotFetched) {
     try {
       await createPageSaveRequest({
         userId: uid,
@@ -981,6 +986,50 @@ export const moveToFolderResolver = authorized<
   }
 })
 
+export const fetchContentResolver = authorized<
+  FetchContentSuccess,
+  FetchContentError,
+  MutationFetchContentArgs
+>(async (_, { id }, { uid, log, pubsub }) => {
+  analytics.track({
+    userId: uid,
+    event: 'fetch_content',
+    properties: {
+      id,
+    },
+  })
+
+  const item = await findLibraryItemById(id, uid)
+  if (!item) {
+    return {
+      errorCodes: [FetchContentErrorCode.Unauthorized],
+    }
+  }
+
+  // if the content is not fetched yet, create a page save request
+  if (item.state === LibraryItemState.ContentNotFetched) {
+    try {
+      await createPageSaveRequest({
+        userId: uid,
+        url: item.originalUrl,
+        articleSavingRequestId: id,
+        priority: 'high',
+        pubsub,
+      })
+    } catch (error) {
+      log.error('fetchContentResolver error', error)
+
+      return {
+        errorCodes: [FetchContentErrorCode.BadRequest],
+      }
+    }
+  }
+
+  return {
+    success: true,
+  }
+})
+
 const getUpdateReason = (libraryItem: LibraryItem, since: Date) => {
   if (libraryItem.deletedAt) {
     return UpdateReason.Deleted

packages/api/src/routers/svc/following.ts

@@ -1,8 +1,19 @@
 /* eslint-disable @typescript-eslint/no-misused-promises */
 import express from 'express'
+import {
+  ArticleSavingRequestStatus,
+  PreparedDocumentInput,
+} from '../../generated/graphql'
 import { createAndSaveLabelsInLibraryItem } from '../../services/labels'
-import { saveFeedItemInFollowing } from '../../services/library_item'
+import { createLibraryItem } from '../../services/library_item'
+import { parsedContentToLibraryItem } from '../../services/save_page'
+import { cleanUrl, generateSlug } from '../../utils/helpers'
+import { createThumbnailUrl } from '../../utils/imageproxy'
 import { logger } from '../../utils/logger'
+import {
+  ParsedContentPuppeteer,
+  parsePreparedContent,
+} from '../../utils/parser'
 
 type SourceOfFollowing = 'feed' | 'newsletter' | 'user'
 
@@ -35,6 +46,8 @@ function isSaveFollowingItemRequest(
   )
 }
 
+const FOLDER = 'following'
+
 export function followingServiceRouter() {
   const router = express.Router()
 
@@ -58,20 +71,68 @@ export function followingServiceRouter() {
       const userId = req.body.userIds[0]
       logger.info('saving feed item', userId)
 
-      const result = await saveFeedItemInFollowing(req.body, userId)
-      if (result.identifiers.length === 0) {
-        logger.error('error saving feed item in following')
-        return res.status(500).send('ERROR_SAVING_FEED_ITEM')
+      const feedUrl = req.body.addedToFollowingBy
+      const thumbnail =
+        req.body.thumbnail && createThumbnailUrl(req.body.thumbnail)
+      const url = cleanUrl(req.body.url)
+
+      const preparedDocument: PreparedDocumentInput = {
+        document: req.body.previewContent || '',
+        pageInfo: {
+          title: req.body.title,
+          author: req.body.author,
+          canonicalUrl: url,
+          contentType: req.body.previewContentType,
+          description: req.body.description,
+          previewImage: thumbnail,
+        },
+      }
+      let parsedResult: ParsedContentPuppeteer | undefined
+
+      // parse the content if we have a preview content
+      if (req.body.previewContent) {
+        parsedResult = await parsePreparedContent(url, preparedDocument)
       }
 
+      const { pathname } = new URL(url)
+      const croppedPathname = decodeURIComponent(
+        pathname
+          .split('/')
+          [pathname.split('/').length - 1].split('.')
+          .slice(0, -1)
+          .join('.')
+      ).replace(/_/gi, ' ')
+
+      const slug = generateSlug(
+        parsedResult?.parsedContent?.title || croppedPathname
+      )
+      const itemToSave = parsedContentToLibraryItem({
+        url,
+        title: req.body.title,
+        parsedContent: parsedResult?.parsedContent || null,
+        userId,
+        slug,
+        croppedPathname,
+        originalHtml: req.body.previewContent,
+        itemType: parsedResult?.pageType || 'unknown',
+        canonicalUrl: url,
+        folder: FOLDER,
+        rssFeedUrl: feedUrl,
+        preparedDocument,
+        savedAt: req.body.savedAt,
+        publishedAt: req.body.publishedAt,
+        state: ArticleSavingRequestStatus.ContentNotFetched,
+      })
+
+      const newItem = await createLibraryItem(itemToSave, userId)
       logger.info('feed item saved in following')
 
       // save RSS label in the item
       await createAndSaveLabelsInLibraryItem(
-        result.identifiers[0].id,
+        newItem.id,
         userId,
         [{ name: 'RSS' }],
-        req.body.addedToFollowingBy
+        feedUrl
       )
 
       logger.info('RSS label added to the item')

packages/api/src/schema.ts

@@ -1111,6 +1111,7 @@ const schema = gql`
     FAILED
     DELETED
     ARCHIVED
+    CONTENT_NOT_FETCHED
   }
 
   type ArticleSavingRequest {
@@ -2721,6 +2722,21 @@ const schema = gql`
     BAD_REQUEST
   }
 
+  union FetchContentResult = FetchContentSuccess | FetchContentError
+
+  type FetchContentSuccess {
+    success: Boolean!
+  }
+
+  type FetchContentError {
+    errorCodes: [FetchContentErrorCode!]!
+  }
+
+  enum FetchContentErrorCode {
+    UNAUTHORIZED
+    BAD_REQUEST
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -2828,6 +2844,7 @@ const schema = gql`
       input: UpdateSubscriptionInput!
     ): UpdateSubscriptionResult!
     moveToFolder(id: ID!, folder: String!): MoveToFolderResult!
+    fetchContent(id: ID!): FetchContentResult!
   }
 
   # FIXME: remove sort from feedArticles after all cached tabs are closed

packages/api/src/services/create_page_save_request.ts

@@ -137,7 +137,7 @@ export const createPageSaveRequest = async ({
       pubsub
     )
   }
-  // reset state to processing if not in following
+  // reset state to processing
   if (libraryItem.state !== LibraryItemState.Processing) {
     libraryItem = await updateLibraryItem(
       libraryItem.id,

packages/api/src/services/library_item.ts

@@ -10,9 +10,7 @@ import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
 import { createPubSubClient, EntityType } from '../pubsub'
 import { authTrx, getColumns } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
-import { SaveFollowingItemRequest } from '../routers/svc/following'
-import { generateSlug, wordsCount } from '../utils/helpers'
-import { createThumbnailUrl } from '../utils/imageproxy'
+import { wordsCount } from '../utils/helpers'
 import { parseSearchQuery } from '../utils/search'
 
 enum ReadFilter {
@@ -847,38 +845,6 @@ export const createLibraryItem = async (
   return newLibraryItem
 }
 
-export const saveFeedItemInFollowing = (
-  input: SaveFollowingItemRequest,
-  userId: string
-) => {
-  const thumbnail = input.thumbnail && createThumbnailUrl(input.thumbnail)
-
-  return authTrx(
-    async (tx) => {
-      const itemToSave: QueryDeepPartialEntity<LibraryItem> = {
-        ...input,
-        user: { id: userId },
-        originalUrl: input.url,
-        subscription: input.addedToFollowingBy,
-        folder: InFilter.FOLLOWING,
-        slug: generateSlug(input.title),
-        thumbnail,
-      }
-
-      return tx
-        .getRepository(LibraryItem)
-        .createQueryBuilder()
-        .insert()
-        .values(itemToSave)
-        .orIgnore() // ignore if the item already exists
-        .returning('*')
-        .execute()
-    },
-    undefined,
-    userId
-  )
-}
-
 export const findLibraryItemsByPrefix = async (
   prefix: string,
   userId: string,

packages/api/src/utils/parser.ts

@@ -180,7 +180,7 @@ const getPurifiedContent = (html: string): Document => {
 const getReadabilityResult = async (
   url: string,
   html: string,
-  document: Document,
+  document?: Document,
   isNewsletter?: boolean
 ): Promise<Readability.ParseResult | null> => {
   // First attempt to read the article as is.

packages/db/migrations/0151.do.add_content_not_fetched_to_library_item_state.sql

@@ -0,0 +1,9 @@
+-- Type: DO
+-- Name: add_content_not_fetched_to_library_item_state
+-- Description: Add CONTENT_NOT_FETCHED to the library_item_state enum
+
+BEGIN;
+
+ALTER TYPE library_item_state ADD VALUE IF NOT EXISTS 'CONTENT_NOT_FETCHED';
+
+COMMIT;

packages/db/migrations/0151.undo.add_content_not_fetched_to_library_item_state.sql

@@ -0,0 +1,7 @@
+-- Type: UNDO
+-- Name: add_content_not_fetched_to_library_item_state
+-- Description: Add CONTENT_NOT_FETCHED to the library_item_state enum
+
+BEGIN;
+
+COMMIT;