diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 447de3bfa..5765c1469 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -24,6 +24,7 @@ export enum LibraryItemState { Succeeded = 'SUCCEEDED', Deleted = 'DELETED', Archived = 'ARCHIVED', + ContentNotFetched = 'CONTENT_NOT_FETCHED', } export enum ContentReaderType { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index aed124467..080b4614e 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -191,6 +191,7 @@ export type ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavi export enum ArticleSavingRequestStatus { Archived = 'ARCHIVED', + ContentNotFetched = 'CONTENT_NOT_FETCHED', Deleted = 'DELETED', Failed = 'FAILED', Processing = 'PROCESSING', @@ -819,6 +820,23 @@ export type FeedsSuccess = { pageInfo: PageInfo; }; +export type FetchContentError = { + __typename?: 'FetchContentError'; + errorCodes: Array; +}; + +export enum FetchContentErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type FetchContentResult = FetchContentError | FetchContentSuccess; + +export type FetchContentSuccess = { + __typename?: 'FetchContentSuccess'; + success: Scalars['Boolean']; +}; + export type Filter = { __typename?: 'Filter'; category?: Maybe; @@ -1334,6 +1352,7 @@ export type Mutation = { deleteNewsletterEmail: DeleteNewsletterEmailResult; deleteRule: DeleteRuleResult; deleteWebhook: DeleteWebhookResult; + fetchContent: FetchContentResult; generateApiKey: GenerateApiKeyResult; googleLogin: LoginResult; googleSignup: GoogleSignupResult; @@ -1466,6 +1485,11 @@ export type MutationDeleteWebhookArgs = { }; +export type MutationFetchContentArgs = { + id: Scalars['ID']; +}; + + export type MutationGenerateApiKeyArgs = { input: GenerateApiKeyInput; }; @@ -3623,6 +3647,10 @@ export type ResolversTypes = { FeedsInput: FeedsInput; FeedsResult: ResolversTypes['FeedsError'] | ResolversTypes['FeedsSuccess']; FeedsSuccess: ResolverTypeWrapper; + FetchContentError: ResolverTypeWrapper; + FetchContentErrorCode: FetchContentErrorCode; + FetchContentResult: ResolversTypes['FetchContentError'] | ResolversTypes['FetchContentSuccess']; + FetchContentSuccess: ResolverTypeWrapper; Filter: ResolverTypeWrapper; FiltersError: ResolverTypeWrapper; FiltersErrorCode: FiltersErrorCode; @@ -4118,6 +4146,9 @@ export type ResolversParentTypes = { FeedsInput: FeedsInput; FeedsResult: ResolversParentTypes['FeedsError'] | ResolversParentTypes['FeedsSuccess']; FeedsSuccess: FeedsSuccess; + FetchContentError: FetchContentError; + FetchContentResult: ResolversParentTypes['FetchContentError'] | ResolversParentTypes['FetchContentSuccess']; + FetchContentSuccess: FetchContentSuccess; Filter: Filter; FiltersError: FiltersError; FiltersResult: ResolversParentTypes['FiltersError'] | ResolversParentTypes['FiltersSuccess']; @@ -4986,6 +5017,20 @@ export type FeedsSuccessResolvers; }; +export type FetchContentErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type FetchContentResultResolvers = { + __resolveType: TypeResolveFn<'FetchContentError' | 'FetchContentSuccess', ParentType, ContextType>; +}; + +export type FetchContentSuccessResolvers = { + success?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FilterResolvers = { category?: Resolver, ParentType, ContextType>; createdAt?: Resolver; @@ -5376,6 +5421,7 @@ export type MutationResolvers>; deleteRule?: Resolver>; deleteWebhook?: Resolver>; + fetchContent?: Resolver>; generateApiKey?: Resolver>; googleLogin?: Resolver>; googleSignup?: Resolver>; @@ -6558,6 +6604,9 @@ export type Resolvers = { FeedsError?: FeedsErrorResolvers; FeedsResult?: FeedsResultResolvers; FeedsSuccess?: FeedsSuccessResolvers; + FetchContentError?: FetchContentErrorResolvers; + FetchContentResult?: FetchContentResultResolvers; + FetchContentSuccess?: FetchContentSuccessResolvers; Filter?: FilterResolvers; FiltersError?: FiltersErrorResolvers; FiltersResult?: FiltersResultResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 44f35d093..97fbbe750 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -155,6 +155,7 @@ union ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequ enum ArticleSavingRequestStatus { ARCHIVED + CONTENT_NOT_FETCHED DELETED FAILED PROCESSING @@ -727,6 +728,21 @@ type FeedsSuccess { pageInfo: PageInfo! } +type FetchContentError { + errorCodes: [FetchContentErrorCode!]! +} + +enum FetchContentErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union FetchContentResult = FetchContentError | FetchContentSuccess + +type FetchContentSuccess { + success: Boolean! +} + type Filter { category: String createdAt: Date! @@ -1197,6 +1213,7 @@ type Mutation { deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult! deleteRule(id: ID!): DeleteRuleResult! deleteWebhook(id: ID!): DeleteWebhookResult! + fetchContent(id: ID!): FetchContentResult! generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! googleLogin(input: GoogleLoginInput!): LoginResult! googleSignup(input: GoogleSignupInput!): GoogleSignupResult! diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 7d3fea678..ec7bd1e8e 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -21,11 +21,15 @@ import { CreateArticleError, CreateArticleErrorCode, CreateArticleSuccess, + FetchContentError, + FetchContentErrorCode, + FetchContentSuccess, MoveToFolderError, MoveToFolderErrorCode, MoveToFolderSuccess, MutationBulkActionArgs, MutationCreateArticleArgs, + MutationFetchContentArgs, MutationMoveToFolderArgs, MutationSaveArticleReadingProgressArgs, MutationSetBookmarkArticleArgs, @@ -67,6 +71,7 @@ import { } from '../../services/labels' import { createLibraryItem, + findLibraryItemById, findLibraryItemByUrl, findLibraryItemsByPrefix, searchLibraryItems, @@ -955,7 +960,7 @@ export const moveToFolderResolver = authorized< ) // if the content is not fetched yet, create a page save request - if (!item.readableContent) { + if (item.state === LibraryItemState.ContentNotFetched) { try { await createPageSaveRequest({ userId: uid, @@ -981,6 +986,50 @@ export const moveToFolderResolver = authorized< } }) +export const fetchContentResolver = authorized< + FetchContentSuccess, + FetchContentError, + MutationFetchContentArgs +>(async (_, { id }, { uid, log, pubsub }) => { + analytics.track({ + userId: uid, + event: 'fetch_content', + properties: { + id, + }, + }) + + const item = await findLibraryItemById(id, uid) + if (!item) { + return { + errorCodes: [FetchContentErrorCode.Unauthorized], + } + } + + // if the content is not fetched yet, create a page save request + if (item.state === LibraryItemState.ContentNotFetched) { + try { + await createPageSaveRequest({ + userId: uid, + url: item.originalUrl, + articleSavingRequestId: id, + priority: 'high', + pubsub, + }) + } catch (error) { + log.error('fetchContentResolver error', error) + + return { + errorCodes: [FetchContentErrorCode.BadRequest], + } + } + } + + return { + success: true, + } +}) + const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { if (libraryItem.deletedAt) { return UpdateReason.Deleted diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index d2f31d1b6..bb84c295a 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -1,8 +1,19 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ import express from 'express' +import { + ArticleSavingRequestStatus, + PreparedDocumentInput, +} from '../../generated/graphql' import { createAndSaveLabelsInLibraryItem } from '../../services/labels' -import { saveFeedItemInFollowing } from '../../services/library_item' +import { createLibraryItem } from '../../services/library_item' +import { parsedContentToLibraryItem } from '../../services/save_page' +import { cleanUrl, generateSlug } from '../../utils/helpers' +import { createThumbnailUrl } from '../../utils/imageproxy' import { logger } from '../../utils/logger' +import { + ParsedContentPuppeteer, + parsePreparedContent, +} from '../../utils/parser' type SourceOfFollowing = 'feed' | 'newsletter' | 'user' @@ -35,6 +46,8 @@ function isSaveFollowingItemRequest( ) } +const FOLDER = 'following' + export function followingServiceRouter() { const router = express.Router() @@ -58,20 +71,68 @@ export function followingServiceRouter() { const userId = req.body.userIds[0] logger.info('saving feed item', userId) - const result = await saveFeedItemInFollowing(req.body, userId) - if (result.identifiers.length === 0) { - logger.error('error saving feed item in following') - return res.status(500).send('ERROR_SAVING_FEED_ITEM') + const feedUrl = req.body.addedToFollowingBy + const thumbnail = + req.body.thumbnail && createThumbnailUrl(req.body.thumbnail) + const url = cleanUrl(req.body.url) + + const preparedDocument: PreparedDocumentInput = { + document: req.body.previewContent || '', + pageInfo: { + title: req.body.title, + author: req.body.author, + canonicalUrl: url, + contentType: req.body.previewContentType, + description: req.body.description, + previewImage: thumbnail, + }, + } + let parsedResult: ParsedContentPuppeteer | undefined + + // parse the content if we have a preview content + if (req.body.previewContent) { + parsedResult = await parsePreparedContent(url, preparedDocument) } + const { pathname } = new URL(url) + const croppedPathname = decodeURIComponent( + pathname + .split('/') + [pathname.split('/').length - 1].split('.') + .slice(0, -1) + .join('.') + ).replace(/_/gi, ' ') + + const slug = generateSlug( + parsedResult?.parsedContent?.title || croppedPathname + ) + const itemToSave = parsedContentToLibraryItem({ + url, + title: req.body.title, + parsedContent: parsedResult?.parsedContent || null, + userId, + slug, + croppedPathname, + originalHtml: req.body.previewContent, + itemType: parsedResult?.pageType || 'unknown', + canonicalUrl: url, + folder: FOLDER, + rssFeedUrl: feedUrl, + preparedDocument, + savedAt: req.body.savedAt, + publishedAt: req.body.publishedAt, + state: ArticleSavingRequestStatus.ContentNotFetched, + }) + + const newItem = await createLibraryItem(itemToSave, userId) logger.info('feed item saved in following') // save RSS label in the item await createAndSaveLabelsInLibraryItem( - result.identifiers[0].id, + newItem.id, userId, [{ name: 'RSS' }], - req.body.addedToFollowingBy + feedUrl ) logger.info('RSS label added to the item') diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 74b4b3658..49c64e669 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1111,6 +1111,7 @@ const schema = gql` FAILED DELETED ARCHIVED + CONTENT_NOT_FETCHED } type ArticleSavingRequest { @@ -2721,6 +2722,21 @@ const schema = gql` BAD_REQUEST } + union FetchContentResult = FetchContentSuccess | FetchContentError + + type FetchContentSuccess { + success: Boolean! + } + + type FetchContentError { + errorCodes: [FetchContentErrorCode!]! + } + + enum FetchContentErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2828,6 +2844,7 @@ const schema = gql` input: UpdateSubscriptionInput! ): UpdateSubscriptionResult! moveToFolder(id: ID!, folder: String!): MoveToFolderResult! + fetchContent(id: ID!): FetchContentResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 4e1fd6b07..bd336f28c 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -137,7 +137,7 @@ export const createPageSaveRequest = async ({ pubsub ) } - // reset state to processing if not in following + // reset state to processing if (libraryItem.state !== LibraryItemState.Processing) { libraryItem = await updateLibraryItem( libraryItem.id, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4864055da..99f143f48 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -10,9 +10,7 @@ import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { SaveFollowingItemRequest } from '../routers/svc/following' -import { generateSlug, wordsCount } from '../utils/helpers' -import { createThumbnailUrl } from '../utils/imageproxy' +import { wordsCount } from '../utils/helpers' import { parseSearchQuery } from '../utils/search' enum ReadFilter { @@ -847,38 +845,6 @@ export const createLibraryItem = async ( return newLibraryItem } -export const saveFeedItemInFollowing = ( - input: SaveFollowingItemRequest, - userId: string -) => { - const thumbnail = input.thumbnail && createThumbnailUrl(input.thumbnail) - - return authTrx( - async (tx) => { - const itemToSave: QueryDeepPartialEntity = { - ...input, - user: { id: userId }, - originalUrl: input.url, - subscription: input.addedToFollowingBy, - folder: InFilter.FOLLOWING, - slug: generateSlug(input.title), - thumbnail, - } - - return tx - .getRepository(LibraryItem) - .createQueryBuilder() - .insert() - .values(itemToSave) - .orIgnore() // ignore if the item already exists - .returning('*') - .execute() - }, - undefined, - userId - ) -} - export const findLibraryItemsByPrefix = async ( prefix: string, userId: string, diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index dff855b4f..54373baea 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -180,7 +180,7 @@ const getPurifiedContent = (html: string): Document => { const getReadabilityResult = async ( url: string, html: string, - document: Document, + document?: Document, isNewsletter?: boolean ): Promise => { // First attempt to read the article as is. diff --git a/packages/db/migrations/0151.do.add_content_not_fetched_to_library_item_state.sql b/packages/db/migrations/0151.do.add_content_not_fetched_to_library_item_state.sql new file mode 100755 index 000000000..68e5f5762 --- /dev/null +++ b/packages/db/migrations/0151.do.add_content_not_fetched_to_library_item_state.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: add_content_not_fetched_to_library_item_state +-- Description: Add CONTENT_NOT_FETCHED to the library_item_state enum + +BEGIN; + +ALTER TYPE library_item_state ADD VALUE IF NOT EXISTS 'CONTENT_NOT_FETCHED'; + +COMMIT; diff --git a/packages/db/migrations/0151.undo.add_content_not_fetched_to_library_item_state.sql b/packages/db/migrations/0151.undo.add_content_not_fetched_to_library_item_state.sql new file mode 100755 index 000000000..f6da5fd37 --- /dev/null +++ b/packages/db/migrations/0151.undo.add_content_not_fetched_to_library_item_state.sql @@ -0,0 +1,7 @@ +-- Type: UNDO +-- Name: add_content_not_fetched_to_library_item_state +-- Description: Add CONTENT_NOT_FETCHED to the library_item_state enum + +BEGIN; + +COMMIT;