From fd781644f1ffb85c56bdedef2c69240a65b4c616 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Nov 2023 18:03:25 +0800 Subject: [PATCH] feat: fetch content for rss feed items in following folder --- packages/api/src/generated/graphql.ts | 4 +++ packages/api/src/generated/schema.graphql | 4 +++ packages/api/src/resolvers/article/index.ts | 4 +++ packages/api/src/schema.ts | 4 +++ .../src/services/create_page_save_request.ts | 4 +++ packages/api/src/services/save_file.ts | 27 +++++++------------ packages/api/src/services/save_page.ts | 6 ++++- packages/api/src/services/save_url.ts | 1 + packages/api/src/utils/createTask.ts | 3 +++ packages/puppeteer-parse/index.js | 19 +++++++++++-- packages/rss-handler/src/index.ts | 11 ++++---- 11 files changed, 61 insertions(+), 26 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index fe6a395b8..ecd90982e 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -271,6 +271,7 @@ export enum CreateArticleErrorCode { export type CreateArticleInput = { articleSavingRequestId?: InputMaybe; + folder?: InputMaybe; labels?: InputMaybe>; preparedDocument?: InputMaybe; skipParsing?: InputMaybe; @@ -2205,6 +2206,7 @@ export enum SaveErrorCode { export type SaveFileInput = { clientRequestId: Scalars['ID']; + folder?: InputMaybe; labels?: InputMaybe>; source: Scalars['String']; state?: InputMaybe; @@ -2240,6 +2242,7 @@ export type SaveFilterSuccess = { export type SavePageInput = { clientRequestId: Scalars['ID']; + folder?: InputMaybe; labels?: InputMaybe>; originalContent: Scalars['String']; parseResult?: InputMaybe; @@ -2262,6 +2265,7 @@ export type SaveSuccess = { export type SaveUrlInput = { clientRequestId: Scalars['ID']; + folder?: InputMaybe; labels?: InputMaybe>; locale?: InputMaybe; publishedAt?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 2903d85fb..1b09bab9b 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -228,6 +228,7 @@ enum CreateArticleErrorCode { input CreateArticleInput { articleSavingRequestId: ID + folder: String labels: [CreateLabelInput!] preparedDocument: PreparedDocumentInput skipParsing: Boolean @@ -1665,6 +1666,7 @@ enum SaveErrorCode { input SaveFileInput { clientRequestId: ID! + folder: String labels: [CreateLabelInput!] source: String! state: ArticleSavingRequestStatus @@ -1698,6 +1700,7 @@ type SaveFilterSuccess { input SavePageInput { clientRequestId: ID! + folder: String labels: [CreateLabelInput!] originalContent: String! parseResult: ParseResult @@ -1719,6 +1722,7 @@ type SaveSuccess { input SaveUrlInput { clientRequestId: ID! + folder: String labels: [CreateLabelInput!] locale: String publishedAt: Date diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 7ac91c69c..683c45256 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -137,6 +137,7 @@ export const createArticleResolver = authorized< source, state, labels: inputLabels, + folder, }, }, { log, uid, pubsub } @@ -250,6 +251,7 @@ export const createArticleResolver = authorized< url, state: state || undefined, labels: inputLabels || undefined, + folder: folder || undefined, }) return DUMMY_RESPONSE } else if (!skipParsing && preparedDocument?.document) { @@ -271,6 +273,7 @@ export const createArticleResolver = authorized< url, state: state || undefined, labels: inputLabels || undefined, + folder: folder || undefined, }) return DUMMY_RESPONSE } @@ -290,6 +293,7 @@ export const createArticleResolver = authorized< canonicalUrl, uploadFileId, state, + folder, }) log.info('New article saving', { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 1e774fc0b..b5e817c83 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -496,6 +496,7 @@ const schema = gql` source: String state: ArticleSavingRequestStatus labels: [CreateLabelInput!] + folder: String } enum CreateArticleErrorCode { UNABLE_TO_FETCH @@ -540,6 +541,7 @@ const schema = gql` uploadFileId: ID! state: ArticleSavingRequestStatus labels: [CreateLabelInput!] + folder: String } input ParseResult { @@ -569,6 +571,7 @@ const schema = gql` rssFeedUrl: String savedAt: Date publishedAt: Date + folder: String } input SaveUrlInput { @@ -581,6 +584,7 @@ const schema = gql` timezone: String savedAt: Date publishedAt: Date + folder: String } union SaveResult = SaveSuccess | SaveError diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 01db62eb2..15d0e60f1 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -35,6 +35,7 @@ interface PageSaveRequest { timezone?: string savedAt?: Date publishedAt?: Date + folder?: string } const SAVING_CONTENT = 'Your link is being saved...' @@ -89,6 +90,7 @@ export const createPageSaveRequest = async ({ timezone, savedAt, publishedAt, + folder, }: PageSaveRequest): Promise => { try { validateUrl(url) @@ -125,6 +127,7 @@ export const createPageSaveRequest = async ({ originalUrl: url, state: LibraryItemState.Processing, publishedAt, + folder, }, userId, pubsub @@ -157,6 +160,7 @@ export const createPageSaveRequest = async ({ timezone, savedAt, publishedAt, + folder, }) return libraryItemToArticleSavingRequest(user, libraryItem) diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index bffe06fb8..165fc1244 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -1,12 +1,7 @@ import { LibraryItemState } from '../entity/library_item' import { User } from '../entity/user' import { homePageURL } from '../env' -import { - ArticleSavingRequestStatus, - SaveErrorCode, - SaveFileInput, - SaveResult, -} from '../generated/graphql' +import { SaveErrorCode, SaveFileInput, SaveResult } from '../generated/graphql' import { getStorageFileDetails } from '../utils/uploads' import { findOrCreateLabels, saveLabelsInLibraryItem } from './labels' import { updateLibraryItem } from './library_item' @@ -33,23 +28,21 @@ export const saveFile = async ( } } - if (input.state || input.labels) { + if (input.state || input.folder) { await updateLibraryItem( input.clientRequestId, { - state: LibraryItemState.Succeeded, - folder: - input.state === ArticleSavingRequestStatus.Archived - ? 'archive' - : 'inbox', + state: (input.state as unknown as LibraryItemState) || undefined, + folder: input.folder || undefined, }, user.id ) - // add labels to item - if (input.labels) { - const labels = await findOrCreateLabels(input.labels, user.id) - await saveLabelsInLibraryItem(labels, input.clientRequestId, user.id) - } + } + + // add labels to item + if (input.labels) { + const labels = await findOrCreateLabels(input.labels, user.id) + await saveLabelsInLibraryItem(labels, input.clientRequestId, user.id) } return { diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 6eb5ebff3..7b6c65f74 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -97,6 +97,7 @@ export const savePage = async ( publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, state: input.state || undefined, rssFeedUrl: input.rssFeedUrl, + folder: input.folder, }) const isImported = input.source === 'csv-importer' || input.source === 'pocket' @@ -110,6 +111,7 @@ export const savePage = async ( articleSavingRequestId: clientRequestId || undefined, state: input.state || undefined, labels: input.labels || undefined, + folder: input.folder || undefined, }) } catch (e) { return { @@ -216,6 +218,7 @@ export const parsedContentToLibraryItem = ({ publishedAt, state, rssFeedUrl, + folder, }: { url: string userId: string @@ -234,6 +237,7 @@ export const parsedContentToLibraryItem = ({ publishedAt?: Date | null state?: ArticleSavingRequestStatus | null rssFeedUrl?: string | null + folder?: string | null }): DeepPartial & { originalUrl: string } => { return { id: itemId || undefined, @@ -272,7 +276,7 @@ export const parsedContentToLibraryItem = ({ wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, - folder: 'inbox', + folder: folder || 'inbox', archivedAt: state === ArticleSavingRequestStatus.Archived ? new Date() : null, } diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 3165bb8fb..806db0e53 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -20,6 +20,7 @@ export const saveUrl = async ( timezone: input.timezone || undefined, savedAt: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, + folder: input.folder || undefined, }) return { diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 3c11e5028..0ef8913e4 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -232,6 +232,7 @@ export const enqueueParseRequest = async ({ timezone, savedAt, publishedAt, + folder, }: { url: string userId: string @@ -244,6 +245,7 @@ export const enqueueParseRequest = async ({ timezone?: string savedAt?: Date publishedAt?: Date + folder?: string }): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { @@ -256,6 +258,7 @@ export const enqueueParseRequest = async ({ timezone, savedAt, publishedAt, + folder, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 64f8b91a7..dbc43c877 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -231,7 +231,7 @@ const sendCreateArticleMutation = async (userId, input) => { } }`, variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), + input, }, }); @@ -308,6 +308,10 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId url: encodeURI(url), articleSavingRequestId, uploadFileId: uploadFileId, + state, + labels, + source, + folder, }, ); }; @@ -349,6 +353,7 @@ async function fetchContent(req, res) { const rssFeedUrl = req.body.rssFeedUrl; const savedAt = req.body.savedAt; const publishedAt = req.body.publishedAt; + const folder = req.body.folder; let logRecord = { url: urlStr, @@ -365,6 +370,7 @@ async function fetchContent(req, res) { rssFeedUrl, savedAt, publishedAt, + folder, }; console.info(`Article parsing request`, logRecord); @@ -406,7 +412,15 @@ async function fetchContent(req, res) { if (contentType === 'application/pdf') { const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); - const uploadedPdf = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); + const uploadedPdf = await sendCreateArticleMutation(userId, { + url: encodeURI(finalUrl), + articleSavingRequestId, + uploadFileId: uploadedFileId, + state, + labels, + source, + folder, + }); if (!uploadedPdf) { statusCode = 500; logRecord.error = 'error while saving uploaded pdf'; @@ -474,6 +488,7 @@ async function fetchContent(req, res) { savedAt, publishedAt, source, + folder, }); if (!apiResponse) { logRecord.error = 'error while saving page'; diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 75a621ee5..e0fbb494a 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -157,17 +157,15 @@ const createTask = async ( item: RssFeedItem, autoAddToLibrary: boolean ) => { - if (autoAddToLibrary) { - return createSavingItemTask(userId, feedUrl, item) - } - - return createFollowingTask(userId, feedUrl, item) + const folder = autoAddToLibrary ? 'inbox' : 'following' + return createSavingItemTask(userId, feedUrl, item, folder) } const createSavingItemTask = async ( userId: string, feedUrl: string, - item: RssFeedItem + item: RssFeedItem, + folder: string ) => { const input = { userId, @@ -178,6 +176,7 @@ const createSavingItemTask = async ( rssFeedUrl: feedUrl, savedAt: item.isoDate, publishedAt: item.isoDate, + folder, } try {