From 3759e106158fb427500f1ce22d3237c416046439 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 28 Nov 2023 17:44:28 +0800 Subject: [PATCH 1/3] fix feed url in pdf file not saved --- packages/api/src/generated/graphql.ts | 3 +++ packages/api/src/generated/schema.graphql | 3 +++ packages/api/src/resolvers/article/index.ts | 16 +++++++++++++++- packages/api/src/schema.ts | 3 +++ .../api/src/services/create_page_save_request.ts | 5 +++++ packages/api/src/services/save_page.ts | 9 ++++----- packages/api/src/utils/createTask.ts | 3 +++ packages/puppeteer-parse/index.js | 5 ++++- 8 files changed, 40 insertions(+), 7 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index c50b7510f..d485da3a2 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -275,6 +275,9 @@ export type CreateArticleInput = { folder?: InputMaybe; labels?: InputMaybe>; preparedDocument?: InputMaybe; + publishedAt?: InputMaybe; + rssFeedUrl?: InputMaybe; + savedAt?: InputMaybe; skipParsing?: InputMaybe; source?: InputMaybe; state?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 7a426ab80..15cb6e347 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -232,6 +232,9 @@ input CreateArticleInput { folder: String labels: [CreateLabelInput!] preparedDocument: PreparedDocumentInput + publishedAt: Date + rssFeedUrl: String + savedAt: Date skipParsing: Boolean source: String state: ArticleSavingRequestStatus diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 49e025502..792ab2399 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -138,6 +138,9 @@ export const createArticleResolver = authorized< state, labels: inputLabels, folder, + rssFeedUrl, + savedAt, + publishedAt, }, }, { log, uid, pubsub } @@ -215,9 +218,11 @@ export const createArticleResolver = authorized< readingProgressAnchorIndex: 0, readingProgressPercent: 0, highlights: [], - savedAt: new Date(), + savedAt: savedAt || new Date(), updatedAt: new Date(), folder: '', + publishedAt, + subscription: rssFeedUrl, }, } @@ -252,6 +257,9 @@ export const createArticleResolver = authorized< state: state || undefined, labels: inputLabels || undefined, folder: folder || undefined, + savedAt, + publishedAt, + subscription: rssFeedUrl || undefined, }) return DUMMY_RESPONSE } else if (!skipParsing && preparedDocument?.document) { @@ -274,6 +282,9 @@ export const createArticleResolver = authorized< state: state || undefined, labels: inputLabels || undefined, folder: folder || undefined, + savedAt, + publishedAt, + subscription: rssFeedUrl || undefined, }) return DUMMY_RESPONSE } @@ -294,6 +305,9 @@ export const createArticleResolver = authorized< uploadFileId, state, folder, + publishedAt, + rssFeedUrl, + savedAt, }) log.info('New article saving', { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4078c8652..aba01f9e2 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -497,6 +497,9 @@ const schema = gql` state: ArticleSavingRequestStatus labels: [CreateLabelInput!] folder: String + rssFeedUrl: String + savedAt: Date + publishedAt: Date } enum CreateArticleErrorCode { UNABLE_TO_FETCH diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 15d0e60f1..4e1fd6b07 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -36,6 +36,7 @@ interface PageSaveRequest { savedAt?: Date publishedAt?: Date folder?: string + subscription?: string } const SAVING_CONTENT = 'Your link is being saved...' @@ -91,6 +92,7 @@ export const createPageSaveRequest = async ({ savedAt, publishedAt, folder, + subscription, }: PageSaveRequest): Promise => { try { validateUrl(url) @@ -128,6 +130,8 @@ export const createPageSaveRequest = async ({ state: LibraryItemState.Processing, publishedAt, folder, + subscription, + savedAt, }, userId, pubsub @@ -161,6 +165,7 @@ export const createPageSaveRequest = async ({ savedAt, publishedAt, folder, + rssFeedUrl: subscription, }) return libraryItemToArticleSavingRequest(user, libraryItem) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 7b6c65f74..af602eca8 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -93,7 +93,7 @@ export const savePage = async ( itemType: parseResult.pageType, originalHtml: parseResult.domContent, canonicalUrl: parseResult.canonicalUrl, - saveTime: input.savedAt ? new Date(input.savedAt) : new Date(), + savedAt: input.savedAt ? new Date(input.savedAt) : new Date(), publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, state: input.state || undefined, rssFeedUrl: input.rssFeedUrl, @@ -214,7 +214,7 @@ export const parsedContentToLibraryItem = ({ itemType, uploadFileHash, uploadFileId, - saveTime, + savedAt, publishedAt, state, rssFeedUrl, @@ -233,7 +233,7 @@ export const parsedContentToLibraryItem = ({ canonicalUrl?: string | null uploadFileHash?: string | null uploadFileId?: string | null - saveTime?: Date + savedAt?: Date publishedAt?: Date | null state?: ArticleSavingRequestStatus | null rssFeedUrl?: string | null @@ -268,8 +268,7 @@ export const parsedContentToLibraryItem = ({ state: state ? (state as unknown as LibraryItemState) : LibraryItemState.Succeeded, - createdAt: validatedDate(saveTime), - savedAt: validatedDate(saveTime), + savedAt: validatedDate(savedAt), siteName: parsedContent?.siteName, itemLanguage: parsedContent?.language, siteIcon: parsedContent?.siteIcon, diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index a14af990a..cad757348 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -237,6 +237,7 @@ export const enqueueParseRequest = async ({ savedAt, publishedAt, folder, + rssFeedUrl, }: { url: string userId: string @@ -250,6 +251,7 @@ export const enqueueParseRequest = async ({ savedAt?: Date publishedAt?: Date folder?: string + rssFeedUrl?: string }): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { @@ -263,6 +265,7 @@ export const enqueueParseRequest = async ({ savedAt, publishedAt, folder, + rssFeedUrl, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 9288b370c..39eeb617d 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -415,11 +415,14 @@ async function fetchContent(req, res) { const uploadedPdf = await sendCreateArticleMutation(userId, { url: encodeURI(finalUrl), articleSavingRequestId, - uploadFileId: uploadedFileId, + uploadFileId, state, labels, source, folder, + rssFeedUrl, + savedAt, + publishedAt, }); if (!uploadedPdf) { statusCode = 500; From ad63c75e63d5b52fbf76b538ab6992f425e047e3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 28 Nov 2023 17:57:26 +0800 Subject: [PATCH 2/3] fix typo --- packages/puppeteer-parse/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 39eeb617d..76ccbc4d6 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -411,7 +411,7 @@ async function fetchContent(req, res) { } if (contentType === 'application/pdf') { - const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); + const uploadFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); const uploadedPdf = await sendCreateArticleMutation(userId, { url: encodeURI(finalUrl), articleSavingRequestId, From 36777568ac9508cfa9e9242aa3255b3ae483c141 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 1 Dec 2023 15:00:36 +0800 Subject: [PATCH 3/3] create/add RSS labels to the library item when feedUrl is there --- packages/api/src/resolvers/article/index.ts | 14 +++++----- packages/api/src/routers/svc/following.ts | 20 ++++---------- packages/api/src/services/labels.ts | 29 ++++++++++++++++++++- packages/api/src/services/save_email.ts | 14 ++++------ packages/api/src/services/save_file.ts | 11 ++++---- packages/api/src/services/save_page.ts | 13 ++++----- 6 files changed, 59 insertions(+), 42 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 792ab2399..08f156690 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -61,9 +61,9 @@ import { createPageSaveRequest } from '../../services/create_page_save_request' import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, + createAndSaveLabelsInLibraryItem, findLabelsByIds, findOrCreateLabels, - saveLabelsInLibraryItem, } from '../../services/labels' import { createLibraryItem, @@ -355,11 +355,13 @@ export const createArticleResolver = authorized< ) } - // save labels in item - if (inputLabels) { - const labels = await findOrCreateLabels(inputLabels, user.id) - await saveLabelsInLibraryItem(labels, libraryItemToReturn.id, user.id) - } + await createAndSaveLabelsInLibraryItem( + libraryItemToReturn.id, + uid, + inputLabels, + rssFeedUrl, + pubsub + ) log.info( 'item created in database', diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index ef962f7b4..73d8558ac 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -1,9 +1,6 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ import express from 'express' -import { - findOrCreateLabels, - saveLabelsInLibraryItem, -} from '../../services/labels' +import { createAndSaveLabelsInLibraryItem } from '../../services/labels' import { saveFeedItemInFollowing } from '../../services/library_item' import { logger } from '../../utils/logger' @@ -69,19 +66,12 @@ export function followingServiceRouter() { logger.info('feed item saved in following') - // add RSS label to the item - const labels = await findOrCreateLabels( - [ - { - name: 'RSS', - }, - ], - userId - ) - await saveLabelsInLibraryItem( - labels, + // save RSS label in the item + await createAndSaveLabelsInLibraryItem( result.identifiers[0].id, userId, + [{ name: 'RSS' }], + undefined, undefined, true ) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index a015098be..cb4c18310 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -3,7 +3,7 @@ import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity import { EntityLabel } from '../entity/entity_label' import { Label } from '../entity/label' import { LibraryItem } from '../entity/library_item' -import { createPubSubClient, EntityType } from '../pubsub' +import { createPubSubClient, EntityType, PubsubClient } from '../pubsub' import { authTrx } from '../repository' import { CreateLabelInput, labelRepository } from '../repository/label' import { libraryItemRepository } from '../repository/library_item' @@ -65,6 +65,33 @@ export const findOrCreateLabels = async ( ) } +export const createAndSaveLabelsInLibraryItem = async ( + libraryItemId: string, + userId: string, + labels?: CreateLabelInput[] | null, + rssFeedUrl?: string | null, + pubsub?: PubsubClient, + skipPubSub?: boolean +) => { + if (rssFeedUrl) { + // add rss label to labels + labels = (labels || []).concat({ name: 'RSS' }) + } + + // save labels in item + if (labels && labels.length > 0) { + const newLabels = await findOrCreateLabels(labels, userId) + + await saveLabelsInLibraryItem( + newLabels, + libraryItemId, + userId, + pubsub, + skipPubSub + ) + } +} + export const saveLabelsInLibraryItem = async ( labels: Label[], libraryItemId: string, diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 17ce08f27..e5a79d265 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -1,5 +1,4 @@ import { LibraryItem, LibraryItemState } from '../entity/library_item' -import { getInternalLabelWithColor } from '../repository/label' import { enqueueThumbnailTask } from '../utils/createTask' import { cleanUrl, @@ -16,7 +15,7 @@ import { parsePreparedContent, parseUrlMetadata, } from '../utils/parser' -import { findOrCreateLabels, saveLabelsInLibraryItem } from './labels' +import { createAndSaveLabelsInLibraryItem } from './labels' import { createLibraryItem, findLibraryItemByUrl, @@ -81,8 +80,6 @@ export const saveEmail = async ( return updatedLibraryItem } - const newsletterLabel = getInternalLabelWithColor('newsletter') - // start a transaction to create the library item and update the received email const newLibraryItem = await createLibraryItem( { @@ -123,11 +120,10 @@ export const saveEmail = async ( }) } - if (newsletterLabel) { - // add newsletter label - const labels = await findOrCreateLabels([newsletterLabel], input.userId) - await saveLabelsInLibraryItem(labels, newLibraryItem.id, input.userId) - } + // save newsletter label in the item + await createAndSaveLabelsInLibraryItem(newLibraryItem.id, input.userId, [ + { name: 'Newsletter' }, + ]) await updateReceivedEmail(input.receivedEmailId, 'article', input.userId) diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index 165fc1244..e6c58148b 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -3,7 +3,7 @@ import { User } from '../entity/user' import { homePageURL } from '../env' import { SaveErrorCode, SaveFileInput, SaveResult } from '../generated/graphql' import { getStorageFileDetails } from '../utils/uploads' -import { findOrCreateLabels, saveLabelsInLibraryItem } from './labels' +import { createAndSaveLabelsInLibraryItem } from './labels' import { updateLibraryItem } from './library_item' import { findUploadFileById, setFileUploadComplete } from './upload_file' @@ -40,10 +40,11 @@ export const saveFile = async ( } // add labels to item - if (input.labels) { - const labels = await findOrCreateLabels(input.labels, user.id) - await saveLabelsInLibraryItem(labels, input.clientRequestId, user.id) - } + await createAndSaveLabelsInLibraryItem( + input.clientRequestId, + user.id, + input.labels + ) return { clientRequestId: input.clientRequestId, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index af602eca8..442dcea93 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -28,7 +28,7 @@ import { parsePreparedContent } from '../utils/parser' import { contentReaderForLibraryItem } from '../utils/uploads' import { createPageSaveRequest } from './create_page_save_request' import { createHighlight } from './highlights' -import { findOrCreateLabels, saveLabelsInLibraryItem } from './labels' +import { createAndSaveLabelsInLibraryItem } from './labels' import { createLibraryItem, updateLibraryItem } from './library_item' // where we can use APIs to fetch their underlying content. @@ -160,11 +160,12 @@ export const savePage = async ( clientRequestId = newItem.id } - // save labels in item - if (input.labels) { - const labels = await findOrCreateLabels(input.labels, user.id) - await saveLabelsInLibraryItem(labels, clientRequestId, user.id) - } + await createAndSaveLabelsInLibraryItem( + clientRequestId, + user.id, + input.labels, + input.rssFeedUrl + ) } // we don't want to create thumbnail for imported pages