From 153662ce6a8535003cafceb9212a79bd8f5dc5f2 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 14 Jul 2023 12:21:34 +0800
Subject: [PATCH] set savedAt and publishedAt timestamp for rss item

---
 packages/api/src/generated/graphql.ts     |  2 ++
 packages/api/src/generated/schema.graphql |  2 ++
 packages/api/src/schema.ts                |  2 ++
 packages/api/src/services/save_page.ts    | 12 +++++++++---
 packages/puppeteer-parse/index.js         |  8 +++++++-
 packages/rss-handler/src/index.ts         |  2 ++
 6 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 0124ea373..ad9738e17 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -2257,7 +2257,9 @@ export type SavePageInput = {
   labels?: InputMaybe<Array<CreateLabelInput>>;
   originalContent: Scalars['String'];
   parseResult?: InputMaybe<ParseResult>;
+  publishedAt?: InputMaybe<Scalars['Date']>;
   rssFeedUrl?: InputMaybe<Scalars['String']>;
+  savedAt?: InputMaybe<Scalars['Date']>;
   source: Scalars['String'];
   state?: InputMaybe<ArticleSavingRequestStatus>;
   title?: InputMaybe<Scalars['String']>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 6fe0e53b5..d069781f2 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1637,7 +1637,9 @@ input SavePageInput {
   labels: [CreateLabelInput!]
   originalContent: String!
   parseResult: ParseResult
+  publishedAt: Date
   rssFeedUrl: String
+  savedAt: Date
   source: String!
   state: ArticleSavingRequestStatus
   title: String
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index bde299fd6..907c1558e 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -562,6 +562,8 @@ const schema = gql`
     state: ArticleSavingRequestStatus
     labels: [CreateLabelInput!]
     rssFeedUrl: String
+    savedAt: Date
+    publishedAt: Date
   }
 
   input SaveUrlInput {
diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts
index 158e9eb10..cbc025ba1 100644
--- a/packages/api/src/services/save_page.ts
+++ b/packages/api/src/services/save_page.ts
@@ -104,6 +104,8 @@ export const savePage = async (
     originalHtml: parseResult.domContent,
     canonicalUrl: parseResult.canonicalUrl,
     rssFeedUrl: input.rssFeedUrl,
+    saveTime: input.savedAt ? new Date(input.savedAt) : undefined,
+    publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
   })
 
   // save state
@@ -223,6 +225,7 @@ export const parsedContentToPage = ({
   uploadFileId,
   saveTime,
   rssFeedUrl,
+  publishedAt,
 }: {
   url: string
   userId: string
@@ -239,6 +242,7 @@ export const parsedContentToPage = ({
   uploadFileId?: string | null
   saveTime?: Date
   rssFeedUrl?: string | null
+  publishedAt?: Date | null
 }): Page => {
   return {
     id: pageId || '',
@@ -259,13 +263,15 @@ export const parsedContentToPage = ({
     pageType,
     hash: uploadFileHash || stringToHash(parsedContent?.content || url),
     image: parsedContent?.previewImage ?? undefined,
-    publishedAt: validatedDate(parsedContent?.publishedDate ?? undefined),
+    publishedAt: validatedDate(
+      publishedAt || parsedContent?.publishedDate || undefined
+    ),
     uploadFileId,
     readingProgressPercent: 0,
     readingProgressAnchorIndex: 0,
     state: ArticleSavingRequestStatus.Succeeded,
-    createdAt: saveTime || new Date(),
-    savedAt: saveTime || new Date(),
+    createdAt: validatedDate(saveTime) || new Date(),
+    savedAt: validatedDate(saveTime) || new Date(),
     siteName: parsedContent?.siteName ?? undefined,
     language: parsedContent?.language ?? undefined,
     siteIcon: parsedContent?.siteIcon ?? undefined,
diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js
index 355d3f907..6cac26b65 100644
--- a/packages/puppeteer-parse/index.js
+++ b/packages/puppeteer-parse/index.js
@@ -335,7 +335,9 @@ async function fetchContent(req, res) {
   const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
   const locale = (req.query ? req.query.locale : undefined) || (req.body ? req.body.locale : undefined);
   const timezone = (req.query ? req.query.timezone : undefined) || (req.body ? req.body.timezone : undefined);
-  const rssFeedUrl = (req.query ? req.query.rssFeedUrl : undefined) || (req.body ? req.body.rssFeedUrl : undefined);
+  const rssFeedUrl = req.body.rssFeedUrl;
+  const savedAt = req.body.savedAt;
+  const publishedAt = req.body.publishedAt;
 
   let logRecord = {
     url: urlStr,
@@ -350,6 +352,8 @@ async function fetchContent(req, res) {
     locale,
     timezone,
     rssFeedUrl,
+    savedAt,
+    publishedAt,
   };
 
   console.info(`Article parsing request`, logRecord);
@@ -456,6 +460,8 @@ async function fetchContent(req, res) {
         state,
         labels,
         rssFeedUrl,
+        savedAt,
+        publishedAt,
       });
       if (!apiResponse) {
         logRecord.error = 'error while saving page';
diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts
index 1ee659a8a..07ed4b49c 100644
--- a/packages/rss-handler/src/index.ts
+++ b/packages/rss-handler/src/index.ts
@@ -139,6 +139,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
           saveRequestId: '',
           labels: [{ name: 'RSS', color: '#f26522' }],
           rssFeedUrl: feedUrl,
+          savedAt: publishedAt,
+          publishedAt,
         }
 
         try {