add savedAt and publishedAt to saveUrl api

packages/api/src/elastic/pages.ts

@@ -422,7 +422,7 @@ export const createPage = async (
       body: {
         ...page,
         updatedAt: new Date(),
-        savedAt: new Date(),
+        savedAt: page.savedAt || new Date(),
         wordsCount: page.wordsCount ?? wordsCount(page.content),
       },
       refresh: 'wait_for', // wait for the index to be refreshed before returning

packages/api/src/generated/graphql.ts

@@ -2285,6 +2285,8 @@ export type SaveUrlInput = {
   clientRequestId: Scalars['ID'];
   labels?: InputMaybe<Array<CreateLabelInput>>;
   locale?: InputMaybe<Scalars['String']>;
+  publishedAt?: InputMaybe<Scalars['Date']>;
+  savedAt?: InputMaybe<Scalars['Date']>;
   source: Scalars['String'];
   state?: InputMaybe<ArticleSavingRequestStatus>;
   timezone?: InputMaybe<Scalars['String']>;

packages/api/src/generated/schema.graphql

@@ -1664,6 +1664,8 @@ input SaveUrlInput {
   clientRequestId: ID!
   labels: [CreateLabelInput!]
   locale: String
+  publishedAt: Date
+  savedAt: Date
   source: String!
   state: ArticleSavingRequestStatus
   timezone: String

packages/api/src/schema.ts

@@ -574,6 +574,8 @@ const schema = gql`
     labels: [CreateLabelInput!]
     locale: String
     timezone: String
+    savedAt: Date
+    publishedAt: Date
   }
 
   union SaveResult = SaveSuccess | SaveError

packages/api/src/services/create_page_save_request.ts

@@ -33,6 +33,8 @@ interface PageSaveRequest {
   user?: User | null
   locale?: string
   timezone?: string
+  savedAt?: Date
+  publishedAt?: Date
 }
 
 const SAVING_CONTENT = 'Your link is being saved...'
@@ -86,6 +88,8 @@ export const createPageSaveRequest = async ({
   user,
   locale,
   timezone,
+  savedAt,
+  publishedAt,
 }: PageSaveRequest): Promise<ArticleSavingRequest> => {
   try {
     validateUrl(url)
@@ -138,7 +142,8 @@ export const createPageSaveRequest = async ({
       url,
       state: ArticleSavingRequestStatus.Processing,
       createdAt: new Date(),
-      savedAt: new Date(),
+      savedAt: savedAt || new Date(),
+      publishedAt,
       archivedAt,
       labels,
     }
@@ -177,6 +182,9 @@ export const createPageSaveRequest = async ({
     labels: labelsInput,
     locale,
     timezone,
+    // unix timestamp
+    savedAt: savedAt?.getTime(),
+    publishedAt: publishedAt?.getTime(),
   })
 
   return pageToArticleSavingRequest(user, page)

packages/api/src/services/save_url.ts

@@ -28,8 +28,8 @@ export const saveUrl = async (
       : undefined
 
     const pageSaveRequest = await createPageSaveRequest({
+      ...input,
       userId: ctx.uid,
-      url: input.url,
       pubsub: ctx.pubsub,
       articleSavingRequestId: input.clientRequestId,
       archivedAt,
@@ -37,6 +37,8 @@ export const saveUrl = async (
       user,
       locale: input.locale || undefined,
       timezone: input.timezone || undefined,
+      savedAt: input.savedAt ? new Date(input.savedAt) : undefined,
+      publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
     })
 
     return {

packages/api/src/utils/createTask.ts

@@ -229,6 +229,8 @@ export const enqueueParseRequest = async ({
   labels,
   locale,
   timezone,
+  savedAt,
+  publishedAt,
 }: {
   url: string
   userId: string
@@ -239,6 +241,8 @@ export const enqueueParseRequest = async ({
   labels?: CreateLabelInput[]
   locale?: string
   timezone?: string
+  savedAt?: number // unix timestamp
+  publishedAt?: number // unix timestamp
 }): Promise<string> => {
   const { GOOGLE_CLOUD_PROJECT } = process.env
   const payload = {
@@ -249,6 +253,8 @@ export const enqueueParseRequest = async ({
     labels,
     locale,
     timezone,
+    savedAt,
+    publishedAt,
   }
 
   // If there is no Google Cloud Project Id exposed, it means that we are in local environment

packages/puppeteer-parse/index.js

@@ -142,13 +142,14 @@ const getBrowserPromise = (async () => {
 
 const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => {
   try {
-    const stream = await axios.get(contentObjUrl, { responseType: 'stream' });
+    const stream = await axios.get(contentObjUrl, { responseType: 'stream', timeout: REQUEST_TIMEOUT });
     return axios.put(uploadSignedUrl, stream.data, {
       headers: {
         'Content-Type': contentType,
       },
       maxBodyLength: 1000000000,
       maxContentLength: 100000000,
+      timeout: REQUEST_TIMEOUT,
     });
   } catch (error) {
     console.error('error uploading to signed url', error.message);