diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 17348e464..0545b1093 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -422,7 +422,7 @@ export const createPage = async ( body: { ...page, updatedAt: new Date(), - savedAt: new Date(), + savedAt: page.savedAt || new Date(), wordsCount: page.wordsCount ?? wordsCount(page.content), }, refresh: 'wait_for', // wait for the index to be refreshed before returning diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 99fa8611d..7721c6038 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2285,6 +2285,8 @@ export type SaveUrlInput = { clientRequestId: Scalars['ID']; labels?: InputMaybe>; locale?: InputMaybe; + publishedAt?: InputMaybe; + savedAt?: InputMaybe; source: Scalars['String']; state?: InputMaybe; timezone?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index f06cea0fa..9e2fdcdf6 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1664,6 +1664,8 @@ input SaveUrlInput { clientRequestId: ID! labels: [CreateLabelInput!] locale: String + publishedAt: Date + savedAt: Date source: String! state: ArticleSavingRequestStatus timezone: String diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 15b5d31c1..150eda9a8 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -574,6 +574,8 @@ const schema = gql` labels: [CreateLabelInput!] locale: String timezone: String + savedAt: Date + publishedAt: Date } union SaveResult = SaveSuccess | SaveError diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 1a370f508..f51510148 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -33,6 +33,8 @@ interface PageSaveRequest { user?: User | null locale?: string timezone?: string + savedAt?: Date + publishedAt?: Date } const SAVING_CONTENT = 'Your link is being saved...' @@ -86,6 +88,8 @@ export const createPageSaveRequest = async ({ user, locale, timezone, + savedAt, + publishedAt, }: PageSaveRequest): Promise => { try { validateUrl(url) @@ -138,7 +142,8 @@ export const createPageSaveRequest = async ({ url, state: ArticleSavingRequestStatus.Processing, createdAt: new Date(), - savedAt: new Date(), + savedAt: savedAt || new Date(), + publishedAt, archivedAt, labels, } @@ -177,6 +182,9 @@ export const createPageSaveRequest = async ({ labels: labelsInput, locale, timezone, + // unix timestamp + savedAt: savedAt?.getTime(), + publishedAt: publishedAt?.getTime(), }) return pageToArticleSavingRequest(user, page) diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 2f3f7ac54..f42f34115 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -28,8 +28,8 @@ export const saveUrl = async ( : undefined const pageSaveRequest = await createPageSaveRequest({ + ...input, userId: ctx.uid, - url: input.url, pubsub: ctx.pubsub, articleSavingRequestId: input.clientRequestId, archivedAt, @@ -37,6 +37,8 @@ export const saveUrl = async ( user, locale: input.locale || undefined, timezone: input.timezone || undefined, + savedAt: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, }) return { diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index c18239529..273eaee94 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -229,6 +229,8 @@ export const enqueueParseRequest = async ({ labels, locale, timezone, + savedAt, + publishedAt, }: { url: string userId: string @@ -239,6 +241,8 @@ export const enqueueParseRequest = async ({ labels?: CreateLabelInput[] locale?: string timezone?: string + savedAt?: number // unix timestamp + publishedAt?: number // unix timestamp }): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { @@ -249,6 +253,8 @@ export const enqueueParseRequest = async ({ labels, locale, timezone, + savedAt, + publishedAt, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index f7259a3f2..11726b9ae 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -142,13 +142,14 @@ const getBrowserPromise = (async () => { const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { try { - const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); + const stream = await axios.get(contentObjUrl, { responseType: 'stream', timeout: REQUEST_TIMEOUT }); return axios.put(uploadSignedUrl, stream.data, { headers: { 'Content-Type': contentType, }, maxBodyLength: 1000000000, maxContentLength: 100000000, + timeout: REQUEST_TIMEOUT, }); } catch (error) { console.error('error uploading to signed url', error.message);