diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index d48cd9f1c..9eccf5efb 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -70,7 +70,7 @@ interface SavePageResponse { interface FetchResult { finalUrl: string - title: string + title?: string content?: string contentType?: string readabilityResult?: Readability.ParseResult diff --git a/packages/content-fetch/src/job.ts b/packages/content-fetch/src/job.ts index 3d3946b18..ba70478af 100644 --- a/packages/content-fetch/src/job.ts +++ b/packages/content-fetch/src/job.ts @@ -29,7 +29,7 @@ const getPriority = (job: savePageJob): number => { return 100 } - return 5 + return 1 } const getAttempts = (job: savePageJob): number => { @@ -57,7 +57,7 @@ export const queueSavePageJob = async (savePageJobs: savePageJob[]) => { data: job.data, opts: getOpts(job), })) - console.log('queue save page jobs:', { jobs }) + console.log('queue save page jobs:', JSON.stringify(jobs, null, 2)) return queue.addBulk(jobs) } diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 9672344d3..0106d8058 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -47,7 +47,7 @@ interface LogRecord { interface FetchResult { finalUrl: string - title: string + title?: string content?: string contentType?: string readabilityResult?: unknown diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index bfe7fde72..2629689c2 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -154,7 +154,7 @@ export const fetchContent = async ( let context: BrowserContext | undefined, page: Page | undefined, - title = '', + title: string | undefined, content: string | undefined, contentType: string | undefined, readabilityResult: Readability.ParseResult | null | undefined @@ -214,7 +214,7 @@ export const fetchContent = async ( const sbResult = await fetchContentWithScrapingBee(url) title = sbResult.title content = sbResult.domContent - } else if (result.title && result.domContent) { + } else { title = result.title content = result.domContent }