Merge pull request #3417 from omnivore-app/fix/puppeteer-parse

fix/puppeteer parse

packages/api/src/jobs/save_page.ts

@@ -70,7 +70,7 @@ interface SavePageResponse {
 
 interface FetchResult {
   finalUrl: string
-  title: string
+  title?: string
   content?: string
   contentType?: string
   readabilityResult?: Readability.ParseResult

packages/content-fetch/src/job.ts

@@ -29,7 +29,7 @@ const getPriority = (job: savePageJob): number => {
     return 100
   }
 
-  return 5
+  return 1
 }
 
 const getAttempts = (job: savePageJob): number => {
@@ -57,7 +57,7 @@ export const queueSavePageJob = async (savePageJobs: savePageJob[]) => {
     data: job.data,
     opts: getOpts(job),
   }))
-  console.log('queue save page jobs:', { jobs })
+  console.log('queue save page jobs:', JSON.stringify(jobs, null, 2))
 
   return queue.addBulk(jobs)
 }

packages/content-fetch/src/request_handler.ts

@@ -47,7 +47,7 @@ interface LogRecord {
 
 interface FetchResult {
   finalUrl: string
-  title: string
+  title?: string
   content?: string
   contentType?: string
   readabilityResult?: unknown

packages/puppeteer-parse/src/index.ts

@@ -154,7 +154,7 @@ export const fetchContent = async (
 
   let context: BrowserContext | undefined,
     page: Page | undefined,
-    title = '',
+    title: string | undefined,
     content: string | undefined,
     contentType: string | undefined,
     readabilityResult: Readability.ParseResult | null | undefined
@@ -214,7 +214,7 @@ export const fetchContent = async (
           const sbResult = await fetchContentWithScrapingBee(url)
           title = sbResult.title
           content = sbResult.domContent
-        } else if (result.title && result.domContent) {
+        } else {
           title = result.title
           content = result.domContent
         }