From f0865654dc738e01e0060def267dc0e6b2a17255 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 22 Aug 2024 15:14:41 +0800
Subject: [PATCH 1/3] fix: allow content-fetch to still process http requests

---
 packages/content-fetch/src/app.ts | 42 ++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/packages/content-fetch/src/app.ts b/packages/content-fetch/src/app.ts
index 8ffb1ef6b..3354667a9 100644
--- a/packages/content-fetch/src/app.ts
+++ b/packages/content-fetch/src/app.ts
@@ -2,13 +2,21 @@ import { RedisDataSource } from '@omnivore/utils'
 import { JobType } from 'bullmq'
 import express, { Express } from 'express'
 import asyncHandler from 'express-async-handler'
+import { JobData, processFetchContentJob } from './request_handler'
 import { createWorker, getQueue, QUEUE } from './worker'
 
 const main = () => {
   console.log('Starting worker...')
 
+  if (!process.env.VERIFICATION_TOKEN) {
+    console.error('VERIFICATION_TOKEN is required')
+    process.exit(1)
+  }
+
   const app: Express = express()
-  const port = process.env.PORT || 3002
+
+  app.use(express.json())
+  app.use(express.urlencoded({ extended: true }))
 
   // create redis source
   const redisDataSource = new RedisDataSource({
@@ -75,6 +83,38 @@ const main = () => {
     })
   )
 
+  app.all(
+    '/',
+    asyncHandler(async (req, res) => {
+      console.log('Received http request')
+
+      if (req.method !== 'GET' && req.method !== 'POST') {
+        console.error('request method is not GET or POST')
+        res.sendStatus(405)
+        return
+      }
+
+      if (req.query.token !== process.env.VERIFICATION_TOKEN) {
+        console.error('query does not include valid token')
+        res.sendStatus(403)
+        return
+      }
+
+      try {
+        const data = <JobData>req.body
+        const attempt = parseInt(req.get('X-CloudTasks-TaskRetryCount') || '0')
+        await processFetchContentJob(redisDataSource, data, attempt)
+      } catch (error) {
+        console.error('Error fetching content', { error })
+        res.sendStatus(500)
+        return
+      }
+
+      res.sendStatus(200)
+    })
+  )
+
+  const port = process.env.PORT || 3002
   const server = app.listen(port, () => {
     console.log('Worker started')
   })

From eec2734f5aff026d0ddb3816526aa8581697fd50 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 22 Aug 2024 15:27:53 +0800
Subject: [PATCH 2/3] keep up to 1 hour complete tasks and up to 1 day failed
 tasks

---
 packages/content-fetch/src/worker.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/content-fetch/src/worker.ts b/packages/content-fetch/src/worker.ts
index 0d28d4f5a..618448b95 100644
--- a/packages/content-fetch/src/worker.ts
+++ b/packages/content-fetch/src/worker.ts
@@ -16,10 +16,10 @@ export const getQueue = async (
         delay: 2000, // 2 seconds
       },
       removeOnComplete: {
-        age: 24 * 3600, // keep up to 24 hours
+        age: 3600, // keep up to 1 hour
       },
       removeOnFail: {
-        age: 7 * 24 * 3600, // keep up to 7 days
+        age: 24 * 3600, // keep up to 1 day
       },
     },
   })

From 47cea15480ee0089d5bfdf29e48b24555227a9bd Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 22 Aug 2024 15:27:59 +0800
Subject: [PATCH 3/3] keep up to 1 hour complete tasks and up to 1 day failed
 tasks

---
 packages/api/src/queue-processor.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index aa5f2e7ed..c0030300f 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -111,10 +111,10 @@ export const getQueue = async (
         delay: 2000, // 2 seconds
       },
       removeOnComplete: {
-        age: 24 * 3600, // keep up to 24 hours
+        age: 3600, // keep up to 1 hour
       },
       removeOnFail: {
-        age: 7 * 24 * 3600, // keep up to 7 days
+        age: 24 * 3600, // keep up to 1 day
       },
     },
   })