From f0865654dc738e01e0060def267dc0e6b2a17255 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 22 Aug 2024 15:14:41 +0800
Subject: [PATCH] fix: allow content-fetch to still process http requests

---
 packages/content-fetch/src/app.ts | 42 ++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/packages/content-fetch/src/app.ts b/packages/content-fetch/src/app.ts
index 8ffb1ef6b..3354667a9 100644
--- a/packages/content-fetch/src/app.ts
+++ b/packages/content-fetch/src/app.ts
@@ -2,13 +2,21 @@ import { RedisDataSource } from '@omnivore/utils'
 import { JobType } from 'bullmq'
 import express, { Express } from 'express'
 import asyncHandler from 'express-async-handler'
+import { JobData, processFetchContentJob } from './request_handler'
 import { createWorker, getQueue, QUEUE } from './worker'
 
 const main = () => {
   console.log('Starting worker...')
 
+  if (!process.env.VERIFICATION_TOKEN) {
+    console.error('VERIFICATION_TOKEN is required')
+    process.exit(1)
+  }
+
   const app: Express = express()
-  const port = process.env.PORT || 3002
+
+  app.use(express.json())
+  app.use(express.urlencoded({ extended: true }))
 
   // create redis source
   const redisDataSource = new RedisDataSource({
@@ -75,6 +83,38 @@ const main = () => {
     })
   )
 
+  app.all(
+    '/',
+    asyncHandler(async (req, res) => {
+      console.log('Received http request')
+
+      if (req.method !== 'GET' && req.method !== 'POST') {
+        console.error('request method is not GET or POST')
+        res.sendStatus(405)
+        return
+      }
+
+      if (req.query.token !== process.env.VERIFICATION_TOKEN) {
+        console.error('query does not include valid token')
+        res.sendStatus(403)
+        return
+      }
+
+      try {
+        const data = <JobData>req.body
+        const attempt = parseInt(req.get('X-CloudTasks-TaskRetryCount') || '0')
+        await processFetchContentJob(redisDataSource, data, attempt)
+      } catch (error) {
+        console.error('Error fetching content', { error })
+        res.sendStatus(500)
+        return
+      }
+
+      res.sendStatus(200)
+    })
+  )
+
+  const port = process.env.PORT || 3002
   const server = app.listen(port, () => {
     console.log('Worker started')
   })