create batch save page jobs after fetching content

2024-01-18 15:45:14 +08:00
parent 8a3b50f07f
commit 0015a946b2
5 changed files with 202 additions and 74 deletions
--- a/packages/content-fetch/package.json
+++ b/packages/content-fetch/package.json
@ -8,8 +8,10 @@
  ],
  "dependencies": {
    "axios": "^0.27.2",
+    "bullmq": "^5.1.1",
    "dotenv": "^8.2.0",
    "express": "^4.17.1",
+    "ioredis": "^5.3.2",
    "jsonwebtoken": "^8.5.1",
    "@google-cloud/functions-framework": "^3.0.0",
    "@omnivore/puppeteer-parse": "^1.0.0",