From 53f63b90d7616816b873d3befb34663f982c3840 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 5 Feb 2024 16:26:05 +0800
Subject: [PATCH 1/2] increase the seed valud of backoff delay to 2 seconds

---
 packages/api/src/jobs/rss/refreshAllFeeds.ts |  2 --
 packages/api/src/queue-processor.ts          |  8 ++++++++
 packages/api/src/utils/createTask.ts         | 14 --------------
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts
index 5a50e41ce..ef49bacac 100644
--- a/packages/api/src/jobs/rss/refreshAllFeeds.ts
+++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts
@@ -137,8 +137,6 @@ export const queueRSSRefreshFeedJob = async (
   }
   return queue.add('refresh-feed', payload, {
     jobId: jobid,
-    removeOnComplete: true,
-    removeOnFail: true,
     priority: options.priority == 'low' ? 10 : 50,
   })
 }
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index b93b327d9..cde76f885 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -49,6 +49,14 @@ export const getBackendQueue = async (): Promise<Queue | undefined> => {
   }
   backendQueue = new Queue(QUEUE_NAME, {
     connection: redisDataSource.workerRedisClient,
+    defaultJobOptions: {
+      backoff: {
+        type: 'exponential',
+        delay: 2000, // 2 seconds
+      },
+      removeOnComplete: true,
+      removeOnFail: true,
+    },
   })
   await backendQueue.waitUntilReady()
   return backendQueue
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 3a1145aec..cfb266724 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -666,8 +666,6 @@ export const enqueueTriggerRuleJob = async (data: TriggerRuleJobData) => {
   return queue.add(TRIGGER_RULE_JOB_NAME, data, {
     priority: 5,
     attempts: 1,
-    removeOnComplete: true,
-    removeOnFail: true,
   })
 }
 
@@ -680,8 +678,6 @@ export const enqueueWebhookJob = async (data: CallWebhookJobData) => {
   return queue.add(CALL_WEBHOOK_JOB_NAME, data, {
     priority: 5,
     attempts: 1,
-    removeOnComplete: true,
-    removeOnFail: true,
   })
 }
 
@@ -697,10 +693,6 @@ export const bulkEnqueueUpdateLabels = async (data: UpdateLabelsData[]) => {
     opts: {
       attempts: 3,
       priority: 1,
-      backoff: {
-        type: 'exponential',
-        delay: 1000,
-      },
     },
   }))
 
@@ -722,10 +714,6 @@ export const enqueueUpdateHighlight = async (data: UpdateHighlightData) => {
     return queue.add(UPDATE_HIGHLIGHT_JOB, data, {
       attempts: 3,
       priority: 1,
-      backoff: {
-        type: 'exponential',
-        delay: 1000,
-      },
     })
   } catch (error) {
     logger.error('error enqueuing update highlight job', error)
@@ -745,8 +733,6 @@ export const enqueueBulkAction = async (data: BulkActionData) => {
       attempts: 1,
       priority: 10,
       jobId, // deduplication
-      removeOnComplete: true,
-      removeOnFail: true,
     })
   } catch (error) {
     logger.error('error enqueuing bulk action job', error)

From 95fabe1dbfae7ac54d9254add66f7e9bb045fe22 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 6 Feb 2024 14:21:00 +0800
Subject: [PATCH 2/2] keep completed jobs for up to 24 hours and failed jobs
 for 7 days

---
 packages/api/src/queue-processor.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index cde76f885..8eb4d40de 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -54,8 +54,12 @@ export const getBackendQueue = async (): Promise<Queue | undefined> => {
         type: 'exponential',
         delay: 2000, // 2 seconds
       },
-      removeOnComplete: true,
-      removeOnFail: true,
+      removeOnComplete: {
+        age: 24 * 3600, // keep up to 24 hours
+      },
+      removeOnFail: {
+        age: 7 * 24 * 3600, // keep up to 7 days
+      },
     },
   })
   await backendQueue.waitUntilReady()