From 7eadda02e8f0735bea52849057095ba7d02aa34c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:18:58 +0800 Subject: [PATCH 1/4] Post the avg latency in the update message --- packages/queue-manager/src/index.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index f26246914..47383ba58 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -98,15 +98,13 @@ const checkShouldPauseQueues = async () => { (acc, point) => acc + (point.value?.doubleValue ?? 0), 0 ) / ts.points.length - console.log('avgLatency: ', avgLatency) if (avgLatency > LATENCY_THRESHOLD) { - shouldPauseQueues = true - break + return [true, avgLatency] } } } - return shouldPauseQueues + return [false, 0] } const getQueueTaskCount = async (queueName: string) => { @@ -147,7 +145,7 @@ async function checkMetricsAndPauseQueues() { throw new Error('environment not supplied.') } - const shouldPauseQueues = await checkShouldPauseQueues() + const [shouldPauseQueues, avgLatency] = await checkShouldPauseQueues() if (shouldPauseQueues) { let rssQueueCount: number | string = 'unknown' @@ -159,7 +157,7 @@ async function checkMetricsAndPauseQueues() { console.log('error fetching queue counts', err) } - const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + const message = `Both queues have been paused due to API latency threshold exceedance (${avgLatency}).\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` await pauseQueues() await postToDiscord(message) From d8a5ed42e8d52c422fb8e67fdd41a208e36685d4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:28:56 +0800 Subject: [PATCH 2/4] Remove unused var --- packages/queue-manager/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 47383ba58..27d13e22b 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -79,8 +79,6 @@ const checkShouldPauseQueues = async () => { }, }) - let shouldPauseQueues = false - for (const ts of timeSeries) { // We only want to look at the backend service right now if ( From fd57e075d7ca112dbdb883b30845df11d4f7a083 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:30:57 +0800 Subject: [PATCH 3/4] Clean up return signature --- packages/queue-manager/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 27d13e22b..37124ce7b 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -97,12 +97,12 @@ const checkShouldPauseQueues = async () => { 0 ) / ts.points.length if (avgLatency > LATENCY_THRESHOLD) { - return [true, avgLatency] + return { shouldPauseQueues: true, avgLatency: avgLatency } } } } - return [false, 0] + return { shouldPauseQueues: false, avgLatency: 0 } } const getQueueTaskCount = async (queueName: string) => { @@ -143,7 +143,7 @@ async function checkMetricsAndPauseQueues() { throw new Error('environment not supplied.') } - const [shouldPauseQueues, avgLatency] = await checkShouldPauseQueues() + const { shouldPauseQueues, avgLatency } = await checkShouldPauseQueues() if (shouldPauseQueues) { let rssQueueCount: number | string = 'unknown' From 1053c9b53a719627c62b7dccb1d811641427b8c3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 09:11:11 +0800 Subject: [PATCH 4/4] Bump threshold for testing --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 37124ce7b..9574d0684 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -30,7 +30,7 @@ if ( const LATENCY_THRESHOLD = 500 const RSS_QUEUE_THRESHOLD = 20_000 -const IMPORT_QUEUE_THRESHOLD = 200_000 +const IMPORT_QUEUE_THRESHOLD = 250_000 const postToDiscord = async (message: string) => { console.log('notify message', { message })