From 6bb11ce99bfb03a17818cf4c369f33fbd410e95b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 1 Jul 2024 11:02:42 +0800 Subject: [PATCH 1/5] fix: returns a stub score (0) in case score api throws an error --- packages/api/src/services/score.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index 256840b5c..cae0bebf0 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -1,5 +1,6 @@ import axios from 'axios' import { env } from '../env' +import { logError } from '../utils/logger' export interface Feature { library_item_id?: string @@ -63,14 +64,23 @@ class ScoreClientImpl implements ScoreClient { } async getScores(data: ScoreApiRequestBody): Promise { - const response = await axios.post(this.apiUrl, data, { - headers: { - 'Content-Type': 'application/json', - }, - timeout: 5000, - }) + try { + const response = await axios.post(this.apiUrl, data, { + headers: { + 'Content-Type': 'application/json', + }, + timeout: 5000, + }) - return response.data + return response.data + } catch (error) { + logError(error) + + // Returns a stub score (0) in case of an error + return { + [Object.keys(data.items)[0]]: { score: 0 }, + } + } } } From 9f7065c5c3f6cd8f4abf2bf5198bcfe06154b873 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 1 Jul 2024 11:51:57 +0800 Subject: [PATCH 2/5] do not update library item score if 0 --- packages/api/src/jobs/score_library_item.ts | 26 ++++++++++----------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts index 2e89f9955..a3f0b9bd5 100644 --- a/packages/api/src/jobs/score_library_item.ts +++ b/packages/api/src/jobs/score_library_item.ts @@ -97,24 +97,22 @@ export const scoreLibraryItem = async ( items: itemFeatures, }) - logger.info('Scores', scores) - const score = scores[libraryItem.id]['score'] + const score = scores[libraryItem.id].score if (!score) { logger.error('Failed to score library item', data) - throw new Error('Failed to score library item') + } else { + await updateLibraryItem( + libraryItem.id, + { + score, + }, + userId, + undefined, + true + ) + logger.info('Library item score updated', data) } - await updateLibraryItem( - libraryItem.id, - { - score, - }, - userId, - undefined, - true - ) - logger.info('Library item scored', data) - try { await enqueueUpdateHomeJob({ userId, From 550cef241cc616623ce9a3a98dbd60da568f6c8b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 1 Jul 2024 12:01:42 +0800 Subject: [PATCH 3/5] observe digest-score api lantency in prometheus --- packages/api/src/jobs/update_home.ts | 2 -- packages/api/src/services/score.ts | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index d97b0b260..b57f71adc 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -494,8 +494,6 @@ const latency = new client.Histogram({ buckets: [0.1, 0.5, 1, 2, 5, 10], }) -latency.observe(10) - registerMetric(latency) export const updateHome = async (data: UpdateHomeJobData) => { diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index cae0bebf0..aed8f4b7b 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -1,5 +1,7 @@ import axios from 'axios' +import client from 'prom-client' import { env } from '../env' +import { registerMetric } from '../prometheus' import { logError } from '../utils/logger' export interface Feature { @@ -38,6 +40,15 @@ export type ScoreBody = { score: number } +// use prometheus to monitor the latency of digest score api +const latency = new client.Histogram({ + name: 'omnivore_digest_score_latency', + help: 'Latency of digest score API in seconds', + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 30, 60], +}) + +registerMetric(latency) + export type ScoreApiResponse = Record // item_id -> score interface ScoreClient { getScores(data: ScoreApiRequestBody): Promise @@ -65,6 +76,8 @@ class ScoreClientImpl implements ScoreClient { async getScores(data: ScoreApiRequestBody): Promise { try { + const start = Date.now() + const response = await axios.post(this.apiUrl, data, { headers: { 'Content-Type': 'application/json', @@ -72,6 +85,9 @@ class ScoreClientImpl implements ScoreClient { timeout: 5000, }) + const duration = (Date.now() - start) / 1000 // in seconds + latency.observe(duration) + return response.data } catch (error) { logError(error) From ad88cbaea8bbd92d2f13d6a623e902ec9891b39b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 1 Jul 2024 12:05:48 +0800 Subject: [PATCH 4/5] observe failed digest-score api call lantency in prometheus too --- packages/api/src/jobs/update_home.ts | 2 +- packages/api/src/services/score.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index b57f71adc..e161f967b 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -248,7 +248,7 @@ const rankCandidates = async ( const scores = await scoreClient.getScores(data) // update scores for candidates candidates.forEach((item) => { - item.score = scores[item.id]['score'] || 0 + item.score = scores[item.id].score || 0 }) // rank candidates by score in descending order diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index aed8f4b7b..92428cd40 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -75,9 +75,9 @@ class ScoreClientImpl implements ScoreClient { } async getScores(data: ScoreApiRequestBody): Promise { - try { - const start = Date.now() + const start = Date.now() + try { const response = await axios.post(this.apiUrl, data, { headers: { 'Content-Type': 'application/json', @@ -85,9 +85,6 @@ class ScoreClientImpl implements ScoreClient { timeout: 5000, }) - const duration = (Date.now() - start) / 1000 // in seconds - latency.observe(duration) - return response.data } catch (error) { logError(error) @@ -96,6 +93,9 @@ class ScoreClientImpl implements ScoreClient { return { [Object.keys(data.items)[0]]: { score: 0 }, } + } finally { + const duration = (Date.now() - start) / 1000 // in seconds + latency.observe(duration) } } } From 70829a6b880d0d58bdd21b70a68ed553e9463414 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 1 Jul 2024 12:08:56 +0800 Subject: [PATCH 5/5] remove initial observation --- packages/api/src/queue-processor.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 7416fcb92..d9b762937 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -92,8 +92,6 @@ const jobLatency = new client.Histogram({ buckets: [0, 1, 5, 10, 50, 100, 500], }) -jobLatency.observe(10) - registerMetric(jobLatency) export const getBackendQueue = async (