From b7e6888395578283dc4c070286286e498fe4ee5c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 May 2024 17:42:30 +0800 Subject: [PATCH 1/5] remove debugging logs --- packages/api/src/resolvers/function_resolvers.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 4de6224fa..d4692561f 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -647,16 +647,13 @@ export const functionResolvers = { ctx: WithDataSourcesContext ) { const items = section.items - console.log('items', items) const libraryItemIds = items .filter((item) => item.type === 'library_item') .map((item) => item.id) - console.log('libraryItemIds', libraryItemIds) const libraryItems = ( await ctx.dataLoaders.libraryItems.loadMany(libraryItemIds) ).filter((libraryItem) => !isError(libraryItem)) as Array - console.log('libraryItems', libraryItems) const publicItemIds = section.items .filter((item) => item.type === 'public_item') @@ -667,11 +664,9 @@ export const functionResolvers = { return items .map((item) => { - console.log('item', item) const libraryItem = libraryItems.find( (libraryItem) => item.id === libraryItem.id ) - console.log('libraryItem', libraryItem) if (libraryItem) { return { id: libraryItem.id, From 69dbbd7635bd8e6c43d11f6404f79765710443a9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 May 2024 17:45:51 +0800 Subject: [PATCH 2/5] add score api url to env var --- packages/api/src/services/score.ts | 9 +++------ packages/api/src/util.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index 981eb6f08..545049999 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -1,3 +1,5 @@ +import { env } from '../env' + export interface Feature { library_item_id?: string title: string @@ -29,12 +31,7 @@ export type ScoreApiResponse = Record // item_id -> score export const getScores = async ( data: ScoreApiRequestBody ): Promise => { - const API_URL = 'http://digest-score/batch' - // const token = process.env.SCORE_API_TOKEN - - // if (!token) { - // throw new Error('No score API token found') - // } + const API_URL = env.score.apiUrl const response = await fetch(API_URL, { method: 'POST', diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index fceea60b3..096acf0a0 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -119,6 +119,9 @@ export interface BackendEnv { clientSecret: string authUrl: string } + score: { + apiUrl: string + } } const nullableEnvVars = [ @@ -175,6 +178,7 @@ const nullableEnvVars = [ 'NOTION_CLIENT_ID', 'NOTION_CLIENT_SECRET', 'NOTION_AUTH_URL', + 'SCORE_API_URL', ] // Allow some vars to be null/empty const envParser = @@ -329,6 +333,9 @@ export function getEnv(): BackendEnv { clientSecret: parse('NOTION_CLIENT_SECRET'), authUrl: parse('NOTION_AUTH_URL'), } + const score = { + apiUrl: parse('SCORE_API_URL') || 'http://digest-score/batch', + } return { pg, @@ -352,6 +359,7 @@ export function getEnv(): BackendEnv { subscription, redis, notion, + score, } } From d1d7f93fc34ec8495a478908d3d91e0052c5dd79 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 May 2024 17:58:28 +0800 Subject: [PATCH 3/5] time each steps in the update home job --- packages/api/src/jobs/update_home.ts | 26 +++++++++++++++++++++----- packages/api/src/services/score.ts | 1 - 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index 504b9aaa9..64ab91ff1 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -389,8 +389,12 @@ export const updateHome = async (data: UpdateHomeJobData) => { logger.info(`Updating home for user ${userId}`) + logger.profile('selecting') const candidates = await selectCandidates(user) - logger.info(`Found ${candidates.length} candidates`) + logger.profile('selecting', { + level: 'info', + message: `Found ${candidates.length} candidates`, + }) if (candidates.length <= 10) { logger.info('Not enough candidates found') @@ -399,20 +403,32 @@ export const updateHome = async (data: UpdateHomeJobData) => { // TODO: integrity check on candidates - logger.info('Ranking candidates') + logger.profile('ranking') const rankedCandidates = await rankCandidates(userId, candidates) if (rankedCandidates.length === 0) { logger.info('No candidates found') return } + logger.profile('ranking', { + level: 'info', + message: `Ranked ${rankedCandidates.length} candidates`, + }) // TODO: filter candidates - logger.info('Mix home items to create sections') + logger.profile('mixing') const rankedSections = mixHomeItems(rankedCandidates) - logger.info(`Created ${rankedSections.length} sections`) + logger.profile('mixing', { + level: 'info', + message: `Created ${rankedSections.length} sections`, + }) - logger.info('Appending sections to home') + logger.profile('saving') await appendSectionsToHome(userId, rankedSections, cursor) + logger.profile('saving', { + level: 'info', + message: 'Sections appended to home', + }) + logger.info('Home updated for user', { userId }) } diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index 545049999..bad4519be 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -37,7 +37,6 @@ export const getScores = async ( method: 'POST', headers: { 'Content-Type': 'application/json', - // Authorization: `Bearer ${token}`, }, body: JSON.stringify(data), }) From c7de24e9f8478d0d07b45787e57377d6dc42b6cd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 May 2024 18:22:16 +0800 Subject: [PATCH 4/5] do not rank candidates and just return them all if less than 10 candidates --- packages/api/src/generated/graphql.ts | 4 ++-- packages/api/src/generated/schema.graphql | 2 +- packages/api/src/jobs/update_home.ts | 13 +++++++------ packages/api/src/schema.ts | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index a4392d52e..158adacca 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1310,7 +1310,7 @@ export type HomeItem = { likeCount?: Maybe; previewContent?: Maybe; saveCount?: Maybe; - score: Scalars['Float']; + score?: Maybe; seen_at?: Maybe; slug?: Maybe; source?: Maybe; @@ -6103,7 +6103,7 @@ export type HomeItemResolvers, ParentType, ContextType>; previewContent?: Resolver, ParentType, ContextType>; saveCount?: Resolver, ParentType, ContextType>; - score?: Resolver; + score?: Resolver, ParentType, ContextType>; seen_at?: Resolver, ParentType, ContextType>; slug?: Resolver, ParentType, ContextType>; source?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 1642622d7..18a8fca83 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1179,7 +1179,7 @@ type HomeItem { likeCount: Int previewContent: String saveCount: Int - score: Float! + score: Float seen_at: Date slug: String source: HomeItemSource diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index 64ab91ff1..a4fa33796 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -159,6 +159,11 @@ const rankCandidates = async ( userId: string, candidates: Array ): Promise> => { + if (candidates.length <= 10) { + logger.info('Not enough candidates to rank') + return candidates + } + const unscoredCandidates = candidates.filter((item) => item.score === 0) const data = { @@ -396,8 +401,8 @@ export const updateHome = async (data: UpdateHomeJobData) => { message: `Found ${candidates.length} candidates`, }) - if (candidates.length <= 10) { - logger.info('Not enough candidates found') + if (candidates.length === 0) { + logger.info('No candidates found') return } @@ -405,10 +410,6 @@ export const updateHome = async (data: UpdateHomeJobData) => { logger.profile('ranking') const rankedCandidates = await rankCandidates(userId, candidates) - if (rankedCandidates.length === 0) { - logger.info('No candidates found') - return - } logger.profile('ranking', { level: 'info', message: `Ranked ${rankedCandidates.length} candidates`, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 2e51e3b00..329961d37 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -3148,7 +3148,7 @@ const schema = gql` canShare: Boolean canArchive: Boolean canDelete: Boolean - score: Float! + score: Float } type HomeSection { From 84a6530743410bc8208a10beb51486242c229bc4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 29 May 2024 18:29:48 +0800 Subject: [PATCH 5/5] log job error --- packages/api/src/jobs/update_home.ts | 98 +++++++++++++++------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index a4fa33796..2b863e6f3 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -9,7 +9,7 @@ import { Feature, getScores } from '../services/score' import { findSubscriptionsByNames } from '../services/subscriptions' import { findActiveUser } from '../services/user' import { lanaugeToCode } from '../utils/helpers' -import { logger } from '../utils/logger' +import { logError, logger } from '../utils/logger' export const UPDATE_HOME_JOB = 'UPDATE_HOME_JOB' @@ -386,50 +386,56 @@ export const updateHome = async (data: UpdateHomeJobData) => { const { userId, cursor } = data logger.info('Updating home for user', data) - const user = await findActiveUser(userId) - if (!user) { - logger.error(`User ${userId} not found`) - return + try { + const user = await findActiveUser(userId) + if (!user) { + logger.error(`User ${userId} not found`) + return + } + + logger.info(`Updating home for user ${userId}`) + + logger.profile('selecting') + const candidates = await selectCandidates(user) + logger.profile('selecting', { + level: 'info', + message: `Found ${candidates.length} candidates`, + }) + + if (candidates.length === 0) { + logger.info('No candidates found') + return + } + + // TODO: integrity check on candidates + + logger.profile('ranking') + const rankedCandidates = await rankCandidates(userId, candidates) + logger.profile('ranking', { + level: 'info', + message: `Ranked ${rankedCandidates.length} candidates`, + }) + + // TODO: filter candidates + + logger.profile('mixing') + const rankedSections = mixHomeItems(rankedCandidates) + logger.profile('mixing', { + level: 'info', + message: `Created ${rankedSections.length} sections`, + }) + + logger.profile('saving') + await appendSectionsToHome(userId, rankedSections, cursor) + logger.profile('saving', { + level: 'info', + message: 'Sections appended to home', + }) + + logger.info('Home updated for user', { userId }) + } catch (error) { + logError(error) + + throw error } - - logger.info(`Updating home for user ${userId}`) - - logger.profile('selecting') - const candidates = await selectCandidates(user) - logger.profile('selecting', { - level: 'info', - message: `Found ${candidates.length} candidates`, - }) - - if (candidates.length === 0) { - logger.info('No candidates found') - return - } - - // TODO: integrity check on candidates - - logger.profile('ranking') - const rankedCandidates = await rankCandidates(userId, candidates) - logger.profile('ranking', { - level: 'info', - message: `Ranked ${rankedCandidates.length} candidates`, - }) - - // TODO: filter candidates - - logger.profile('mixing') - const rankedSections = mixHomeItems(rankedCandidates) - logger.profile('mixing', { - level: 'info', - message: `Created ${rankedSections.length} sections`, - }) - - logger.profile('saving') - await appendSectionsToHome(userId, rankedSections, cursor) - logger.profile('saving', { - level: 'info', - message: 'Sections appended to home', - }) - - logger.info('Home updated for user', { userId }) }