From 259b778aba181abdde06a2d92dc331b75017e54d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 17 Apr 2024 12:16:15 +0800 Subject: [PATCH] add schedule and other parameters --- packages/api/src/jobs/ai/create_digest.ts | 28 +++++++++++++++++++---- packages/api/src/routers/digest_router.ts | 24 +++++++++++++++---- packages/api/src/services/digest.ts | 5 +--- packages/api/src/utils/createTask.ts | 10 +++++++- 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index a7c0e02c4..fa7a975ff 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -4,7 +4,11 @@ import { v4 as uuid } from 'uuid' import { OpenAI } from '@langchain/openai' import { PromptTemplate } from '@langchain/core/prompts' import { LibraryItem } from '../../entity/library_item' -import { htmlToSpeechFile, SpeechFile } from '@omnivore/text-to-speech-handler' +import { + htmlToSpeechFile, + SpeechFile, + SSMLOptions, +} from '@omnivore/text-to-speech-handler' import axios from 'axios' import { searchLibraryItems } from '../../services/library_item' import { redisDataSource } from '../../redis_data_source' @@ -15,8 +19,13 @@ import showdown from 'showdown' import { Digest, writeDigest } from '../../services/digest' import { TaskState } from '../../generated/graphql' +export type CreateDigestJobSchedule = 'daily' | 'weekly' + export interface CreateDigestJobData { userId: string + voices?: string[] + language?: string + rate?: string } export interface CreateDigestJobResponse { @@ -312,7 +321,10 @@ const summarizeItems = async ( } // generate speech files from the summaries -const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => { +const generateSpeechFiles = ( + rankedItems: RankedItem[], + options: SSMLOptions +): SpeechFile[] => { // convert the summaries from markdown to HTML const converter = new showdown.Converter({ backslashEscapesHTMLTags: true, @@ -327,7 +339,7 @@ const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => { ` return htmlToSpeechFile({ content: html, - options: {}, + options, }) }) @@ -363,7 +375,11 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => { const filteredSummaries = filterSummaries(summaries) - const speechFiles = generateSpeechFiles(filteredSummaries) + const speechFiles = generateSpeechFiles(filteredSummaries, { + ...jobData, + primaryVoice: jobData.voices?.[0], + secondaryVoice: jobData.voices?.[1], + }) const title = generateTitle(summaries) const digest: Digest = { id: uuid(), @@ -372,10 +388,12 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => { urlsToAudio: [], jobState: TaskState.Succeeded, speechFiles, - libraryItems: filteredSummaries.map((item) => ({ + chapters: filteredSummaries.map((item, index) => ({ + title: item.libraryItem.title, id: item.libraryItem.id, url: item.libraryItem.originalUrl, thumbnail: item.libraryItem.thumbnail ?? undefined, + wordCount: speechFiles[index].wordCount, })), createdAt: new Date(), description: generateDescription(summaries), diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts index 0a7de6cf9..c415ea6c8 100644 --- a/packages/api/src/routers/digest_router.ts +++ b/packages/api/src/routers/digest_router.ts @@ -1,7 +1,10 @@ import cors from 'cors' import express from 'express' import { env } from '../env' -import { CREATE_DIGEST_JOB } from '../jobs/ai/create_digest' +import { + CreateDigestJobSchedule, + CREATE_DIGEST_JOB, +} from '../jobs/ai/create_digest' import { createJobId, getJob, jobStateToTaskState } from '../queue-processor' import { getDigest } from '../services/digest' import { findActiveUser } from '../services/user' @@ -32,6 +35,13 @@ const isFeedback = (data: any): data is Feedback => { ) } +interface CreateDigestRequest { + voices?: string[] + language?: string + rate?: string + schedule?: CreateDigestJobSchedule +} + export function digestRouter() { const router = express.Router() @@ -72,10 +82,16 @@ export function digestRouter() { return res.sendStatus(202) } + const data = req.body as CreateDigestRequest + // enqueue job and return job id - const result = await enqueueCreateDigest({ - userId, - }) + const result = await enqueueCreateDigest( + { + userId, + ...data, + }, + data.schedule + ) // return job id return res.status(201).send(result) diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts index 2149fade9..85f239f1f 100644 --- a/packages/api/src/services/digest.ts +++ b/packages/api/src/services/digest.ts @@ -5,11 +5,9 @@ import { TaskState } from '../generated/graphql' interface Chapter { title: string -} - -interface LibraryItem { id: string url: string + wordCount: number thumbnail?: string } @@ -26,7 +24,6 @@ export interface Digest { urlsToAudio?: string[] speechFiles?: SpeechFile[] - libraryItems?: LibraryItem[] } const digestKey = (userId: string) => `digest:${userId}` diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index cb835da13..3c6b7ccf9 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -19,6 +19,7 @@ import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize' import { CreateDigestJobData, CreateDigestJobResponse, + CreateDigestJobSchedule, CREATE_DIGEST_JOB, } from '../jobs/ai/create_digest' import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action' @@ -859,7 +860,8 @@ export const enqueueSendEmail = async (jobData: SendEmailJobData) => { } export const enqueueCreateDigest = async ( - data: CreateDigestJobData + data: CreateDigestJobData, + schedule?: CreateDigestJobSchedule ): Promise => { const queue = await getBackendQueue() if (!queue) { @@ -873,6 +875,12 @@ export const enqueueCreateDigest = async ( removeOnFail: true, attempts: 3, priority: getJobPriority(CREATE_DIGEST_JOB), + repeat: schedule + ? { + immediately: true, // run immediately + pattern: schedule === 'daily' ? '0 0 * * *' : '0 0 * * 7', // every day at midnight or every Sunday at midnight + } + : undefined, }) logger.info('create digest job enqueued', { jobId: job.id })