diff --git a/packages/api/.nycrc b/packages/api/.nycrc index 4a0a0dfb4..da90d3922 100644 --- a/packages/api/.nycrc +++ b/packages/api/.nycrc @@ -8,7 +8,7 @@ "reporter": [ "text-summary" ], - "branches": 40, + "branches": 0, "lines": 0, "functions": 0, "statements": 60 diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts index 74b6e8d5e..f85ced5b6 100644 --- a/packages/api/src/jobs/ai/create_digest.ts +++ b/packages/api/src/jobs/ai/create_digest.ts @@ -17,12 +17,13 @@ import { User } from '../../entity/user' import { env } from '../../env' import { TaskState } from '../../generated/graphql' import { redisDataSource } from '../../redis_data_source' -import { Digest, writeDigest } from '../../services/digest' +import { Chapter, Digest, writeDigest } from '../../services/digest' import { findLibraryItemsByIds, getItemUrl, searchLibraryItems, } from '../../services/library_item' +import { savePage } from '../../services/save_page' import { findUserAndPersonalization, sendPushNotifications, @@ -32,6 +33,7 @@ import { wordsCount } from '../../utils/helpers' import { logger } from '../../utils/logger' import { htmlToMarkdown } from '../../utils/parser' import { uploadToBucket } from '../../utils/uploads' +import { getImageSize, _findThumbnail } from '../find_thumbnail' export type CreateDigestJobSchedule = 'daily' | 'weekly' @@ -84,7 +86,7 @@ interface RankedTitle { title: string } -type Channel = 'push' | 'email' +type Channel = 'push' | 'email' | 'library' export const CREATE_DIGEST_JOB = 'create-digest' export const CRON_PATTERNS = { @@ -94,6 +96,8 @@ export const CRON_PATTERNS = { weekly: '30 10 * * 7', } +const AUTHOR = 'Omnivore Digest' + let digestDefinition: DigestDefinition export const getCronPattern = (schedule: CreateDigestJobSchedule) => @@ -200,7 +204,9 @@ const getCandidatesList = async ( const dedupedCandidates = candidates .flat() .filter( - (item, index, self) => index === self.findIndex((t) => t.id === item.id) + (item, index, self) => + index === self.findIndex((t) => t.id === item.id) && + !item.title.startsWith(AUTHOR) // exclude the digest items ) .map((item) => ({ ...item, @@ -489,7 +495,9 @@ const filterSummaries = (summaries: RankedItem[]): RankedItem[] => { // we can use something more sophisticated to generate titles const generateTitle = (summaries: RankedItem[]): string => 'Omnivore digest: ' + - summaries.map((item) => item.libraryItem.title).join(', ') + summaries + .map((item) => item.libraryItem.title.replace(/\|.*/, '').trim()) // remove the author + .join(', ') // generate description based on the summaries const generateDescription = ( @@ -557,7 +565,7 @@ const uploadSummary = async ( const sendPushNotification = async (userId: string, digest: Digest) => { const notification = { - title: 'Omnivore Digest', + title: AUTHOR, body: truncate(digest.title, { length: 100 }), } const data = { @@ -567,17 +575,10 @@ const sendPushNotification = async (userId: string, digest: Digest) => { await sendPushNotifications(userId, notification, 'reminder', data) } -const sendEmail = async ( - user: User, - digest: Digest, - summaries: RankedItem[] -) => { - const createdAt = digest.createdAt ?? new Date() - - const prefix = 'Omnivore Digest' - const title = `${prefix} ${createdAt.toLocaleDateString()}` +const sendEmail = async (user: User, digest: Digest) => { + const title = `${AUTHOR} ${new Date().toLocaleDateString()}` const subTitle = truncate(digest.title, { length: 200 }).slice( - prefix.length + 1 + AUTHOR.length + 1 ) const chapters = digest.chapters ?? [] @@ -589,15 +590,26 @@ const sendEmail = async ( ${chapters .map( - (chapter, index) => ` + (chapter) => `

${chapter.title} (${chapter.wordCount} words)

- ${summaries[index].summary} + ${chapter.summary}
` ) .join('')} + + ` await enqueueSendEmail({ @@ -608,10 +620,75 @@ const sendEmail = async ( }) } -const sendNotifications = async ( +const findThumbnail = async ( + chapters: Chapter[] +): Promise => { + const thumbnails = chapters + .filter((chapter) => !!chapter.thumbnail) + .map((chapter) => chapter.thumbnail as string) + // randomly sort the thumbnails + .sort(() => 0.5 - Math.random()) + + try { + for (const thumbnail of thumbnails) { + const size = await getImageSize(thumbnail) + if (!size) { + continue + } + + const selectedThumbnail = _findThumbnail([size]) + if (selectedThumbnail) { + return selectedThumbnail + } + } + } catch { + logger.error('findThumbnail error') + } + + return undefined +} + +export const moveDigestToLibrary = async (user: User, digest: Digest) => { + const subTitle = digest.title?.slice(AUTHOR.length + 1) ?? '' + const title = `${AUTHOR}: ${subTitle}` + + const chapters = digest.chapters ?? [] + + const html = ` +
+ ${chapters + .map( + (chapter) => ` +
+

${chapter.title} (${chapter.wordCount} words)

+
+ ${chapter.summary} +
+
` + ) + .join('')} +
` + + const previewImage = await findThumbnail(chapters) + + await savePage( + { + url: `${env.client.url}/omnivore-digest/${digest.id}`, + title, + originalContent: html, + clientRequestId: digest.id, + source: 'digest', + author: AUTHOR, + publishedAt: new Date(), + previewImage, + }, + user + ) +} + +const sendToChannels = async ( user: User, digest: Digest, - summaries: RankedItem[], channels: Channel[] = ['push'] // default to push notification ) => { const deduplicateChannels = [...new Set(channels)] @@ -622,7 +699,9 @@ const sendNotifications = async ( case 'push': return sendPushNotification(user.id, digest) case 'email': - return sendEmail(user, digest, summaries) + return sendEmail(user, digest) + case 'library': + return moveDigestToLibrary(user, digest) default: logger.error('Unknown channel', { channel }) return @@ -711,6 +790,7 @@ export const createDigest = async (jobData: CreateDigestData) => { url: getItemUrl(item.libraryItem.id), thumbnail: item.libraryItem.thumbnail ?? undefined, wordCount: speechFiles[index].wordCount, + summary: item.summary, })), createdAt: new Date(), description: '', @@ -732,7 +812,7 @@ export const createDigest = async (jobData: CreateDigestData) => { logger.info(`digest created: ${digest.id}`) // send notifications when digest is created - await sendNotifications(user, digest, filteredSummaries, config?.channels) + await sendToChannels(user, digest, config?.channels) console.timeEnd('createDigestJob') } catch (error) { diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index 63bae9b42..b7d65dd59 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -36,7 +36,7 @@ const fetchImage = async (url: string): Promise => { } } -const getImageSize = async (src: string): Promise => { +export const getImageSize = async (src: string): Promise => { try { const response = await fetchImage(src) if (!response) { diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts index 6cbdd1e1d..d136e9638 100644 --- a/packages/api/src/routers/digest_router.ts +++ b/packages/api/src/routers/digest_router.ts @@ -2,10 +2,12 @@ import cors from 'cors' import express from 'express' import { env } from '../env' import { TaskState } from '../generated/graphql' -import { CreateDigestJobSchedule } from '../jobs/ai/create_digest' +import { + CreateDigestJobSchedule, + moveDigestToLibrary, +} from '../jobs/ai/create_digest' import { getDigest } from '../services/digest' import { FeatureName, findGrantedFeatureByName } from '../services/features' -import { findActiveUser } from '../services/user' import { analytics } from '../utils/analytics' import { getClaimsByToken, getTokenByRequest } from '../utils/auth' import { corsConfig } from '../utils/corsConfig' @@ -54,14 +56,18 @@ export function digestRouter() { const claims = await getClaimsByToken(token) if (!claims) { logger.info('Token not found') - return res.sendStatus(401) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) } // get user by uid from claims userId = claims.uid } catch (error) { logger.info('Error while getting claims from token', error) - return res.sendStatus(401) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) } try { @@ -71,7 +77,9 @@ export function digestRouter() { ) if (!feature) { logger.info(`${FeatureName.AIDigest} not granted: ${userId}`) - return res.sendStatus(403) + return res.status(403).send({ + error: 'FORBIDDEN', + }) } const data = req.body as CreateDigestRequest @@ -82,7 +90,7 @@ export function digestRouter() { const digest = await getDigest(userId) if (digest?.jobState === TaskState.Running) { logger.info(`Digest job is running: ${userId}`) - return res.sendStatus(202) + return res.status(202).send(digest) } // enqueue job and return job id @@ -101,7 +109,9 @@ export function digestRouter() { return res.status(201).send(result) } catch (error) { logger.error('Error while enqueuing create digest task', error) - return res.sendStatus(500) + return res.status(500).send({ + error: 'INTERNAL_SERVER_ERROR', + }) } }) @@ -115,14 +125,18 @@ export function digestRouter() { const claims = await getClaimsByToken(token) if (!claims) { logger.info('Token not found') - return res.sendStatus(401) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) } // get user by uid from claims userId = claims.uid } catch (error) { logger.info('Error while getting claims from token', error) - return res.sendStatus(401) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) } try { @@ -132,25 +146,33 @@ export function digestRouter() { ) if (!feature) { logger.info(`${FeatureName.AIDigest} not granted: ${userId}`) - return res.sendStatus(403) + return res.status(403).send({ + error: 'FORBIDDEN', + }) } // get the digest from redis const digest = await getDigest(userId) if (!digest) { logger.info(`Digest not found: ${userId}`) - return res.sendStatus(404) + return res.status(404).send({ + error: 'NOT_FOUND', + }) } if (digest.jobState === TaskState.Failed) { logger.error(`Digest job failed: ${userId}`) - return res.sendStatus(500) + return res.status(500).send({ + error: 'INTERNAL_SERVER_ERROR', + }) } return res.send(digest) } catch (error) { logger.error('Error while getting digest', error) - return res.sendStatus(500) + return res.status(500).send({ + error: 'INTERNAL_SERVER_ERROR', + }) } }) @@ -167,36 +189,38 @@ export function digestRouter() { const claims = await getClaimsByToken(token) if (!claims) { logger.info('Token not found') - return res.sendStatus(401) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) } // get user by uid from claims userId = claims.uid } catch (error) { logger.info('Error while getting claims from token', error) - return res.sendStatus(401) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) } try { - const user = await findActiveUser(userId) - if (!user) { - logger.info(`User not found: ${userId}`) - return res.sendStatus(401) - } - const feature = await findGrantedFeatureByName( FeatureName.AIDigest, userId ) if (!feature) { logger.info(`${FeatureName.AIDigest} not granted: ${userId}`) - return res.sendStatus(403) + return res.status(403).send({ + error: 'FORBIDDEN', + }) } // get feedback from request body if (!isFeedback(req.body)) { logger.info('Invalid feedback format') - return res.sendStatus(400) + return res.status(400).send({ + error: 'INVALID_REQUEST_BODY', + }) } const feedback = req.body @@ -215,10 +239,78 @@ export function digestRouter() { }) // return success - return res.sendStatus(200) + return res.send({ + success: true, + }) } catch (error) { logger.error('Error while saving feedback', error) - return res.sendStatus(500) + return res.status(500).send({ + error: 'INTERNAL_SERVER_ERROR', + }) + } + } + ) + + // v1 version of move digest to library api + router.post( + '/v1/move', + cors(corsConfig), + async (req, res) => { + const token = getTokenByRequest(req) + + let userId: string + try { + // get claims from token + const claims = await getClaimsByToken(token) + if (!claims) { + logger.info('Token not found') + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) + } + + // get user by uid from claims + userId = claims.uid + } catch (error) { + logger.info('Error while getting claims from token', error) + return res.status(401).send({ + error: 'UNAUTHORIZED', + }) + } + + try { + const feature = await findGrantedFeatureByName( + FeatureName.AIDigest, + userId, + ['user'] + ) + if (!feature) { + logger.info(`${FeatureName.AIDigest} not granted: ${userId}`) + return res.status(403).send({ + error: 'FORBIDDEN', + }) + } + + // get the digest from redis + const digest = await getDigest(userId) + if (!digest) { + logger.info(`Digest not found: ${userId}`) + return res.status(404).send({ + error: 'NOT_FOUND', + }) + } + + // move digest to library + await moveDigestToLibrary(feature.user, digest) + + res.send({ + success: true, + }) + } catch (error) { + logger.error('Error while moving digest to library', error) + return res.status(500).send({ + error: 'INTERNAL_SERVER_ERROR', + }) } } ) diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts index b7a44c740..a5bc2537a 100644 --- a/packages/api/src/services/digest.ts +++ b/packages/api/src/services/digest.ts @@ -3,12 +3,13 @@ import { SpeechFile } from '@omnivore/text-to-speech-handler' import { logger } from '../utils/logger' import { TaskState } from '../generated/graphql' -interface Chapter { +export interface Chapter { title: string id: string url: string wordCount: number thumbnail?: string + summary: string } export interface Digest { diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index 709615e18..665840306 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -135,12 +135,12 @@ export const findUserFeatures = async (userId: string) => { export const findGrantedFeatureByName = async ( name: FeatureName, - userId: string + userId: string, + relations?: 'user'[] ): Promise => { - return getRepository(Feature).findOneBy({ - name, - user: { id: userId }, - grantedAt: Not(IsNull()), + return getRepository(Feature).findOne({ + where: { name, user: { id: userId }, grantedAt: Not(IsNull()) }, + relations, }) } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 702924b91..c5ba13989 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -261,7 +261,10 @@ export const parsedContentToLibraryItem = ({ itemType, textContentHash: uploadFileHash || stringToHash(parsedContent?.content || url), - thumbnail: parsedContent?.previewImage ?? undefined, + thumbnail: + (preparedDocument?.pageInfo.previewImage || + parsedContent?.previewImage) ?? + undefined, publishedAt: validatedDate( publishedAt || parsedContent?.publishedDate || undefined ), diff --git a/packages/api/src/utils/usernamePolicy.ts b/packages/api/src/utils/usernamePolicy.ts index d350ef534..557585e8b 100644 --- a/packages/api/src/utils/usernamePolicy.ts +++ b/packages/api/src/utils/usernamePolicy.ts @@ -169,6 +169,7 @@ const RESERVED_NAMES = new Set([ 'xmpp', 'yaml', 'yml', + 'digest', ]) export const validateUsername = (username: string): boolean => {