Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

format summary of digest in library and email

Browse Source
This commit is contained in:
Hongbo Wu
2024-05-09 17:38:21 +08:00
parent ab85805e24
commit 4c48a44512
3 changed files with 29 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
packages/api/src/jobs/ai/create_digest.ts
View File

@ -9,7 +9,6 @@ import {
} from '@omnivore/text-to-speech-handler'
import axios from 'axios'
import { truncate } from 'lodash'
import showdown from 'showdown'
import { v4 as uuid } from 'uuid'
import yaml from 'yaml'
import { LibraryItem } from '../../entity/library_item'
@ -32,7 +31,7 @@ import { analytics } from '../../utils/analytics'
import { enqueueSendEmail } from '../../utils/createTask'
import { wordsCount } from '../../utils/helpers'
import { logger } from '../../utils/logger'
import { htmlToMarkdown } from '../../utils/parser'
import { htmlToMarkdown, markdownToHtml } from '../../utils/parser'
import { uploadToBucket } from '../../utils/uploads'
import { getImageSize, _findThumbnail } from '../find_thumbnail'
@ -455,22 +454,18 @@ const summarizeItems = async (
// generate speech files from the summaries
const generateSpeechFiles = (
rankedItems: RankedItem[],
summariesInHtml: string[],
options: SSMLOptions
): SpeechFile[] => {
console.time('generateSpeechFiles')
// convert the summaries from markdown to HTML
const converter = new showdown.Converter({
backslashEscapesHTMLTags: true,
})
const speechFiles = rankedItems.map((item) => {
const speechFiles = summariesInHtml.map((summary) => {
const html = `
<div id="readability-content">
<div id="readability-page-1">
${converter.makeHtml(item.summary)}
</div>
</div>`
<div id="readability-content">
<div id="readability-page-1">
${summary}
</div>
</div>`
return htmlToSpeechFile({
content: html,
options,
@ -596,7 +591,7 @@ const sendEmail = async (user: User, digest: Digest, channels: Channel[]) => {
<div>
<a href="${chapter.url}"><h3>${chapter.title} (${chapter.wordCount} words)</h3></a>
<div>
${chapter.summary}
${chapter.html}
</div>
</div>`
)
@ -677,14 +672,14 @@ export const moveDigestToLibrary = async (user: User, digest: Digest) => {
const html = `
<html>
<body>
<div style="text-align: justify;" class="_omnivore_digest">
<div class="_omnivore_digest">
${chapters
.map(
(chapter) => `
<div>
<a href="${chapter.url}"><h3>${chapter.title} (${chapter.wordCount} words)</h3></a>
<div>
${chapter.summary}
${chapter.html}
</div>
</div>`
)
@ -807,8 +802,16 @@ export const createDigest = async (jobData: CreateDigestData) => {
console.timeEnd('summarizeItems')
const filteredSummaries = filterSummaries(summaries)
const summariesInHtml = filteredSummaries.map((item) => {
try {
return markdownToHtml(item.summary)
} catch (error) {
logger.error('markdownToHtml error', error)
return ''
}
})
const speechFiles = generateSpeechFiles(filteredSummaries, {
const speechFiles = generateSpeechFiles(summariesInHtml, {
...jobData,
primaryVoice: jobData.voices?.[0],
secondaryVoice: jobData.voices?.[1],
@ -826,7 +829,7 @@ export const createDigest = async (jobData: CreateDigestData) => {
url: getItemUrl(item.libraryItem.id),
thumbnail: item.libraryItem.thumbnail ?? undefined,
wordCount: speechFiles[index].wordCount,
summary: item.summary,
html: summariesInHtml[index],
})),
createdAt: new Date(),
description: '',

6
packages/api/src/services/digest.ts
View File

@ -1,7 +1,7 @@
import { redisDataSource } from '../redis_data_source'
import { SpeechFile } from '@omnivore/text-to-speech-handler'
import { logger } from '../utils/logger'
import { TaskState } from '../generated/graphql'
import { redisDataSource } from '../redis_data_source'
import { logger } from '../utils/logger'
export interface Chapter {
title: string
@ -9,7 +9,7 @@ export interface Chapter {
url: string
wordCount: number
thumbnail?: string
summary: string
html: string
}
export interface Digest {

6
packages/api/src/utils/parser.ts
View File

@ -16,6 +16,7 @@ import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
import { ElementNode } from 'node-html-markdown/dist/nodes'
import Parser from 'rss-parser'
import { parser } from 'sax'
import showdown from 'showdown'
import { ILike } from 'typeorm'
import { promisify } from 'util'
import { v4 as uuid } from 'uuid'
@ -702,7 +703,10 @@ export const htmlToMarkdown = (html: string) => {
}
export const markdownToHtml = (markdown: string) => {
return nhm.translate(/* markdown */ markdown)
const converter = new showdown.Converter({
backslashEscapesHTMLTags: true,
})
return converter.makeHtml(markdown)
}
export const getDistillerResult = async (