diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index ed8bc47ff..a0e3718fd 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -219,4 +219,7 @@ export class LibraryItem { @Column('float') score?: number + + @Column('text') + previewContent?: string } diff --git a/packages/api/src/jobs/generate_preview_content.ts b/packages/api/src/jobs/generate_preview_content.ts new file mode 100644 index 000000000..6b4c9aadd --- /dev/null +++ b/packages/api/src/jobs/generate_preview_content.ts @@ -0,0 +1,59 @@ +import { parseHTML } from 'linkedom' +import { + findLibraryItemById, + updateLibraryItem, +} from '../services/library_item' +import { findActiveUser } from '../services/user' +import { logger } from '../utils/logger' + +export const GENERATE_PREVIEW_CONTENT_JOB = 'generate-preview' + +interface GeneratePreviewContentData { + libraryItemId: string + userId: string +} + +export const generatePreviewContent = async ( + job: GeneratePreviewContentData +) => { + const { libraryItemId, userId } = job + const user = await findActiveUser(userId) + if (!user) { + logger.error(`User not found: ${userId}`) + return + } + + const libraryItem = await findLibraryItemById(libraryItemId, userId) + if (!libraryItem) { + logger.error(`Library item not found: ${libraryItemId}`) + return + } + + const content = libraryItem.readableContent + if (!content) { + logger.error(`Library item has no content: ${libraryItemId}`) + return + } + + // Generate preview content + logger.info(`Generating preview for library item: ${libraryItemId}`) + // the preview content should be within 600 characters + const document = parseHTML(content).document + const previewContent = document.documentElement.textContent?.slice(0, 600) + if (!previewContent) { + logger.error( + `Failed to generate preview for library item: ${libraryItemId}` + ) + return + } + + await updateLibraryItem( + libraryItemId, + { + previewContent, + }, + userId, + undefined, + true + ) +} diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 188c37efb..f77f92a55 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -31,6 +31,10 @@ import { } from './jobs/email/inbound_emails' import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email' import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail' +import { + generatePreviewContent, + GENERATE_PREVIEW_CONTENT_JOB, +} from './jobs/generate_preview_content' import { exportAllItems, EXPORT_ALL_ITEMS_JOB_NAME, @@ -194,6 +198,8 @@ export const createWorker = (connection: ConnectionOptions) => return updateHome(job.data) case SCORE_LIBRARY_ITEM_JOB: return scoreLibraryItem(job.data) + case GENERATE_PREVIEW_CONTENT_JOB: + return generatePreviewContent(job.data) default: logger.warning(`[queue-processor] unhandled job: ${job.name}`) } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 6ed9de486..01391c7da 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -686,7 +686,8 @@ export const functionResolvers = { canComment: false, canShare: true, dir: libraryItem.directionality, - previewContent: libraryItem.description, + previewContent: + libraryItem.previewContent || libraryItem.description, subscription: libraryItem.subscription, siteName: libraryItem.siteName, siteIcon: libraryItem.siteIcon, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index cf1a1e69f..677ff3733 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -16,7 +16,10 @@ import { SaveResult, } from '../generated/graphql' import { Merge } from '../util' -import { enqueueThumbnailJob } from '../utils/createTask' +import { + enqueueGeneratePreviewContentJob, + enqueueThumbnailJob, +} from '../utils/createTask' import { cleanUrl, generateSlug, @@ -174,6 +177,16 @@ export const savePage = async ( } } + const excerpt = parseResult.parsedContent?.excerpt + // generate preview content if excerpt is less than 180 characters + if (!excerpt || excerpt.length < 180) { + try { + await enqueueGeneratePreviewContentJob(clientRequestId, user.id) + } catch (e) { + logger.error('Failed to enqueue generate preview job', e) + } + } + if (parseResult.highlightData) { const highlight: DeepPartial = { ...parseResult.highlightData, @@ -255,6 +268,7 @@ export const parsedContentToLibraryItem = ({ originalContent: originalHtml, readableContent: parsedContent?.content || '', description: parsedContent?.excerpt, + previewContent: parsedContent?.excerpt, title: title || parsedContent?.title || diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 26ef6362b..212628d38 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -29,6 +29,7 @@ import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action' import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook' import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email' import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' +import { GENERATE_PREVIEW_CONTENT_JOB } from '../jobs/generate_preview_content' import { EXPORT_ALL_ITEMS_JOB_NAME } from '../jobs/integration/export_all_items' import { ExportItemJobData, @@ -110,6 +111,7 @@ export const getJobPriority = (jobName: string): number => { case EXPORT_ALL_ITEMS_JOB_NAME: case REFRESH_ALL_FEEDS_JOB_NAME: case THUMBNAIL_JOB: + case GENERATE_PREVIEW_CONTENT_JOB: return 100 default: @@ -1024,4 +1026,29 @@ export const enqueueScoreJob = async (data: ScoreLibraryItemJobData) => { }) } +export const enqueueGeneratePreviewContentJob = async ( + libraryItemId: string, + userId: string +) => { + const queue = await getBackendQueue() + if (!queue) { + return undefined + } + + return queue.add( + GENERATE_PREVIEW_CONTENT_JOB, + { + libraryItemId, + userId, + }, + { + jobId: `${GENERATE_PREVIEW_CONTENT_JOB}_${libraryItemId}_${JOB_VERSION}`, + removeOnComplete: true, + removeOnFail: true, + priority: getJobPriority(GENERATE_PREVIEW_CONTENT_JOB), + attempts: 3, + } + ) +} + export default createHttpTaskWithToken diff --git a/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql b/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql new file mode 100755 index 000000000..56e6be51d --- /dev/null +++ b/packages/db/migrations/0179.do.add_preview_content_to_library_item.sql @@ -0,0 +1,10 @@ +-- Type: DO +-- Name: add_preview_content_to_library_item +-- Description: Add preview_content column to library_item table + +BEGIN; + +ALTER TABLE omnivore.library_item + ADD COLUMN preview_content TEXT; + +COMMIT; diff --git a/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql b/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql new file mode 100755 index 000000000..fba4a41c2 --- /dev/null +++ b/packages/db/migrations/0179.undo.add_preview_content_to_library_item.sql @@ -0,0 +1,10 @@ +-- Type: UNDO +-- Name: add_preview_content_to_library_item +-- Description: Add preview_content column to library_item table + +BEGIN; + +ALTER TABLE omnivore.library_item + DROP COLUMN preview_content; + +COMMIT;