From 9ef91ff1cae5a9b552477a2460876b64d1b2b7d3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 13 Jun 2024 14:48:14 +0800 Subject: [PATCH] add an REST API to trigger the prune trash job --- packages/api/src/jobs/prune_trash.ts | 13 ++++- packages/api/src/queue-processor.ts | 14 +---- packages/api/src/routers/svc/links.ts | 53 ++++++------------- packages/api/src/services/library_item.ts | 14 ----- packages/api/src/utils/createTask.ts | 19 +++++++ .../0181.do.batch_delete_trash_items.sql | 6 ++- 6 files changed, 50 insertions(+), 69 deletions(-) diff --git a/packages/api/src/jobs/prune_trash.ts b/packages/api/src/jobs/prune_trash.ts index 953c060be..55f54694c 100644 --- a/packages/api/src/jobs/prune_trash.ts +++ b/packages/api/src/jobs/prune_trash.ts @@ -1,5 +1,14 @@ -import { pruneTrash } from '../services/library_item' +import { appDataSource } from '../data_source' export const PRUNE_TRASH_JOB = 'prune_trash' -export const pruneTrashJob = async () => pruneTrash() +interface PruneTrashJobData { + numDays: number +} + +export const pruneTrashJob = async (jobData: PruneTrashJobData) => { + // call the stored procedure to delete trash items older than {numDays} days + await appDataSource.query( + `CALL omnivore.batch_delete_trash_items(${jobData.numDays});` + ) +} diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 44df2e905..00440da53 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -216,7 +216,7 @@ export const createWorker = (connection: ConnectionOptions) => case GENERATE_PREVIEW_CONTENT_JOB: return generatePreviewContent(job.data) case PRUNE_TRASH_JOB: - return pruneTrashJob() + return pruneTrashJob(job.data) default: logger.warning(`[queue-processor] unhandled job: ${job.name}`) } @@ -251,18 +251,6 @@ const setupCronJobs = async () => { }, } ) - - await queue.add( - PRUNE_TRASH_JOB, - {}, - { - priority: getJobPriority(PRUNE_TRASH_JOB), - repeat: { - // daily at 3am - pattern: '0 3 * * *', - }, - } - ) } const main = async () => { diff --git a/packages/api/src/routers/svc/links.ts b/packages/api/src/routers/svc/links.ts index f247b6ffa..bd35351cc 100644 --- a/packages/api/src/routers/svc/links.ts +++ b/packages/api/src/routers/svc/links.ts @@ -2,12 +2,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' -import { LessThan } from 'typeorm' -import { LibraryItemState } from '../../entity/library_item' import { readPushSubscription } from '../../pubsub' import { userRepository } from '../../repository/user' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { deleteLibraryItemsByAdmin } from '../../services/library_item' +import { enqueuePruneTrashJob } from '../../utils/createTask' import { logger } from '../../utils/logger' interface CreateLinkRequestMessage { @@ -16,28 +14,7 @@ interface CreateLinkRequestMessage { } type PruneMessage = { - expireInDays: number - folder?: string - state?: LibraryItemState -} - -const isPruneMessage = (obj: any): obj is PruneMessage => 'expireInDays' in obj - -const getPruneMessage = (msgStr: string): PruneMessage => { - try { - const obj = JSON.parse(msgStr) as unknown - if (isPruneMessage(obj)) { - return obj - } - } catch (err) { - logger.error('error deserializing event: ', { msgStr, err }) - } - - // default to prune following folder items older than 30 days - return { - folder: 'following', - expireInDays: 30, - } + ttlInDays?: number } export function linkServiceRouter() { @@ -84,28 +61,28 @@ export function linkServiceRouter() { } }) - router.post('/prune', async (req, res) => { + router.post('/pruneTrash', async (req, res) => { const { message: msgStr, expired } = readPushSubscription(req) - if (!msgStr) { - return res.status(200).send('Bad Request') - } - if (expired) { logger.info('discarding expired message') return res.status(200).send('Expired') } - const pruneMessage = getPruneMessage(msgStr) - const expireTime = pruneMessage.expireInDays * 1000 * 60 * 60 * 24 // convert days to milliseconds + // default to prune trash items older than 14 days + let ttlInDays = 14 + + if (msgStr) { + const pruneMessage = JSON.parse(msgStr) as PruneMessage + + if (pruneMessage.ttlInDays) { + ttlInDays = pruneMessage.ttlInDays + } + } try { - const result = await deleteLibraryItemsByAdmin({ - folder: pruneMessage.folder, - state: pruneMessage.state, - updatedAt: LessThan(new Date(Date.now() - expireTime)), - }) - logger.info('prune result', result) + const job = await enqueuePruneTrashJob(ttlInDays) + logger.info('enqueue prune trash job', { id: job?.id }) return res.sendStatus(200) } catch (error) { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 7a7a898f3..2727cedf5 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1354,17 +1354,6 @@ export const deleteLibraryItemsByUserId = async (userId: string) => { ) } -export const deleteLibraryItemsByAdmin = async ( - criteria: FindOptionsWhere -) => { - return authTrx( - async (tx) => tx.withRepository(libraryItemRepository).delete(criteria), - undefined, - undefined, - 'admin' - ) -} - export const batchDelete = async (criteria: FindOptionsWhere) => { const batchSize = 1000 @@ -1779,6 +1768,3 @@ export const downloadOriginalContent = async ( }) ) } - -export const pruneTrash = async () => - appDataSource.query(`CALL omnivore.batch_delete_trash_items();`) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index f10971332..2957926fd 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -1053,4 +1053,23 @@ export const enqueueGeneratePreviewContentJob = async ( ) } +export const enqueuePruneTrashJob = async (numDays: number) => { + const queue = await getBackendQueue() + if (!queue) { + return undefined + } + + return queue.add( + PRUNE_TRASH_JOB, + { numDays }, + { + jobId: `${PRUNE_TRASH_JOB}_${numDays}_${JOB_VERSION}`, + removeOnComplete: true, + removeOnFail: true, + priority: getJobPriority(PRUNE_TRASH_JOB), + attempts: 3, + } + ) +} + export default createHttpTaskWithToken diff --git a/packages/db/migrations/0181.do.batch_delete_trash_items.sql b/packages/db/migrations/0181.do.batch_delete_trash_items.sql index 38518b6b6..afcdcc880 100755 --- a/packages/db/migrations/0181.do.batch_delete_trash_items.sql +++ b/packages/db/migrations/0181.do.batch_delete_trash_items.sql @@ -4,7 +4,9 @@ BEGIN; -CREATE OR REPLACE PROCEDURE omnivore.batch_delete_trash_items() +CREATE OR REPLACE PROCEDURE omnivore.batch_delete_trash_items( + num_days INT +) LANGUAGE plpgsql AS $$ DECLARE user_record RECORD; @@ -26,7 +28,7 @@ BEGIN WHERE user_id = user_record.id AND state = 'DELETED' - AND deleted_at < NOW() - INTERVAL '14 days'; + AND deleted_at < NOW() - INTERVAL '1 day' * num_days; COMMIT; END;