From aa2caca9443f2bb17b3ad84e045153368b679bde Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 21 Aug 2024 18:47:23 +0800 Subject: [PATCH] do not cache content from https://jacksonh.org --- packages/api/src/services/library_item.ts | 2 +- packages/content-fetch/src/request_handler.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 2a64bd77a..bd64cf1e0 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1368,7 +1368,7 @@ export const deleteLibraryItemsByUserId = async (userId: string) => { } export const batchDelete = async (criteria: FindOptionsWhere) => { - const batchSize = 1000 + const batchSize = 20 const qb = libraryItemRepository.createQueryBuilder().where(criteria) const countSql = queryBuilderToRawSql(qb.select('COUNT(1)')) diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 78310bf56..e5ff9bc97 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -67,6 +67,7 @@ const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files' const NO_CACHE_URLS = [ 'https://deviceandbrowserinfo.com/are_you_a_bot', 'https://deviceandbrowserinfo.com/info_device', + 'https://jacksonh.org', ] const signToken = promisify(jwt.sign) @@ -275,6 +276,7 @@ export const processFetchContentJob = async ( const isBlocked = await isDomainBlocked(redisDataSource, domain) if (isBlocked) { console.log('domain is blocked', domain) + logRecord.error = 'domain is blocked' return }