Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

do not cache content from https://jacksonh.org

Browse Source
This commit is contained in:
Hongbo Wu
2024-08-21 18:47:23 +08:00
parent ae9c6cc8b2
commit aa2caca944
2 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/api/src/services/library_item.ts
View File

@ -1368,7 +1368,7 @@ export const deleteLibraryItemsByUserId = async (userId: string) => {
} }
export const batchDelete = async (criteria: FindOptionsWhere<LibraryItem>) => { export const batchDelete = async (criteria: FindOptionsWhere<LibraryItem>) => {
const batchSize = 1000 const batchSize = 20
const qb = libraryItemRepository.createQueryBuilder().where(criteria) const qb = libraryItemRepository.createQueryBuilder().where(criteria)
const countSql = queryBuilderToRawSql(qb.select('COUNT(1)')) const countSql = queryBuilderToRawSql(qb.select('COUNT(1)'))

2
packages/content-fetch/src/request_handler.ts
View File

@ -67,6 +67,7 @@ const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files'
const NO_CACHE_URLS = [ const NO_CACHE_URLS = [
'https://deviceandbrowserinfo.com/are_you_a_bot', 'https://deviceandbrowserinfo.com/are_you_a_bot',
'https://deviceandbrowserinfo.com/info_device', 'https://deviceandbrowserinfo.com/info_device',
'https://jacksonh.org',
] ]
const signToken = promisify(jwt.sign) const signToken = promisify(jwt.sign)
@ -275,6 +276,7 @@ export const processFetchContentJob = async (
const isBlocked = await isDomainBlocked(redisDataSource, domain) const isBlocked = await isDomainBlocked(redisDataSource, domain)
if (isBlocked) { if (isBlocked) {
console.log('domain is blocked', domain) console.log('domain is blocked', domain)
logRecord.error = 'domain is blocked'
return return
} }