Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

do not cache some urls

Browse Source
This commit is contained in:
Hongbo Wu
2024-07-05 19:46:18 +08:00
parent 728059c6f8
commit dd01202374
1 changed files with 1 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
packages/content-fetch/src/request_handler.ts
View File

@ -64,11 +64,6 @@ const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files'
const NO_CACHE_URLS = ['https://deviceandbrowserinfo.com/are_you_a_bot']
const isUrlCacheable = (cacheKey: string) => {
const url = cacheKey.split(':')[1]
return !NO_CACHE_URLS.includes(url)
}
const uploadToBucket = async (filePath: string, data: string) => {
await storage
.bucket(bucketName)
@ -104,11 +99,6 @@ export const cacheFetchResult = async (
key: string,
fetchResult: FetchResult
) => {
if (!isUrlCacheable(key)) {
console.info('url is not cacheable', key)
return undefined
}
// cache the fetch result for 24 hours
const ttl = 24 * 60 * 60
const value = JSON.stringify(fetchResult)
@ -119,11 +109,6 @@ const getCachedFetchResult = async (
redisDataSource: RedisDataSource,
key: string
): Promise<FetchResult | undefined> => {
if (!isUrlCacheable(key)) {
console.info('url is not cacheable', key)
return undefined
}
const result = await redisDataSource.cacheClient.get(key)
if (!result) {
console.info('fetch result is not cached', key)
@ -215,7 +200,7 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
fetchResult = await fetchContent(url, locale, timezone)
console.log('content has been fetched')
if (fetchResult.content) {
if (fetchResult.content && !NO_CACHE_URLS.includes(url)) {
const cacheResult = await cacheFetchResult(
redisDataSource,
key,