From dd0120237442b3cbf4d2cbe4b3b244be9b7813d4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 5 Jul 2024 19:46:18 +0800 Subject: [PATCH] do not cache some urls --- packages/content-fetch/src/request_handler.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 269130634..bbb98f8d2 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -64,11 +64,6 @@ const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files' const NO_CACHE_URLS = ['https://deviceandbrowserinfo.com/are_you_a_bot'] -const isUrlCacheable = (cacheKey: string) => { - const url = cacheKey.split(':')[1] - return !NO_CACHE_URLS.includes(url) -} - const uploadToBucket = async (filePath: string, data: string) => { await storage .bucket(bucketName) @@ -104,11 +99,6 @@ export const cacheFetchResult = async ( key: string, fetchResult: FetchResult ) => { - if (!isUrlCacheable(key)) { - console.info('url is not cacheable', key) - return undefined - } - // cache the fetch result for 24 hours const ttl = 24 * 60 * 60 const value = JSON.stringify(fetchResult) @@ -119,11 +109,6 @@ const getCachedFetchResult = async ( redisDataSource: RedisDataSource, key: string ): Promise => { - if (!isUrlCacheable(key)) { - console.info('url is not cacheable', key) - return undefined - } - const result = await redisDataSource.cacheClient.get(key) if (!result) { console.info('fetch result is not cached', key) @@ -215,7 +200,7 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => { fetchResult = await fetchContent(url, locale, timezone) console.log('content has been fetched') - if (fetchResult.content) { + if (fetchResult.content && !NO_CACHE_URLS.includes(url)) { const cacheResult = await cacheFetchResult( redisDataSource, key,