From c82efb31d3e3ee615898d102be5aee95b9d4e3f0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 13 Jun 2024 14:15:19 +0800 Subject: [PATCH] fetch old items for existing subscribed feeds --- packages/api/src/jobs/rss/refreshFeed.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 532c7de6f..dadd0908c 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -88,12 +88,20 @@ export const isOldItem = ( item: RssFeedItem, mostRecentItemTimestamp: number ) => { - // existing items and items that were published before 24h - const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() - return ( - publishedAt <= new Date(mostRecentItemTimestamp) || - publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) - ) + // always fetch items without isoDate + if (!item.isoDate) { + return false + } + + const publishedAt = new Date(item.isoDate) + + // don't fetch older than 24 hrs items for new feeds + if (!mostRecentItemTimestamp) { + return publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) + } + + // don't fetch existing items for old feeds + return publishedAt <= new Date(mostRecentItemTimestamp) } const feedFetchFailedRedisKey = (feedUrl: string) =>