From 991f984b6fa9bf174dbf3b5ccdcce8dcfe322917 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 15 Dec 2023 13:48:43 +0800 Subject: [PATCH] skip fetching re-published items in the feed to deduplicate --- packages/rss-handler/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index f80d99726..ff3cc8ca6 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -391,7 +391,7 @@ const processSubscription = async ( console.log('Feed last build date', feedLastBuildDate) if ( feedLastBuildDate && - new Date(feedLastBuildDate) < new Date(lastFetchedAt) + new Date(feedLastBuildDate) <= new Date(lastFetchedAt) ) { console.log('Skipping old feed', feedLastBuildDate) return @@ -433,7 +433,7 @@ const processSubscription = async ( // skip old items and items that were published before 24h if ( - publishedAt < new Date(lastFetchedAt) || + publishedAt <= new Date(lastFetchedAt) || publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) ) { console.log('Skipping old feed item', item.link)