From 6531dd30e9fcd372c7de3c367d17c2b291ebddba Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 14 Jul 2023 12:08:08 +0800 Subject: [PATCH] skip rss items which are published before 48 hours --- packages/rss-handler/src/index.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 99c3555aa..1ee659a8a 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -114,15 +114,20 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( // save each item in the feed for await (const item of feed.items) { - const publishedAt = item.pubDate || item.isoDate - console.log('Processing feed item', item.link, publishedAt) + const publishedAtString = item.pubDate || item.isoDate + console.log('Processing feed item', item.link, publishedAtString) - if (!item.link || !publishedAt) { + if (!item.link || !publishedAtString) { console.log('Invalid feed item', item) continue } - if (new Date(publishedAt) <= new Date(lastFetchedAt)) { + // skip old items and items that were published before 48h + const publishedAt = new Date(publishedAtString) + if ( + publishedAt < new Date(Date.now() - 48 * 60 * 60 * 1000) || + publishedAt < new Date(lastFetchedAt) + ) { console.log('Skipping old feed item', item.link) continue }