From 36915ec1eca8ccf795d7c841b175aacee3773bed Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 3 Feb 2024 10:01:18 +0800 Subject: [PATCH 1/3] Add rss.arxiv and daringfireball to content fetch block lists --- packages/api/src/jobs/rss/refreshFeed.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index d111f248d..37b464e7e 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -126,6 +126,12 @@ export const isContentFetchBlocked = (feedUrl: string) => { if (feedUrl.startsWith('https://arxiv.org/')) { return true } + if (feedUrl.startsWith('https://rss.arxiv.org')) { + return true + } + if (feedUrl.startsWith('https://daringfireball.net/feeds/')) { + return true + } if (feedUrl.startsWith('https://lwn.net/headlines/newrss')) { return true } From 46affc0508ce11b514ffc10ebb874dda6cecd506 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 3 Feb 2024 10:02:25 +0800 Subject: [PATCH 2/3] Add xkcd --- packages/api/src/jobs/rss/refreshFeed.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 37b464e7e..7f7a8ac15 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -129,6 +129,9 @@ export const isContentFetchBlocked = (feedUrl: string) => { if (feedUrl.startsWith('https://rss.arxiv.org')) { return true } + if (feedUrl.startsWith('https://xkcd.com')) { + return true + } if (feedUrl.startsWith('https://daringfireball.net/feeds/')) { return true } From 7fffa8ce758cc32a6d3e56494b133c925fa17aff Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 3 Feb 2024 10:04:40 +0800 Subject: [PATCH 3/3] Add some logging --- packages/api/src/jobs/rss/refreshFeed.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 7f7a8ac15..6274920de 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -526,6 +526,9 @@ const processSubscription = async ( // Max limit per-feed update if (itemCount > 99) { + if (itemCount == 100) { + logger.info(`Max limit reached for feed ${feedUrl}`) + } continue }