From 010e2841daf87ff863ebbb79e2e9ddc4b2de5e16 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 27 Sep 2023 10:38:49 +0800 Subject: [PATCH] fix getting 403 when parsing some rss url by setting a user agent in the request --- packages/api/src/resolvers/subscriptions/index.ts | 7 ++++++- packages/rss-handler/src/index.ts | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index a71133d99..226a497b6 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,4 +1,5 @@ import Parser from 'rss-parser' +import { Brackets } from 'typeorm' import { Subscription } from '../../entity/subscription' import { User } from '../../entity/user' import { getRepository } from '../../entity/utils' @@ -31,13 +32,17 @@ import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { enqueueRssFeedFetch } from '../../utils/createTask' import { authorized } from '../../utils/helpers' -import { Brackets } from 'typeorm' type PartialSubscription = Omit const parser = new Parser({ timeout: 30000, // 30 seconds maxRedirects: 5, + headers: { + // some rss feeds require user agent + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + }, }) export type SubscriptionsSuccessPartial = Merge< diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 074554bd3..d751c3f1c 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -118,9 +118,16 @@ Sentry.GCPFunction.init({ const signToken = promisify(jwt.sign) const parser = new Parser({ + timeout: 60000, // 60 seconds + maxRedirects: 10, customFields: { item: [['link', 'links', { keepArray: true }], 'published', 'updated'], }, + headers: { + // some rss feeds require user agent + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + }, }) // get link following the order of preference: via, alternate, self