From a00ce8893ec2d65ee6fa32a9bb3d66fd77efd15d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 6 Nov 2023 20:49:43 +0800
Subject: [PATCH] reduce feed parsing

---
 packages/rss-handler/src/index.ts | 48 +++++++++++++++++++------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts
index e5b9f68f2..dfa7f3294 100644
--- a/packages/rss-handler/src/index.ts
+++ b/packages/rss-handler/src/index.ts
@@ -247,7 +247,20 @@ const processSubscription = async (
   fetchResult: { content: string; checksum: string },
   lastFetchedAt: number,
   scheduledAt: number,
-  lastFetchedChecksum: string
+  lastFetchedChecksum: string,
+  feed: {
+    lastBuildDate: any
+    'syn:updatePeriod': any
+    'syn:updateFrequency': any
+    'sy:updatePeriod': any
+    'sy:updateFrequency': any
+  } & Parser.Output<{
+    published: any
+    updated: any
+    created: any
+    link: any
+    links: any[]
+  }>
 ) => {
   let lastItemFetchedAt: Date | null = null
   let lastValidItem: Item | null = null
@@ -260,8 +273,6 @@ const processSubscription = async (
 
   // fetch feed
   let itemCount = 0
-  const feed = await parser.parseString(fetchResult.content)
-  console.log('Fetched feed', feed.title, new Date())
 
   const feedLastBuildDate = feed.lastBuildDate as string | undefined
   console.log('Feed last build date', feedLastBuildDate)
@@ -392,24 +403,23 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
       console.log('Processing feed', feedUrl)
 
       const fetchResult = await fetchAndChecksum(feedUrl)
+      const feed = await parser.parseString(fetchResult.content)
+      console.log('Fetched feed', feed.title, new Date())
 
-      for (let i = 0; i < subscriptionIds.length; i++) {
-        const subscriptionId = subscriptionIds[i]
-        const lastFetchedAt = lastFetchedTimestamps[i]
-        const scheduledAt = scheduledTimestamps[i]
-        const userId = userIds[i]
-        const lastFetchedChecksum = lastFetchedChecksums[i]
-
-        await processSubscription(
-          subscriptionId,
-          userId,
-          feedUrl,
-          fetchResult,
-          lastFetchedAt,
-          scheduledAt,
-          lastFetchedChecksum
+      await Promise.all(
+        subscriptionIds.map((_, i) =>
+          processSubscription(
+            subscriptionIds[i],
+            userIds[i],
+            feedUrl,
+            fetchResult,
+            lastFetchedTimestamps[i],
+            scheduledTimestamps[i],
+            lastFetchedChecksums[i],
+            feed
+          )
         )
-      }
+      )
 
       res.send('ok')
     } catch (e) {