From a0aab94e8bae38586525e441c98510b3aa4cb03a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 10 Mar 2023 16:42:40 +0800
Subject: [PATCH] Add site_name and published date to saved twitter

---
 packages/api/src/services/integrations.ts     |  3 +-
 .../src/websites/twitter-handler.ts           | 41 +++++++++++--------
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/packages/api/src/services/integrations.ts b/packages/api/src/services/integrations.ts
index a594f4bda..9cff4567b 100644
--- a/packages/api/src/services/integrations.ts
+++ b/packages/api/src/services/integrations.ts
@@ -65,6 +65,7 @@ const validateReadwiseToken = async (token: string): Promise<boolean> => {
 
 const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => {
   if (!page.highlights) return []
+  const category = page.siteName === 'Twitter' ? 'tweets' : 'articles'
   return (
     page.highlights
       // filter out highlights with no quote
@@ -76,7 +77,7 @@ const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => {
           author: page.author || undefined,
           highlight_url: getHighlightUrl(page.slug, highlight.id),
           highlighted_at: new Date(highlight.createdAt).toISOString(),
-          category: 'articles',
+          category,
           image_url: page.image || undefined,
           // location: highlight.highlightPositionAnchorIndex || undefined,
           location_type: 'order',
diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index 121e380a4..1116d5b7f 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -228,23 +228,8 @@ const getTweetIds = async (
 
       const ids: Set<string> = new Set()
 
-      // Find the first Show thread button and click it
-      const showRepliesButton = Array.from(
-        document.querySelectorAll('div[dir="auto"]')
-      )
-        .filter(
-          (node) => node.children[0] && node.children[0].tagName === 'SPAN'
-        )
-        .find((node) => node.children[0].innerHTML === 'Show replies')
-
-      if (showRepliesButton) {
-        ;(showRepliesButton as HTMLElement).click()
-
-        await waitFor(2000)
-      }
-
       const distance = 1080
-      const scrollHeight = document.body.scrollHeight
+      let scrollHeight = document.body.scrollHeight
       let currentHeight = 0
       // keep scrolling until there are no more elements
       while (currentHeight < scrollHeight) {
@@ -269,13 +254,31 @@ const getTweetIds = async (
           const id = match[2]
           const username = match[1]
 
-          // skip non-author replies
-          username === author && ids.add(id)
+          // stop at non-author replies
+          if (username !== author) return Array.from(ids)
+          ids.add(id)
         }
 
         window.scrollBy(0, distance)
         await waitFor(500)
         currentHeight += distance
+
+        // Find the show replies button and click it
+        if (currentHeight >= scrollHeight) {
+          const showRepliesButton = Array.from(
+            document.querySelectorAll('div[dir]')
+          )
+            .filter(
+              (node) => node.children[0] && node.children[0].tagName === 'SPAN'
+            )
+            .find((node) => node.children[0].innerHTML === 'Show replies')
+
+          if (showRepliesButton) {
+            ;(showRepliesButton as HTMLElement).click()
+            await waitFor(1000)
+            scrollHeight = document.body.scrollHeight
+          }
+        }
       }
 
       return Array.from(ids)
@@ -371,6 +374,8 @@ export class TwitterHandler extends ContentHandler {
       <meta property="og:image:secure_url" content="${authorImage}" />
       <meta property="og:title" content="${escapedTitle}" />
       <meta property="og:description" content="${description}" />
+      <meta property="article:published_time" content="${tweetData.created_at}" />
+      <meta property="og:site_name" content="Twitter" />
     </head>
     <body>
       <div>