From e22c014385865c18b962ae10f911c3308bdc287f Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 12 Dec 2022 11:13:34 +0800
Subject: [PATCH] For URLs that will be forced to backend fetch dont send full
 content from the extension

This is useful with YouTube, which has massive page sizes.
---
 pkg/extension/src/scripts/background.js            |  7 ++++++-
 pkg/extension/src/scripts/constants.js             | 14 ++++++++++++++
 .../src/scripts/content/prepare-content.js         |  7 +++++++
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/pkg/extension/src/scripts/background.js b/pkg/extension/src/scripts/background.js
index 30cc4d601..914399bdc 100644
--- a/pkg/extension/src/scripts/background.js
+++ b/pkg/extension/src/scripts/background.js
@@ -321,7 +321,12 @@ async function saveArticle (tab) {
     }
 
     const requestId = uuidv4()
-    const { type, pageInfo, doc, uploadContentObjUrl } = response;
+    var { type } = response;
+    const { pageInfo, doc, uploadContentObjUrl } = response;
+
+    if (type == 'html' && handleBackendUrl(tab.url)) {
+      type = 'url'
+    }
 
     switch(type) {
       case 'html': {
diff --git a/pkg/extension/src/scripts/constants.js b/pkg/extension/src/scripts/constants.js
index 4d169fdcf..d99b15948 100644
--- a/pkg/extension/src/scripts/constants.js
+++ b/pkg/extension/src/scripts/constants.js
@@ -98,3 +98,17 @@ window.CREATE_ARTICLE_SAVING_REQUEST_QUERY = `mutation CreateArticleSavingReques
     }
   }
 }`;
+
+function handleBackendUrl(url) {
+  try {
+    const FORCE_CONTENT_FETCH_URLS = [
+      // twitter status url regex
+      /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/,
+      /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/,
+    ]
+    return FORCE_CONTENT_FETCH_URLS.some((regex) => regex.test(url))
+    } catch (error) {
+    console.log('error checking url', url)
+  }
+  return false
+}
diff --git a/pkg/extension/src/scripts/content/prepare-content.js b/pkg/extension/src/scripts/content/prepare-content.js
index fd58f4970..0820bbd03 100644
--- a/pkg/extension/src/scripts/content/prepare-content.js
+++ b/pkg/extension/src/scripts/content/prepare-content.js
@@ -187,6 +187,13 @@
     if (pdfContent) {
       return pdfContent
     }
+    try {
+      if (handleBackendUrl(window.location.href)) {
+        return { type: 'url' }
+      }
+    } catch {
+      console.log('error checking url')
+    }
 
     async function scrollPage () {
       const scrollingEl = (document.scrollingElement || document.body);