Merge pull request #3413 from omnivore-app/fix/thumbnailer

fix: catch images fetching error in thumbailer to prevent from retrying

packages/thumbnail-handler/package.json

@@ -29,8 +29,7 @@
     "dotenv": "^16.0.1",
     "image-size": "^1.0.2",
     "jsonwebtoken": "^8.5.1",
-    "linkedom": "^0.14.26",
-    "urlsafe-base64": "^1.0.0"
+    "linkedom": "^0.14.26"
   },
   "volta": {
     "extends": "../../package.json"

packages/thumbnail-handler/src/index.ts

@@ -1,11 +1,9 @@
 import * as Sentry from '@sentry/serverless'
 import axios, { AxiosResponse } from 'axios'
-import crypto from 'crypto'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
 import sizeOf from 'image-size'
 import * as jwt from 'jsonwebtoken'
 import { parseHTML } from 'linkedom'
-import { encode } from 'urlsafe-base64'
 import { promisify } from 'util'
 
 interface ArticleResponse {
@@ -49,28 +47,6 @@ Sentry.GCPFunction.init({
 const signToken = promisify(jwt.sign)
 const REQUEST_TIMEOUT = 30000 // 30s
 
-const signImageProxyUrl = (url: string, secret: string): string => {
-  return encode(crypto.createHmac('sha256', secret).update(url).digest())
-}
-
-export function createImageProxyUrl(
-  url: string,
-  width = 0,
-  height = 0
-): string {
-  if (!process.env.IMAGE_PROXY_URL || !process.env.IMAGE_PROXY_SECRET) {
-    return url
-  }
-
-  const urlWithOptions = `${url}#${width}x${height}`
-  const signature = signImageProxyUrl(
-    urlWithOptions,
-    process.env.IMAGE_PROXY_SECRET
-  )
-
-  return `${process.env.IMAGE_PROXY_URL}/${width}x${height},s${signature}/${url}`
-}
-
 const articleQuery = async (
   userId: string,
   slug: string
@@ -194,7 +170,7 @@ const fetchImage = async (url: string): Promise<AxiosResponse | null> => {
   console.log('fetching image', url)
   try {
     // get image file by url
-    return axios.get(url, {
+    return await axios.get(url, {
       responseType: 'arraybuffer',
       timeout: 10000, // 10s
       maxContentLength: 20000000, // 20mb
@@ -345,8 +321,7 @@ export const thumbnailHandler = Sentry.GCPFunction.wrapHttpFunction(
       if (page.image) {
         console.log('thumbnail already set')
         // pre-cache thumbnail first if exists
-        const imageProxyUrl = createImageProxyUrl(page.image, 320, 320)
-        const image = await fetchImage(imageProxyUrl)
+        const image = await fetchImage(page.image)
         if (!image) {
           console.log('thumbnail image not found')
           page.image = undefined

yarn.lock

@@ -10745,6 +10745,21 @@ bullmq@^5.1.1:
     tslib "^2.0.0"
     uuid "^9.0.0"
 
+bullmq@^5.1.4:
+  version "5.1.5"
+  resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-5.1.5.tgz#8cdd09f0e9e8b83321bc8f1af822b1b50c3ad8ad"
+  integrity sha512-Rc9QGHrj/wJ8RMENKa839o1pJmdicg7KBTfmVU8YqYuEK2JcMSJaKMg2XrAi7sdYSawgOJgC/kiW9fCGYEj6Yg==
+  dependencies:
+    cron-parser "^4.6.0"
+    glob "^8.0.3"
+    ioredis "^5.3.2"
+    lodash "^4.17.21"
+    msgpackr "^1.10.1"
+    node-abort-controller "^3.1.1"
+    semver "^7.5.4"
+    tslib "^2.0.0"
+    uuid "^9.0.0"
+
 busboy@1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893"