From 5fdb8b337dc4b02f9d0c567da79e4ecae0448929 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 29 Sep 2022 16:08:38 +0800 Subject: [PATCH] Lint --- .../src/handlers/apple-news-handler.ts | 2 +- .../src/handlers/bloomberg-handler.ts | 2 +- .../src/handlers/derstandard-handler.ts | 2 +- .../src/handlers/image-handler.ts | 4 +- .../src/handlers/medium-handler.ts | 2 +- .../src/handlers/pdf-handler.ts | 2 +- .../src/handlers/scrapingBee-handler.ts | 2 +- .../src/handlers/t-dot-co-handler.ts | 2 +- .../src/handlers/twitter-handler.ts | 66 ++++++++++++------- .../src/handlers/youtube-handler.ts | 13 ++-- 10 files changed, 61 insertions(+), 36 deletions(-) diff --git a/packages/content-handler/src/handlers/apple-news-handler.ts b/packages/content-handler/src/handlers/apple-news-handler.ts index a53c8ef02..ab11bdc49 100644 --- a/packages/content-handler/src/handlers/apple-news-handler.ts +++ b/packages/content-handler/src/handlers/apple-news-handler.ts @@ -14,7 +14,7 @@ export class AppleNewsHandler extends ContentHandler { const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT }, }) - const data = response.data + const data = response.data as string const dom = parseHTML(data).document // make sure it's a valid URL by wrapping in new URL const href = dom diff --git a/packages/content-handler/src/handlers/bloomberg-handler.ts b/packages/content-handler/src/handlers/bloomberg-handler.ts index 05100429c..8052e36e0 100644 --- a/packages/content-handler/src/handlers/bloomberg-handler.ts +++ b/packages/content-handler/src/handlers/bloomberg-handler.ts @@ -5,7 +5,7 @@ import { parseHTML } from 'linkedom' class BloombergHandler extends ContentHandler { shouldPreHandle(url: string, _dom: Document): boolean { const BLOOMBERG_URL_MATCH = - /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ + /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/ return BLOOMBERG_URL_MATCH.test(url.toString()) } diff --git a/packages/content-handler/src/handlers/derstandard-handler.ts b/packages/content-handler/src/handlers/derstandard-handler.ts index fd77ce108..1aae94213 100644 --- a/packages/content-handler/src/handlers/derstandard-handler.ts +++ b/packages/content-handler/src/handlers/derstandard-handler.ts @@ -15,7 +15,7 @@ class DerstandardHandler extends ContentHandler { cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`, }, }) - const content = response.data + const content = response.data as string const dom = parseHTML(content).document const titleElement = dom.querySelector('.article-title') diff --git a/packages/content-handler/src/handlers/image-handler.ts b/packages/content-handler/src/handlers/image-handler.ts index 3f994332b..78447fe33 100644 --- a/packages/content-handler/src/handlers/image-handler.ts +++ b/packages/content-handler/src/handlers/image-handler.ts @@ -7,7 +7,7 @@ class ImageHandler extends ContentHandler { } async preHandle(url: string, _document: Document): Promise { - const title = url.toString().split('/').pop() + const title = url.toString().split('/').pop() || 'Image' const content = ` @@ -22,6 +22,6 @@ class ImageHandler extends ContentHandler { ` - return { title, content } + return Promise.resolve({ title, content }) } } diff --git a/packages/content-handler/src/handlers/medium-handler.ts b/packages/content-handler/src/handlers/medium-handler.ts index d87d4338f..03cc1be8f 100644 --- a/packages/content-handler/src/handlers/medium-handler.ts +++ b/packages/content-handler/src/handlers/medium-handler.ts @@ -12,7 +12,7 @@ class MediumHandler extends ContentHandler { try { const res = new URL(url) res.searchParams.delete('source') - return { url: res.toString() } + return Promise.resolve({ url: res.toString() }) } catch (error) { console.error('error prehandling medium url', error) throw error diff --git a/packages/content-handler/src/handlers/pdf-handler.ts b/packages/content-handler/src/handlers/pdf-handler.ts index 54c9e6bf4..fb9b7e045 100644 --- a/packages/content-handler/src/handlers/pdf-handler.ts +++ b/packages/content-handler/src/handlers/pdf-handler.ts @@ -8,6 +8,6 @@ class PdfHandler extends ContentHandler { } async preHandle(_url: string, _document: Document): Promise { - return { contentType: 'application/pdf' } + return Promise.resolve({ contentType: 'application/pdf' }) } } diff --git a/packages/content-handler/src/handlers/scrapingBee-handler.ts b/packages/content-handler/src/handlers/scrapingBee-handler.ts index 1c84ba06b..c9cbc8afa 100644 --- a/packages/content-handler/src/handlers/scrapingBee-handler.ts +++ b/packages/content-handler/src/handlers/scrapingBee-handler.ts @@ -24,7 +24,7 @@ class ScrapingBeeHandler extends ContentHandler { }, }) const dom = parseHTML(response.data).document - return { title: dom.title, content: response.data, url: url } + return { title: dom.title, content: response.data as string, url: url } } catch (error) { console.error('error prehandling url w/scrapingbee', error) throw error diff --git a/packages/content-handler/src/handlers/t-dot-co-handler.ts b/packages/content-handler/src/handlers/t-dot-co-handler.ts index e83443518..3a97849f3 100644 --- a/packages/content-handler/src/handlers/t-dot-co-handler.ts +++ b/packages/content-handler/src/handlers/t-dot-co-handler.ts @@ -8,7 +8,7 @@ class TDotCoHandler extends ContentHandler { } async resolve(url: string) { - return await axios + return axios .get(url, { maxRedirects: 0, validateStatus: null }) .then((res) => { return new URL(res.headers.location).href diff --git a/packages/content-handler/src/handlers/twitter-handler.ts b/packages/content-handler/src/handlers/twitter-handler.ts index 265bfe955..e4234dcbc 100644 --- a/packages/content-handler/src/handlers/twitter-handler.ts +++ b/packages/content-handler/src/handlers/twitter-handler.ts @@ -25,7 +25,11 @@ const getTweetById = async (id: string) => { const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/' const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) - return await axios.get(apiUrl.toString(), { + if (!TWITTER_BEARER_TOKEN) { + throw new Error('No Twitter bearer token found') + } + + return axios.get(apiUrl.toString(), { headers: { Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, redirect: 'follow', @@ -33,7 +37,7 @@ const getTweetById = async (id: string) => { }) } -const titleForAuthor = (author: any) => { +const titleForAuthor = (author: { name: string }) => { return `${author.name} on Twitter` } @@ -53,20 +57,6 @@ class TwitterHandler extends ContentHandler { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } - // version of the handler that uses the oembed API - // This isn't great as it doesn't work well with our - // readability API. But could potentially give a more consistent - // look to the tweets - // prehandle: async (url, env) => { - // const oeTweet = await embeddedTweet(url) - // const dom = new JSDOM(oeTweet.data.html); - // const bq = dom.window.document.querySelector('blockquote') - // console.log('blockquote:', bq); - - // const title = titleForTweet(oeTweet) - // return { title, content: '
' + bq.innerHTML + '
', url: oeTweet.data.url }; - // } - async preHandle(url: string, _document: Document): Promise { console.log('prehandling twitter url', url) @@ -74,18 +64,48 @@ class TwitterHandler extends ContentHandler { if (!tweetId) { throw new Error('could not find tweet id in url') } - const tweetData = (await getTweetById(tweetId)).data + const tweetData = (await getTweetById(tweetId)).data as { + data: { + author_id: string + text: string + entities: { + urls: [ + { + url: string + expanded_url: string + display_url: string + } + ] + } + created_at: string + } + includes: { + users: [ + { + id: string + name: string + profile_image_url: string + username: string + } + ] + media: [ + { + preview_image_url: string + type: string + url: string + } + ] + } + } const authorId = tweetData.data.author_id - const author = tweetData.includes.users.filter( - (u: any) => (u.id = authorId) - )[0] + const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0] // escape html entities in title const title = _.escape(titleForAuthor(author)) const authorImage = author.profile_image_url.replace('_normal', '_400x400') let text = tweetData.data.text if (tweetData.data.entities && tweetData.data.entities.urls) { - for (let urlObj of tweetData.data.entities.urls) { + for (const urlObj of tweetData.data.entities.urls) { text = text.replace( urlObj.url, `${urlObj.display_url}` @@ -98,10 +118,10 @@ class TwitterHandler extends ContentHandler {

${text}

` - var includesHtml = '' + let includesHtml = '' if (tweetData.includes.media) { includesHtml = tweetData.includes.media - .map((m: any) => { + .map((m) => { const linkUrl = m.type == 'photo' ? m.url : url const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url const mediaOpen = ` diff --git a/packages/content-handler/src/handlers/youtube-handler.ts b/packages/content-handler/src/handlers/youtube-handler.ts index 60ae22970..288f42aa0 100644 --- a/packages/content-handler/src/handlers/youtube-handler.ts +++ b/packages/content-handler/src/handlers/youtube-handler.ts @@ -1,9 +1,7 @@ -import { PreHandleResult } from '../index' +import { ContentHandler, PreHandleResult } from '../index' import axios from 'axios' import _ from 'underscore' -const { ContentHandler } = require('../index') - const YOUTUBE_URL_MATCH = /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/ @@ -34,7 +32,14 @@ class YoutubeHandler extends ContentHandler { const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`) - const oembed = (await axios.get(oembedUrl.toString())).data + const oembed = (await axios.get(oembedUrl.toString())).data as { + title: string + width: number + height: number + thumbnail_url: string + author_name: string + author_url: string + } // escape html entities in title const title = _.escape(oembed.title) const ratio = oembed.width / oembed.height