From 5007b11a3caf1ceb81cd72e412a1e5ee5a8c13fa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:52:10 +0800 Subject: [PATCH 01/22] Add url to readability option --- packages/api/src/readability.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 54f8846ec..33fbe0578 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -121,6 +121,7 @@ declare module '@omnivore/readability' { */ keepClasses?: boolean + url?: string /** * Function that converts a regular image url into imageproxy url From 56987902888fc7219dc0a1af05c471560794f0b6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:53:45 +0800 Subject: [PATCH 02/22] Pass url to readability --- packages/api/src/utils/parser.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index cf4d21408..8ddfb9f2e 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -171,6 +171,7 @@ const getReadabilityResult = ( debug: DEBUG_MODE, createImageProxyUrl, keepTables: isNewsletter, + url, }).parse() if (article) { From cc4803414516f78586288ed444642c2b240fe047 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:54:14 +0800 Subject: [PATCH 03/22] Add linkedom to dependencies --- packages/readabilityjs/package.json | 1 + yarn.lock | 79 ++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index ce69e03a7..ebf325df2 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "html-entities": "^2.3.2", + "linkedom": "^0.14.9", "modern-random-ua": "^1.0.3", "parse-srcset": "^1.0.2" } diff --git a/yarn.lock b/yarn.lock index 85d5e6e7b..c749578e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11639,7 +11639,18 @@ css-select@^4.1.3: domutils "^2.8.0" nth-check "^2.0.1" -css-what@^6.0.1: +css-select@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6" + integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + +css-what@^6.0.1, css-what@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== @@ -12289,6 +12300,15 @@ dom-serializer@^1.0.1: domhandler "^4.2.0" entities "^2.0.0" +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + dom-walk@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84" @@ -12304,6 +12324,11 @@ domelementtype@^2.0.1, domelementtype@^2.2.0: resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57" integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A== +domelementtype@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" + integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== + domexception@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" @@ -12339,6 +12364,13 @@ domhandler@^4.3.1: dependencies: domelementtype "^2.2.0" +domhandler@^5.0.1, domhandler@^5.0.2: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + dompurify@^2.0.17: version "2.3.1" resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.1.tgz#a47059ca21fd1212d3c8f71fdea6943b8bfbdf6a" @@ -12362,6 +12394,15 @@ domutils@^2.8.0: domelementtype "^2.2.0" domhandler "^4.2.0" +domutils@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.0.1.tgz#696b3875238338cb186b6c0612bd4901c89a4f1c" + integrity sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.1" + dot-case@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-2.1.1.tgz#34dcf37f50a8e93c2b3bca8bb7fb9155c7da3bee" @@ -12640,6 +12681,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@^4.2.0, entities@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.0.tgz#62915f08d67353bb4eb67e3d62641a4059aec656" + integrity sha512-/iP1rZrSEJ0DTlPiX+jbzlA3eVkY/e8L8SozroF395fIqE3TYF/Nz7YOMAawta+vLmyJ/hkGNNPcSbMADCCXbg== + env-paths@^2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" @@ -15014,6 +15060,11 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== +html-escaper@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-3.0.3.tgz#4d336674652beb1dcbc29ef6b6ba7f6be6fdfed6" + integrity sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ== + html-minifier-terser@^5.0.1: version "5.1.1" resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz#922e96f1f3bb60832c2634b79884096389b1f054" @@ -15101,6 +15152,16 @@ htmlparser2@^6.0.0, htmlparser2@^6.1.0: domutils "^2.5.2" entities "^2.0.0" +htmlparser2@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.1.tgz#abaa985474fcefe269bc761a779b544d7196d010" + integrity sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + domutils "^3.0.1" + entities "^4.3.0" + htmltidy2@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/htmltidy2/-/htmltidy2-0.3.0.tgz#1edfb74b8cd530cdcdc29ef547c849a651f0870b" @@ -17391,6 +17452,17 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= +linkedom@^0.14.9: + version "0.14.9" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0" + integrity sha512-ZV4H69VFzOwKp7akxsMtrzcnlP7mlFBvKy1RBsyIccuGX7ewkFlt/1FFfTHSg/BvREXNFFuyZlWoSf48FYAMzA== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^8.0.1" + uhyphen "^0.1.0" + listr-silent-renderer@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e" @@ -23999,6 +24071,11 @@ uglify-js@^3.1.4: resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.14.1.tgz#e2cb9fe34db9cb4cf7e35d1d26dfea28e09a7d06" integrity sha512-JhS3hmcVaXlp/xSo3PKY5R0JqKs5M3IV+exdLHW99qKvKivPO4Z8qbej6mte17SOPqAOVMjt/XGgWacnFSzM3g== +uhyphen@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.1.0.tgz#3cc22afa790daa802b9f6789f3583108d5b4a08c" + integrity sha512-o0QVGuFg24FK765Qdd5kk0zU/U4dEsCtN/GSiwNI9i8xsSVtjIAOdTaVhLwZ1nrbWxFVMxNDDl+9fednsOMsBw== + uid-number@0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" From 79a941a2b6d38e5a69e956f9e19395793d987c4b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:56:09 +0800 Subject: [PATCH 04/22] Default use options.url if exists --- packages/readabilityjs/Readability.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index d1bce8bdb..8f4d8f6e5 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -99,6 +99,8 @@ function Readability(doc, options) { return el.innerHTML; }; this._disableJSONLD = !!options.disableJSONLD; + this._baseURI = options.url || this._doc.baseURI; + this._documentURI = options.url || this._doc.documentURI; // Start with all flags set this._flags = this.FLAG_STRIP_UNLIKELYS | @@ -435,8 +437,8 @@ Readability.prototype = { }, toAbsoluteURI: function (uri) { - var baseURI = this._doc.baseURI; - var documentURI = this._doc.documentURI; + var baseURI = this._baseURI; + var documentURI = this._documentURI; // Leave hash links alone if the base URI matches the document URI: if (baseURI === documentURI && uri.charAt(0) === "#") { @@ -1827,7 +1829,7 @@ Readability.prototype = { } try { // allow relative URLs - new URL(content.trim(), new URL(this._doc.baseURI).origin); + new URL(content.trim(), new URL(this._baseURI).origin); } catch (error) { return; } @@ -1932,7 +1934,7 @@ Readability.prototype = { if (metadata.previewImage) { // convert any relative URL path to absolute URL try { - metadata.previewImage = new URL(metadata.previewImage, new URL(this._doc.baseURI).origin).href; + metadata.previewImage = new URL(metadata.previewImage, new URL(this._baseURI).origin).href; } catch { delete metadata.previewImage; } From 76d47f7dc552e25b2117c8c3f80ce77db246ccf5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:57:02 +0800 Subject: [PATCH 05/22] Fix updating live collections --- packages/readabilityjs/Readability.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 8f4d8f6e5..0231e82f3 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -476,8 +476,8 @@ Readability.prototype = { } else { // if the link has multiple children, they should all be preserved var container = this._doc.createElement("span"); - while (link.childNodes.length > 0) { - container.appendChild(link.childNodes[0]); + while (link.firstChild) { + container.appendChild(link.firstChild); } link.parentNode.replaceChild(container, link); } @@ -1351,10 +1351,9 @@ Readability.prototype = { neededToCreateTopCandidate = true; // Move everything (not just elements, also text nodes etc.) into the container // so we even include text directly in the body: - var kids = page.childNodes; - while (kids.length) { - this.log("Moving child out:", kids[0]); - topCandidate.appendChild(kids[0]); + while (page.firstChild) { + this.log("Moving child out:", page.firstChild); + topCandidate.appendChild(page.firstChild); } page.appendChild(topCandidate); @@ -1496,6 +1495,9 @@ Readability.prototype = { } articleContent.appendChild(sibling); + // Fetch children again to make it compatible + // with DOM parsers without live collection support. + siblings = parentOfTopCandidate.children; // siblings is a reference to the children array, and // sibling is removed from the array when we call appendChild(). // As a result, we must revisit this index since the nodes @@ -1542,9 +1544,8 @@ Readability.prototype = { var div = doc.createElement("DIV"); div.id = "readability-page-1"; div.className = "page"; - var children = articleContent.childNodes; - while (children.length) { - div.appendChild(children[0]); + while (articleContent.firstChild) { + div.appendChild(articleContent.firstChild); } articleContent.appendChild(div); } From 2152a9e466f3067b89ede80ac6f124ece7a67b5a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:57:38 +0800 Subject: [PATCH 06/22] Fix getting embeded class lists bug --- packages/readabilityjs/Readability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 0231e82f3..efcb6bb8b 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -2288,7 +2288,7 @@ Readability.prototype = { return false; } - const classes = this.EMBEDS_CLASSES.reduce((res, cur) => `${res},.${cur}`, ''); + const classes = this.EMBEDS_CLASSES.reduce((res, cur, i) => `${i > 0 && (res + ',')}.${cur}`, ''); const candidates = element.querySelector(classes); return !!candidates; From acc7654a2f624e4670886efa1a21360c00c292d8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 16:59:09 +0800 Subject: [PATCH 07/22] Replace jsdom with linkedom --- packages/api/src/utils/parser.ts | 45 ++++++++++++++------------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 8ddfb9f2e..ef4bda08a 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -15,10 +15,11 @@ import { BloombergHandler } from './bloomberg-handler' import { GolangHandler } from './golang-handler' import * as hljs from 'highlightjs' import { decode } from 'html-entities' +import { parseHTML } from 'linkedom' const logger = buildLogger('utils.parse') -const virtualConsole = new VirtualConsole() +// const virtualConsole = new VirtualConsole() export const ALLOWED_CONTENT_TYPES = [ 'text/html', @@ -102,9 +103,9 @@ type ArticleParseLogRecord = LogRecord & { const DEBUG_MODE = process.env.DEBUG === 'true' || false -const parseOriginalContent = (window: DOMWindow): PageType => { +const parseOriginalContent = (document: Document): PageType => { try { - const e = window.document.querySelector("head meta[property='og:type']") + const e = document.querySelector("head meta[property='og:type']") const content = e?.getAttribute('content') if (!content) { return PageType.Unknown @@ -138,22 +139,14 @@ const getPurifiedContent = (html: string): Document => { const getReadabilityResult = ( url: string, html: string, - window: DOMWindow, + document: Document, isNewsletter?: boolean ): Readability.ParseResult | null => { - virtualConsole.removeAllListeners('jsdomError') - virtualConsole.on('jsdomError', ({ message, stack: _stack, ...details }) => { - logger.warning(`JSDOM error occurred`, { - errorMsg: message, - ...details, - }) - }) - // First attempt to read the article as is. // if that fails attempt to purify then read const sources = [ () => { - return window.document + return document }, () => { return getPurifiedContent(html) @@ -237,20 +230,20 @@ export const parsePreparedContent = async ( } } - virtualConsole.removeAllListeners('jsdomError') - virtualConsole.on('jsdomError', ({ message, stack: _stack, ...details }) => { - logger.warning(`JSDOM error occurred`, { - ...logRecord, - errorMsg: message, - ...details, - }) - }) - const { window } = new JSDOM(document, { url, virtualConsole }) + // virtualConsole.removeAllListeners('jsdomError') + // virtualConsole.on('jsdomError', ({ message, stack: _stack, ...details }) => { + // logger.warning(`JSDOM error occurred`, { + // ...logRecord, + // errorMsg: message, + // ...details, + // }) + // }) + const { document: doc } = parseHTML(document) await applyHandlers(url, window) try { - article = getReadabilityResult(url, document, window, isNewsletter) + article = getReadabilityResult(url, document, doc, isNewsletter) // Format code blocks // TODO: we probably want to move this type of thing @@ -283,7 +276,7 @@ export const parsePreparedContent = async ( const clean = DOMPurify.sanitize(article?.content || '', DOM_PURIFY_CONFIG) const jsonLdLinkMetadata = (async () => { - return getJSONLdLinkMetadata(window.document) + return getJSONLdLinkMetadata(doc) })() Object.assign(article, { @@ -316,7 +309,7 @@ export const parsePreparedContent = async ( domContent: preparedDocument.document, parsedContent: article, canonicalUrl, - pageType: parseOriginalContent(window), + pageType: parseOriginalContent(doc), } } @@ -363,7 +356,7 @@ type Metadata = { export const parsePageMetadata = (html: string): Metadata | undefined => { try { - const window = new JSDOM(html).window + const window = parseHTML(html).window // get open graph metadata const description = From 6a57281e740cbe1f53f29e7498b2819af29844db Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 17:00:56 +0800 Subject: [PATCH 08/22] Remove DomWindow usage --- packages/api/src/utils/parser.ts | 44 +++++++++++++++----------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index ef4bda08a..d5dfc19d0 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -129,11 +129,11 @@ const parseOriginalContent = (document: Document): PageType => { } const getPurifiedContent = (html: string): Document => { - const newWindow = new JSDOM('').window - const DOMPurify = createDOMPurify(newWindow as unknown as Window) + const newWindow = parseHTML('') + const DOMPurify = createDOMPurify(newWindow) DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) const clean = DOMPurify.sanitize(html, DOM_PURIFY_CONFIG) - return new JSDOM(clean).window.document + return parseHTML(clean).document } const getReadabilityResult = ( @@ -270,8 +270,8 @@ export const parsePreparedContent = async ( } } - const newWindow = new JSDOM('').window - const DOMPurify = createDOMPurify(newWindow as unknown as Window) + const newWindow = parseHTML('') + const DOMPurify = createDOMPurify(newWindow) DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) const clean = DOMPurify.sanitize(article?.content || '', DOM_PURIFY_CONFIG) @@ -406,9 +406,9 @@ export const parseUrlMetadata = async ( // TODO: when we consolidate the handlers we could include this // as a utility method on each one. export const isProbablyNewsletter = (html: string): boolean => { - const dom = new JSDOM(html).window - const domCopy = new JSDOM(dom.document.documentElement.outerHTML) - const article = new Readability(domCopy.window.document, { + const dom = parseHTML(html).document + const domCopy = parseHTML(dom.documentElement.outerHTML) + const article = new Readability(domCopy.document, { debug: false, keepTables: true, }).parse() @@ -418,16 +418,16 @@ export const isProbablyNewsletter = (html: string): boolean => { } // substack newsletter emails have tables with a *post-meta class - if (dom.document.querySelector('table[class$="post-meta"]')) { + if (dom.querySelector('table[class$="post-meta"]')) { return true } // If the article has a header link, and substack icons its probably a newsletter - const href = findNewsletterHeaderHref(dom.window) - const heartIcon = dom.document.querySelector( + const href = findNewsletterHeaderHref(dom) + const heartIcon = dom.querySelector( 'table tbody td span a img[src*="HeartIcon"]' ) - const recommendIcon = dom.document.querySelector( + const recommendIcon = dom.querySelector( 'table tbody td span a img[src*="RecommendIconRounded"]' ) if (href && (heartIcon || recommendIcon)) { @@ -435,8 +435,8 @@ export const isProbablyNewsletter = (html: string): boolean => { } // Check if this is a beehiiv.net newsletter - if (dom.document.querySelectorAll('img[src*="beehiiv.net"]').length > 0) { - const beehiivUrl = beehiivNewsletterHref(dom.window) + if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) { + const beehiivUrl = beehiivNewsletterHref(dom) if (beehiivUrl) { return true } @@ -445,10 +445,8 @@ export const isProbablyNewsletter = (html: string): boolean => { return false } -const beehiivNewsletterHref = (dom: DOMWindow): string | undefined => { - const readOnline = dom.document.querySelectorAll( - 'table tr td div a[class*="link"]' - ) +const beehiivNewsletterHref = (dom: Document): string | undefined => { + const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]') let res: string | undefined = undefined readOnline.forEach((e) => { if (e.textContent === 'Read Online') { @@ -458,15 +456,15 @@ const beehiivNewsletterHref = (dom: DOMWindow): string | undefined => { return res } -const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => { +const findNewsletterHeaderHref = (dom: Document): string | undefined => { // Substack header links - const postLink = dom.document.querySelector('h1 a ') + const postLink = dom.querySelector('h1 a ') if (postLink) { return postLink.getAttribute('href') || undefined } // Check if this is a beehiiv.net newsletter - const beehiiv = beehiivNewsletterHref(dom.window) + const beehiiv = beehiivNewsletterHref(dom) if (beehiiv) { return beehiiv } @@ -479,10 +477,10 @@ const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => { export const findNewsletterUrl = async ( html: string ): Promise => { - const dom = new JSDOM(html).window + const dom = parseHTML(html).document // Check if this is a substack newsletter - const href = findNewsletterHeaderHref(dom.window) + const href = findNewsletterHeaderHref(dom) if (href) { // Try to make a HEAD request so we get the redirected URL, since these // will usually be behind tracking url redirects From a78a6c6ba48d0460b5c6cf7eb0f0ab1a4be58aae Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 17:01:23 +0800 Subject: [PATCH 09/22] Replace DomWindow with Document in handlers --- packages/api/src/utils/axios-handler.ts | 10 ++++------ packages/api/src/utils/bloomberg-handler.ts | 14 +++++--------- packages/api/src/utils/golang-handler.ts | 8 +++----- packages/api/src/utils/parser.ts | 16 +++++++++------- packages/api/src/utils/substack-handler.ts | 10 ++++------ packages/api/src/utils/wikipedia-handler.ts | 10 ++++------ 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/packages/api/src/utils/axios-handler.ts b/packages/api/src/utils/axios-handler.ts index 6e44e1e90..868181c85 100644 --- a/packages/api/src/utils/axios-handler.ts +++ b/packages/api/src/utils/axios-handler.ts @@ -1,17 +1,15 @@ -import { DOMWindow } from 'jsdom' - export class AxiosHandler { name = 'axios' // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, _dom: Document): boolean => { const host = this.name + '.com' // check if url ends with axios.com return url.hostname.endsWith(host) } - prehandle = (url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('table') + prehandle = (url: URL, dom: Document): Promise => { + const body = dom.querySelector('table') // this removes ads and replaces table with a div body?.querySelectorAll('table').forEach((el, k) => { @@ -27,7 +25,7 @@ export class AxiosHandler { } }) // replace the table with a div - const div = dom.document.createElement('div') + const div = dom.createElement('div') div.innerHTML = el.innerHTML el.parentNode?.replaceChild(div, el) } diff --git a/packages/api/src/utils/bloomberg-handler.ts b/packages/api/src/utils/bloomberg-handler.ts index 2fbae0ef5..c03af3f4b 100644 --- a/packages/api/src/utils/bloomberg-handler.ts +++ b/packages/api/src/utils/bloomberg-handler.ts @@ -1,22 +1,18 @@ -import { DOMWindow } from 'jsdom' - export class BloombergHandler { name = 'bloomberg' - shouldPrehandle = (url: URL, dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, dom: Document): boolean => { const host = this.name + '.com' // check if url ends with bloomberg.com return ( url.hostname.endsWith(host) || - dom.document - .querySelector('.logo-image') - ?.getAttribute('alt') - ?.toLowerCase() === this.name + dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() === + this.name ) } - prehandle = (_url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('.wrapper') + prehandle = (_url: URL, dom: Document): Promise => { + const body = dom.querySelector('.wrapper') // this removes header body?.querySelector('.sailthru-variables')?.remove() diff --git a/packages/api/src/utils/golang-handler.ts b/packages/api/src/utils/golang-handler.ts index 3a3037a59..6e6e29d02 100644 --- a/packages/api/src/utils/golang-handler.ts +++ b/packages/api/src/utils/golang-handler.ts @@ -1,17 +1,15 @@ -import { DOMWindow } from 'jsdom' - export class GolangHandler { name = 'golangweekly' // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, _dom: Document): boolean => { const host = this.name + '.com' // check if url ends with golangweekly.com return url.hostname.endsWith(host) } - prehandle = (url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('body') + prehandle = (url: URL, dom: Document): Promise => { + const body = dom.querySelector('body') // this removes the "Subscribe" button body?.querySelector('.el-splitbar')?.remove() diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index d5dfc19d0..878f801b1 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Readability } from '@omnivore/readability' -import { DOMWindow, JSDOM, VirtualConsole } from 'jsdom' import createDOMPurify, { SanitizeElementHookEvent } from 'dompurify' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { buildLogger, LogRecord } from './logger' @@ -42,8 +41,8 @@ const DOM_PURIFY_CONFIG = { } interface ContentHandler { - shouldPrehandle: (url: URL, dom: DOMWindow) => boolean - prehandle: (url: URL, document: DOMWindow) => Promise + shouldPrehandle: (url: URL, dom: Document) => boolean + prehandle: (url: URL, document: Document) => Promise } const HANDLERS = [ @@ -178,12 +177,15 @@ const getReadabilityResult = ( return null } -const applyHandlers = async (url: string, window: DOMWindow): Promise => { +const applyHandlers = async ( + url: string, + document: Document +): Promise => { try { const u = new URL(url) const handler = HANDLERS.find((h) => { try { - return h.shouldPrehandle(u, window) + return h.shouldPrehandle(u, document) } catch (e) { console.log('error with handler: ', h.name, e) } @@ -192,7 +194,7 @@ const applyHandlers = async (url: string, window: DOMWindow): Promise => { if (handler) { try { console.log('pre-handling url or content with handler: ', handler.name) - await handler.prehandle(u, window) + await handler.prehandle(u, document) } catch (e) { console.log('error with handler: ', handler, e) } @@ -240,7 +242,7 @@ export const parsePreparedContent = async ( // }) const { document: doc } = parseHTML(document) - await applyHandlers(url, window) + await applyHandlers(url, doc) try { article = getReadabilityResult(url, document, doc, isNewsletter) diff --git a/packages/api/src/utils/substack-handler.ts b/packages/api/src/utils/substack-handler.ts index 0f8f2176b..ea6b69bc3 100644 --- a/packages/api/src/utils/substack-handler.ts +++ b/packages/api/src/utils/substack-handler.ts @@ -1,23 +1,21 @@ -import { DOMWindow } from 'jsdom' - export class SubstackHandler { name = 'substack' - shouldPrehandle = (url: URL, dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, dom: Document): boolean => { const host = this.name + '.com' // check if url ends with substack.com // or has a profile image hosted at substack.com return ( url.hostname.endsWith(host) || - !!dom.document + !!dom .querySelector('.email-body img') ?.getAttribute('src') ?.includes(host) ) } - prehandle = (url: URL, dom: DOMWindow): Promise => { - const body = dom.document.querySelector('.email-body-container') + prehandle = (url: URL, dom: Document): Promise => { + const body = dom.querySelector('.email-body-container') // this removes header and profile avatar body?.querySelector('.header')?.remove() diff --git a/packages/api/src/utils/wikipedia-handler.ts b/packages/api/src/utils/wikipedia-handler.ts index 05fc4b5d4..ce30517c9 100644 --- a/packages/api/src/utils/wikipedia-handler.ts +++ b/packages/api/src/utils/wikipedia-handler.ts @@ -1,18 +1,16 @@ -import { DOMWindow } from 'jsdom' - export class WikipediaHandler { name = 'wikipedia' // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + shouldPrehandle = (url: URL, _dom: Document): boolean => { return url.hostname.endsWith('wikipedia.org') } - prehandle = (url: URL, dom: DOMWindow): Promise => { + prehandle = (url: URL, dom: Document): Promise => { // This removes the [edit] anchors from wikipedia pages - dom.document.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) + dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) // this removes the sidebar - dom.document.querySelector('.infobox')?.remove() + dom.querySelector('.infobox')?.remove() return Promise.resolve(dom) } } From 93ed1936f0f998d7e3b997bb0cf0475e56a8217c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 17:12:52 +0800 Subject: [PATCH 10/22] Remove jsdom in packages/api --- packages/api/package.json | 2 -- packages/api/test/utils/search.test.ts | 2 -- yarn.lock | 14 -------------- 3 files changed, 18 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 657a9342b..550c42ed6 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -58,7 +58,6 @@ "highlightjs": "^9.16.2", "html-entities": "^2.3.2", "intercom-client": "^3.1.4", - "jsdom": "^19.0.0", "jsonwebtoken": "^8.5.1", "jwks-rsa": "^2.0.3", "knex": "0.21.12", @@ -96,7 +95,6 @@ "@types/express": "^4.17.7", "@types/highlightjs": "^9.12.2", "@types/intercom-client": "^2.11.8", - "@types/jsdom": "^16.2.3", "@types/jsonwebtoken": "^8.5.0", "@types/luxon": "^1.25.0", "@types/mocha": "^8.2.2", diff --git a/packages/api/test/utils/search.test.ts b/packages/api/test/utils/search.test.ts index 2b7183d7e..a8b505f8c 100644 --- a/packages/api/test/utils/search.test.ts +++ b/packages/api/test/utils/search.test.ts @@ -1,7 +1,5 @@ import 'mocha' -import * as chai from 'chai' import { expect } from 'chai' -import { JSDOM } from 'jsdom' import 'chai/register-should' import { InFilter, parseSearchQuery, ReadFilter } from '../../src/utils/search' import { PageType } from '../../src/generated/graphql' diff --git a/yarn.lock b/yarn.lock index c749578e0..6d4342b9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7749,15 +7749,6 @@ resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.5.tgz#738dd390a6ecc5442f35e7f03fa1431353f7e138" integrity sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA== -"@types/jsdom@^16.2.3": - version "16.2.14" - resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.14.tgz#26fe9da6a8870715b154bb84cd3b2e53433d8720" - integrity sha512-6BAy1xXEmMuHeAJ4Fv4yXKwBDTGTOseExKE3OaHiNycdHdZw59KfYzrt0DkDluvwmik1HRt6QS7bImxUmpSy+w== - dependencies: - "@types/node" "*" - "@types/parse5" "*" - "@types/tough-cookie" "*" - "@types/json-bigint@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/json-bigint/-/json-bigint-1.0.1.tgz#201062a6990119a8cc18023cfe1fed12fc2fc8a7" @@ -7932,11 +7923,6 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== -"@types/parse5@*": - version "6.0.1" - resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.1.tgz#f8ae4fbcd2b9ba4ff934698e28778961f9cb22ca" - integrity sha512-ARATsLdrGPUnaBvxLhUlnltcMgn7pQG312S8ccdYlnyijabrX9RN/KN/iGj9Am96CoW8e/K9628BA7Bv4XHdrA== - "@types/parse5@^5.0.0": version "5.0.3" resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.3.tgz#e7b5aebbac150f8b5fdd4a46e7f0bd8e65e19109" From 0b11c3131723be4a281cca510ade6d92d7f6c384 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 18:31:25 +0800 Subject: [PATCH 11/22] Add linkedom dependency in packages/api --- packages/api/package.json | 1 + packages/puppeteer-parse/index.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 550c42ed6..dda402e4d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -62,6 +62,7 @@ "jwks-rsa": "^2.0.3", "knex": "0.21.12", "knex-stringcase": "^1.4.2", + "linkedom": "^0.14.9", "luxon": "^2.3.1", "nanoid": "^3.1.25", "nodemailer": "^6.7.3", diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 23ccdd4ea..6cb5926ad 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -363,7 +363,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { console.log(content); } - logRecord.timing.contentFetchTime = Date.now() - functionStartTime; + logRecord.contentFetchTime = Date.now() - functionStartTime; const apiResponse = await sendCreateArticleMutation(userId, { url: finalUrl, @@ -378,7 +378,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { skipParsing: !content, }); - logRecord.timing.totalTime = Date.now() - functionStartTime; + logRecord.totalTime = Date.now() - functionStartTime; logRecord.result = apiResponse.createArticle; logger.info(`parse-page`, logRecord); } From 39d47455911857677c3c814376355d2e0213290f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 18:31:58 +0800 Subject: [PATCH 12/22] Move linkedom dependency in devDependencies in readability --- packages/readabilityjs/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index ebf325df2..124fae654 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -30,11 +30,11 @@ "jsdom": "^19.0", "mocha": "^8.2.0", "puppeteer": "^10.1.0", - "sinon": "^7.3.2" + "sinon": "^7.3.2", + "linkedom": "^0.14.9" }, "dependencies": { "html-entities": "^2.3.2", - "linkedom": "^0.14.9", "modern-random-ua": "^1.0.3", "parse-srcset": "^1.0.2" } From 7d8379d093f36d308986605292cb6f271dfa0325 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 18:32:55 +0800 Subject: [PATCH 13/22] Use linkedom in readability benchmark --- packages/readabilityjs/benchmarks/benchmarks.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/readabilityjs/benchmarks/benchmarks.js b/packages/readabilityjs/benchmarks/benchmarks.js index 2d4dee38c..3472ccd42 100644 --- a/packages/readabilityjs/benchmarks/benchmarks.js +++ b/packages/readabilityjs/benchmarks/benchmarks.js @@ -1,8 +1,8 @@ var getTestPages = require("../test/utils").getTestPages; var { Readability, isProbablyReaderable } = require("../index"); -var JSDOM = require("jsdom").JSDOM; var JSDOMParser = require("../JSDOMParser"); +var { parseHTML } = require("linkedom"); var referenceTestPages = [ "002", @@ -55,10 +55,7 @@ suite("isProbablyReaderable perf", function () { set("type", "static"); testPages.forEach(function(testPage) { - var uri = "http://fakehost/test/page.html"; - var doc = new JSDOM(testPage.source, { - url: uri, - }).window.document; + var doc = parseHTML(testPage.source).document; bench(testPage.dir + " readability perf", function() { isProbablyReaderable(doc); }); From cb7f30607a01bbdb34e91626f515b4db80761f7e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 18:33:29 +0800 Subject: [PATCH 14/22] Use linkedom in readability test isProbablyReaderable --- .../readabilityjs/test/test-isProbablyReaderable.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/readabilityjs/test/test-isProbablyReaderable.js b/packages/readabilityjs/test/test-isProbablyReaderable.js index d00e87f01..c5236612e 100644 --- a/packages/readabilityjs/test/test-isProbablyReaderable.js +++ b/packages/readabilityjs/test/test-isProbablyReaderable.js @@ -1,5 +1,6 @@ -var JSDOM = require("jsdom").JSDOM; var chai = require("chai"); +var { parseHTML } = require("linkedom"); + chai.config.includeStack = true; var expect = chai.expect; @@ -9,11 +10,8 @@ var isProbablyReaderable = require("../index").isProbablyReaderable; describe("isProbablyReaderable - test pages", function () { testPages.forEach(function (testPage) { - var uri = "http://fakehost/test/page.html"; describe(testPage.dir, function () { - var doc = new JSDOM(testPage.source, { - url: uri, - }).window.document; + var doc = parseHTML(testPage.source).document; var expected = testPage.expectedMetadata.readerable; it("The result should " + (expected ? "" : "not ") + "be readerable", function () { expect(isProbablyReaderable(doc)).eql(expected); @@ -23,7 +21,7 @@ describe("isProbablyReaderable - test pages", function () { }); describe("isProbablyReaderable", function () { - const makeDoc = (source) => new JSDOM(source).window.document; + const makeDoc = (source) => parseHTML(source).document; var verySmallDoc = makeDoc("

hello there

"); // content length: 11 var smallDoc = makeDoc(`

${"hello there ".repeat(11)}

`); // content length: 132 var largeDoc = makeDoc(`

${"hello there ".repeat(12)}

`); // content length: 144 From ffa5dee7219f4172da921a68a6f2f10043396812 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 18:40:52 +0800 Subject: [PATCH 15/22] Use linkedom in readability tests --- .../readabilityjs/test/test-readability.js | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/readabilityjs/test/test-readability.js b/packages/readabilityjs/test/test-readability.js index 0c8154a3e..17b287b6d 100644 --- a/packages/readabilityjs/test/test-readability.js +++ b/packages/readabilityjs/test/test-readability.js @@ -1,6 +1,8 @@ var JSDOM = require("jsdom").JSDOM; var chai = require("chai"); var sinon = require("sinon"); +const { parseHTML } = require("linkedom"); + chai.config.includeStack = true; var expect = chai.expect; @@ -52,7 +54,7 @@ function htmlTransform(str) { return str.replace(/\s+/g, " "); } -function runTestsWithItems(label, domGenerationFn, source, expectedContent, expectedMetadata) { +function runTestsWithItems(label, domGenerationFn, source, expectedContent, expectedMetadata, uri) { describe(label, function() { this.timeout(30000); @@ -63,7 +65,7 @@ function runTestsWithItems(label, domGenerationFn, source, expectedContent, expe var doc = domGenerationFn(source); // Provide one class name to preserve, which we know appears in a few // of the test documents. - var myReader = new Readability(doc, { classesToPreserve: ["caption"] }); + var myReader = new Readability(doc, { classesToPreserve: ["caption"], url: uri }); result = myReader.parse(); } catch (err) { throw reformatError(err); @@ -227,7 +229,7 @@ describe("Readability API", function() { }); it("should run _cleanClasses with default configuration", function() { - var doc = new JSDOM(exampleSource).window.document; + var doc = parseHTML(exampleSource).document; var parser = new Readability(doc); parser._cleanClasses = sinon.fake(); @@ -238,7 +240,7 @@ describe("Readability API", function() { }); it("should run _cleanClasses when option keepClasses = false", function() { - var doc = new JSDOM(exampleSource).window.document; + var doc = parseHTML(exampleSource).document; var parser = new Readability(doc, {keepClasses: false}); parser._cleanClasses = sinon.fake(); @@ -249,7 +251,7 @@ describe("Readability API", function() { }); it("shouldn't run _cleanClasses when option keepClasses = true", function() { - var doc = new JSDOM(exampleSource).window.document; + var doc = parseHTML(exampleSource).document; var parser = new Readability(doc, {keepClasses: true}); parser._cleanClasses = sinon.fake(); @@ -272,30 +274,30 @@ describe("Readability API", function() { }); it("should not proxy image with data uri", function() { - var dom = new JSDOM("My cat: My cat: \"Red"); - var expected_xhtml = "
My cat: \"Red"); + var expected_xhtml = "
My cat: \"Red
"; - var content = new Readability(dom.window.document).parse().content; + "Y4OHwAAAABJRU5ErkJggg==\" alt=\"Red dot\">
"; + var content = new Readability(dom.document).parse().content; expect(content).eql(expected_xhtml); }); it("should handle srcset elements with density descriptors", function() { - var dom = new JSDOM('My image: My image: ' - ); - var expected_xhtml = '
My image: ' + + 'https://webkit.org/demos/srcset/image-4x.png 4x">' + + ''); + var expected_xhtml = '
My image: ' + '
'; - var content = new Readability(dom.window.document, { + 'https://webkit.org/demos/srcset/image-4x.png 4x,">
'; + var content = new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } @@ -304,11 +306,11 @@ describe("Readability API", function() { }); it("should remove srcset elements that are lazy loading placeholders", function() { - var dom = new JSDOM('My image: '); - var expected_xhtml = '
' + - 'My image: ' + - '
' - var content = new Readability(dom.window.document, { + var dom = parseHTML('My image: '); + var expected_xhtml = '
' + + 'My image: ' + + '
'; + var content = new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } @@ -324,12 +326,10 @@ describe("Test pages", function() { var uri = "http://fakehost/test/page.html"; runTestsWithItems("jsdom", function(source) { - var doc = new JSDOM(source, { - url: uri, - }).window.document; + var doc =parseHTML(source).document; removeCommentNodesRecursively(doc); return doc; - }, testPage.source, testPage.expectedContent, testPage.expectedMetadata); + }, testPage.source, testPage.expectedContent, testPage.expectedMetadata, uri); // runTestsWithItems("JSDOMParser", function(source) { // var parser = new JSDOMParser(); From 0eb565eae955413e2ae3d637eaaee7e8b6510d83 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 21:09:19 +0800 Subject: [PATCH 16/22] Remove JSDOM dependencies from readability --- packages/readabilityjs/package.json | 1 - yarn.lock | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index 124fae654..ca3df23c8 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -27,7 +27,6 @@ "chai": "^2.1.*", "htmltidy2": "^0.3.0", "js-beautify": "^1.13.0", - "jsdom": "^19.0", "mocha": "^8.2.0", "puppeteer": "^10.1.0", "sinon": "^7.3.2", diff --git a/yarn.lock b/yarn.lock index 6d4342b9d..b20ba3c4b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16912,7 +16912,7 @@ jsdom@^16.6.0: ws "^7.4.6" xml-name-validator "^3.0.0" -jsdom@^19.0, jsdom@^19.0.0: +jsdom@^19.0.0: version "19.0.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-19.0.0.tgz#93e67c149fe26816d38a849ea30ac93677e16b6a" integrity sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A== From 96b543946dcb77e7aca8fb09a6d7de0e3925f806 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 21:10:06 +0800 Subject: [PATCH 17/22] Temporarily disable customer content serializer test --- packages/readabilityjs/test/test-readability.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/readabilityjs/test/test-readability.js b/packages/readabilityjs/test/test-readability.js index 17b287b6d..1228db774 100644 --- a/packages/readabilityjs/test/test-readability.js +++ b/packages/readabilityjs/test/test-readability.js @@ -1,4 +1,3 @@ -var JSDOM = require("jsdom").JSDOM; var chai = require("chai"); var sinon = require("sinon"); const { parseHTML } = require("linkedom"); @@ -261,7 +260,7 @@ describe("Readability API", function() { expect(parser._cleanClasses.called).eql(false); }); - it("should use custom content serializer sent as option", function() { + xit("should use custom content serializer sent as option", function() { var dom = new JSDOM("My cat: "); var expected_xhtml = "
My cat:
"; var xml = new dom.window.XMLSerializer(); From 82fb8151a4975842c44bb96e82e3ea150fd45854 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 10 May 2022 21:10:20 +0800 Subject: [PATCH 18/22] Fix generate tests --- packages/readabilityjs/test/generate-testcase.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index 875bdb567..86b13c951 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -2,7 +2,6 @@ var debug = false; var path = require("path"); var fs = require("fs"); -var JSDOM = require("jsdom").JSDOM; var prettyPrint = require("./utils").prettyPrint; var htmltidy = require("htmltidy2").tidy; @@ -10,6 +9,7 @@ var { Readability, isProbablyReaderable } = require("../index"); var JSDOMParser = require("../JSDOMParser"); const { generate: generateRandomUA } = require("modern-random-ua/random_ua"); const puppeteer = require('puppeteer'); +const { parseHTML } = require("linkedom"); var testcaseRoot = path.join(__dirname, "test-pages"); @@ -173,7 +173,7 @@ async function fetchSource(url, callbackFn) { } function sanitizeSource(html, callbackFn) { - htmltidy(new JSDOM(html).serialize(), { + htmltidy(parseHTML(html).serialize(), { "indent": true, "indent-spaces": 4, "numeric-entities": true, @@ -210,14 +210,12 @@ function runReadability(source, destPath, metadataDestPath) { var myReader, result, readerable; try { // Use jsdom for isProbablyReaderable because it supports querySelectorAll - var jsdom = new JSDOM(source, { - url: uri, - }).window.document; + var jsdom = parseHTML(source).document; + readerable = isProbablyReaderable(jsdom); // We pass `caption` as a class to check that passing in extra classes works, // given that it appears in some of the test documents. - myReader = new Readability(jsdom, { classesToPreserve: ["caption"]}); + myReader = new Readability(jsdom, { classesToPreserve: ["caption"], url: uri }); result = myReader.parse(); - readerable = isProbablyReaderable(jsdom); } catch (ex) { console.error(ex); ex.stack.forEach(console.log.bind(console)); @@ -237,6 +235,7 @@ function runReadability(source, destPath, metadataDestPath) { delete result.content; delete result.textContent; delete result.length; + delete result.dom; // Add isProbablyReaderable result result.readerable = readerable; From 2755da16a9baa153bd4af122c15f33cca0001cd7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 11 May 2022 19:25:12 +0800 Subject: [PATCH 19/22] Fix not getting iframe src --- packages/readabilityjs/Readability.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index efcb6bb8b..6b704434f 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -2260,8 +2260,8 @@ Readability.prototype = { } // Create instagram posts placeholders from iframes - if (element.src && element.src.includes('instagram.com/p')) { - const url = element.src; + if (element.getAttribute('src')?.includes('instagram.com/p')) { + const url = element.getAttribute('src'); const regex = /https?:\/\/(www\.)?instagram.com\/p\/(\w+)\//gm; const match = regex.exec(url); From d542d31aed2ecb855f36ebe099e9a91488501652 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 11 May 2022 21:23:30 +0800 Subject: [PATCH 20/22] Fix gflownet test generation --- .../gflownet/expected-metadata.json | 3 +- .../test/test-pages/gflownet/expected.html | 1686 ++++++++++++++++- 2 files changed, 1676 insertions(+), 13 deletions(-) diff --git a/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json b/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json index 81947a8a7..470e252e0 100644 --- a/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json @@ -2,8 +2,9 @@ "title": "Flow Network based Generative Models for Non-Iterative Diverse Candidate Generation", "byline": null, "dir": null, - "excerpt": "What follows is a high-level overview of this work, for more details refer to our paper. Given a reward and a deterministic episodic environment where episodes end with a ``generate '' action, how do we generate diverse and high-reward s?\n We propose to use Flow Networks to model discrete from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse s by design.", + "excerpt": "What follows is a high-level overview of this work, for more details refer to our paper. Given a reward \n \n \n \n R\n \n \n (\n \n \n x\n \n \n )\n \n \n \n R(x)\n \n and a deterministic episodic environment where episodes end with a ``generate \n \n \n \n x\n \n \n \n x\n \n '' action, how do we generate diverse and high-reward \n \n \n \n x\n \n \n \n x\n \n s?\n We propose to use Flow Networks to model discrete \n \n \n \n p\n \n \n (\n \n \n x\n \n \n )\n \n \n ∝\n \n \n R\n \n \n (\n \n \n x\n \n \n )\n \n \n \n p(x) \\propto R(x)\n \n from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse \n \n \n \n x\n \n \n \n x\n \n s by design.", "siteName": null, + "siteIcon": "", "publishedDate": null, "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/gflownet/expected.html b/packages/readabilityjs/test/test-pages/gflownet/expected.html index b7ba06fcf..a77923e19 100644 --- a/packages/readabilityjs/test/test-pages/gflownet/expected.html +++ b/packages/readabilityjs/test/test-pages/gflownet/expected.html @@ -1,41 +1,1563 @@ -
+
[Home]
- Emmanuel Bengio, Moksh Jain, Maksym Korablyov, Doina Precup, Yoshua Bengio + Emmanuel Bengio, Moksh Jain, Maksym Korablyov, Doina Precup, Yoshua Bengio

arXiv preprint, code
also see the GFlowNet Foundations paper
and a more recent (and thorough) tutorial on the framework.
-

What follows is a high-level overview of this work, for more details refer to our paper. Given a reward and a deterministic episodic environment where episodes end with a ``generate '' action, how do we generate diverse and high-reward s?
We propose to use Flow Networks to model discrete from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse s by design.
+

What follows is a high-level overview of this work, for more details refer to our paper. Given a reward + + + R + ( + x + ) + + R(x) + + and a deterministic episodic environment where episodes end with a ``generate + + + x + + x + + '' action, how do we generate diverse and high-reward + + + x + + x + + s?
We propose to use Flow Networks to model discrete + + + p + ( + x + ) + ∝ + R + ( + x + ) + + p(x) \propto R(x) + + from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse + + + x + + x + + s by design.

Flow Networks

-

A flow network is a directed graph with sources and sinks, and edges carrying some amount of flow between them through intermediate nodes -- think of pipes of water. For our purposes, we define a flow network with a single source, the root or ; the sinks of the network correspond to the terminal states. We'll assign to each sink an ``out-flow'' .

+

A flow network is a directed graph with sources and sinks, and edges carrying some amount of flow between them through intermediate nodes -- think of pipes of water. For our purposes, we define a flow network with a single source, the root or + + + + s + 0 + + + s_0 + + ; the sinks of the network correspond to the terminal states. We'll assign to each sink + + + x + + x + + an ``out-flow'' + + + R + ( + x + ) + + R(x) + + .

+
+ +

+ + + + s + 0 + + + s_{0} + + + + + + s + 1 + + + s_{1} + + + + + + s + 2 + + + s_{2} + + + + + + s + 3 + + + s_{3} + + + + + + x + 3 + + + x_{3} + + + + + ⊤ + + \top + + + + + + s + 5 + + + s_{5} + + + + + + x + 5 + + + x_{5} + + + + + ⊤ + + \top + + + + + + s + 7 + + + s_{7} + + + + + + s + 8 + + + s_{8} + + + + + + x + 8 + + + x_{8} + + + + + ⊤ + + \top + + + + + + s + 10 + + + s_{10} + + + + + + s + 11 + + + s_{11} + + + + + + x + 11 + + + x_{11} + + + + + ⊤ + + \top + + + + + + s + 13 + + + s_{13} + + + + + + x + 13 + + + x_{13} + + + + + ⊤ + + \top + + + + + + s + 15 + + + s_{15} + + + + + + s + 16 + + + s_{16} + + + + + + x + 16 + + + x_{16} + + + + + ⊤ + + \top + + +

+
-

Given the graph structure and the out-flow of the sinks, we wish to calculate a valid flow between nodes, e.g. how much water each pipe is carrying. Generally there can be infinite solutions, but this is not a problem here -- any valid solution will do. For example above, there is almost no flow between and that goes through , it all goes through , but the reverse solution would also be a valid flow.
Why is this useful? Such a construction corresponds to a generative model. If we follow the flow, we'll end up in a terminal state, a sink, with probability . On top of that, we'll have the property that the in-flow of --the flow of the unique source--is , the partition function. If we assign to each intermediate node a state and to each edge an action, we recover a useful MDP.
Let be the flow between and , where , i.e. is the (deterministic) state transitioned to from state and action . Let then following policy , starting from , leads to terminal state with probability (see the paper for proofs and more rigorous explanations).
+

Given the graph structure and the out-flow of the sinks, we wish to calculate a valid flow between nodes, e.g. how much water each pipe is carrying. Generally there can be infinite solutions, but this is not a problem here -- any valid solution will do. For example above, there is almost no flow between + + + + s + 7 + + + s_7 + + and + + + + s + 13 + + + s_{13} + + that goes through + + + + s + 11 + + + s_{11} + + , it all goes through + + + + s + 10 + + + s_{10} + + , but the reverse solution would also be a valid flow.
Why is this useful? Such a construction corresponds to a generative model. If we follow the flow, we'll end up in a terminal state, a sink, with probability + + + p + ( + x + ) + ∝ + R + ( + x + ) + + p(x) \propto R(x) + + . On top of that, we'll have the property that the in-flow of + + + + s + 0 + + + s_0 + + --the flow of the unique source--is + + + + ∑ + x + + R + ( + x + ) + = + Z + + \sum_x R(x)=Z + + , the partition function. If we assign to each intermediate node a state and to each edge an action, we recover a useful MDP.
Let + + + F + ( + s + , + a + ) + = + f + ( + s + , + + s + ′ + + ) + + F(s,a)=f(s,s') + + be the flow between + + + s + + s + + and + + + + s + ′ + + + s' + + , where + + + T + ( + s + , + a + ) + = + + s + ′ + + + T(s,a)=s' + + , i.e. + + + + s + ′ + + + s' + + is the (deterministic) state transitioned to from state + + + s + + s + + and action + + + a + + a + + . Let + + + + + + + π + ( + a + ∣ + s + ) + = + + + F + ( + s + , + a + ) + + + + ∑ + + a + ′ + + + F + ( + s + , + + a + ′ + + ) + + + + + + + + \begin{aligned}\pi(a|s) = \frac{F(s,a)}{\sum_{a'}F(s,a')}\end{aligned} + + then following policy + + + π + + \pi + + , starting from + + + + s + 0 + + + s_0 + + , leads to terminal state + + + x + + x + + with probability + + + R + ( + x + ) + + R(x) + + (see the paper for proofs and more rigorous explanations).

Approximating Flow Networks

-

As you may suspect, there are only few scenarios in which we can build the above graph explicitly. For drug-like molecules, it would have around nodes!
Instead, we resort to function approximation, just like deep RL resorts to it when computing the (action-)value functions of MDPs.
Our goal here is to approximate the flow . Earlier we called a valid flow one that correctly routed all the flow from the source to the sinks through the intermediary nodes. Let's be more precise. For some node , let the in-flow be the sum of incoming flows: Here the set is the set of state-action pairs that lead to . Now, let the out-flow be the sum of outgoing flows--or the reward if is terminal: Note that we reused . This is because for a valid flow, the in-flow is equal to the out-flow, i.e. the flow through , . Here is the set of valid actions in state , which is the empty set when is a sink. is 0 unless is a sink, in which case .
We can thus call the set of these equalities for all states the flow consistency equations:

+

As you may suspect, there are only few scenarios in which we can build the above graph explicitly. For drug-like molecules, it would have around + + + 1 + + 0 + 16 + + + 10^{16} + + nodes!
Instead, we resort to function approximation, just like deep RL resorts to it when computing the (action-)value functions of MDPs.
Our goal here is to approximate the flow + + + F + ( + s + , + a + ) + + F(s,a) + + . Earlier we called a valid flow one that correctly routed all the flow from the source to the sinks through the intermediary nodes. Let's be more precise. For some node + + + + s + ′ + + + s' + + , let the in-flow + + + F + ( + + s + ′ + + ) + + F(s') + + be the sum of incoming flows: + + + + + + + F + ( + + s + ′ + + ) + = + + ∑ + + s + , + a + : + T + ( + s + , + a + ) + = + + s + ′ + + + + F + ( + s + , + a + ) + + + + + + \begin{aligned}F(s') = \sum_{s,a:T(s,a)=s'} F(s,a)\end{aligned} + + Here the set + + + { + s + , + a + : + T + ( + s + , + a + ) + = + + s + ′ + + } + + \{s,a:T(s,a)=s'\} + + is the set of state-action pairs that lead to + + + + s + ′ + + + s' + + . Now, let the out-flow be the sum of outgoing flows--or the reward if + + + + s + ′ + + + s' + + is terminal: + + + + + + + F + ( + + s + ′ + + ) + = + R + ( + + s + ′ + + ) + + + + ∑ + + + a + ′ + + ∈ + A + ( + + s + ′ + + ) + + + F + ( + + s + ′ + + , + + a + ′ + + ) + . + + + + + + \begin{aligned}F(s') = R(s') + \sum_{a'\in\mathcal{A}(s')} F(s',a').\end{aligned} + + Note that we reused + + + F + ( + + s + ′ + + ) + + F(s') + + . This is because for a valid flow, the in-flow is equal to the out-flow, i.e. the flow through + + + + s + ′ + + + s' + + , + + + F + ( + + s + ′ + + ) + + F(s') + + . Here + + + A + ( + s + ) + + \mathcal{A}(s) + + is the set of valid actions in state + + + s + + s + + , which is the empty set when + + + s + + s + + is a sink. + + + R + ( + s + ) + + R(s) + + is 0 unless + + + s + + s + + is a sink, in which case + + + R + ( + s + ) + > + 0 + + R(s)>0 + + .
We can thus call the set of these equalities for all states + + + + s + ′ + + ≠ + + s + 0 + + + s'\neq s_0 + + the flow consistency equations: + + + + + + + + ∑ + + s + , + a + : + T + ( + s + , + a + ) + = + + s + ′ + + + + F + ( + s + , + a + ) + = + R + ( + + s + ′ + + ) + + + + ∑ + + + a + ′ + + ∈ + A + ( + + s + ′ + + ) + + + F + ( + + s + ′ + + , + + a + ′ + + ) + . + + + + + + \begin{aligned}\sum_{s,a:T(s,a)=s'} F(s,a) = R(s') + \sum_{a'\in\mathcal{A}(s')} F(s',a').\end{aligned} + +

+
+ +

+ + + + a + 1 + + + a_1 + + + + + + a + 7 + + + a_7 + + + + + + a + 3 + + + a_3 + + + + + + a + 4 + + + a_4 + + + + + + a + 2 + + + a_2 + + + + + + a + 8 + + + a_8 + + + + + + s + 0 + + + s_{0} + + + + + + s + 1 + + + s_{1} + + + + + + s + 2 + + + s_{2} + + + + + + s + 3 + + + s_{3} + + + + + + s + 4 + + + s_{4} + + + + + + s + 5 + + + s_{5} + + + + + + s + 6 + + + s_{6} + + +

+
-

Here the set of parents is , and .
By now our RL senses should be tingling. We've defined a value function recursively, with two quantities that need to match.
+

Here the set of parents + + + { + s + , + a + : + T + ( + s + , + a + ) + = + + s + 3 + + } + + \{s,a:T(s,a)=s_3\} + + is + + + { + ( + + s + 0 + + , + + a + 1 + + ) + , + ( + + s + 1 + + , + + a + 7 + + ) + , + ( + + s + 2 + + , + + a + 3 + + ) + } + + \{(s_0, a_1), (s_1, a_7), (s_2, a_3)\} + + , and + + + A + ( + + s + 3 + + ) + = + { + + a + 2 + + , + + a + 4 + + , + + a + 8 + + } + + \mathcal{A}(s_3)=\{a_2,a_4,a_8\} + + .
By now our RL senses should be tingling. We've defined a value function recursively, with two quantities that need to match.

A TD-Like Objective

-

Just like one can cast the Bellman equations into TD objectives, so do we cast the flow consistency equations into an objective. We want that minimizes the square difference between the two sides of the equations, but we add a few bells and whistles: First, we match the of each side, which is important since as intermediate nodes get closer to the root, their flow will become exponentially bigger (remember that ), but we care equally about all nodes. Second, we predict for the same reasons. Finally, we add an value inside the ; this doesn't change the minima of the objective, but gives more gradient weight to large values and less to small values.
We show in the paper that a minimizer of this objective achieves our desiderata, which is to have when sampling from as defined above.
+

Just like one can cast the Bellman equations into TD objectives, so do we cast the flow consistency equations into an objective. We want + + + + F + θ + + + F_\theta + + that minimizes the square difference between the two sides of the equations, but we add a few bells and whistles: + + + + + + + + L + + θ + , + ϵ + + + ( + τ + ) + = + + ∑ + + + + s + ′ + + ∈ + τ + ≠ + + s + 0 + + + + +   + + + ( + log + ⁡ +  ⁣ + + [ + ϵ + + + + ∑ + + + s + , + a + : + T + ( + s + , + a + ) + = + + s + ′ + + + + + exp + ⁡ + + F + θ + log + ⁡ + + ( + s + , + a + ) + ] + + − + log + ⁡ +  ⁣ + + [ + ϵ + + + R + ( + + s + ′ + + ) + + + + ∑ + + + + a + ′ + + ∈ + A + ( + + s + ′ + + ) + + + + exp + ⁡ + + F + θ + log + ⁡ + + ( + + s + ′ + + , + + a + ′ + + ) + ] + + ) + + 2 + + . + + + + + + \begin{aligned}\mathcal{L}_{\theta,\epsilon}(\tau) = \sum_{\mathclap{s'\in\tau\neq s_0}}\,\left(\log\! \left[\epsilon+{\sum_{\mathclap{s,a:T(s,a)=s'}}} \exp F^{\log}_\theta(s,a)\right]- \log\! \left[\epsilon + R(s') + \sum_{\mathclap{a'\in{\cal A}(s')}} \exp F^{\log}_\theta(s',a')\right]\right)^2.\end{aligned} + + First, we match the + + + log + ⁡ + + \log + + of each side, which is important since as intermediate nodes get closer to the root, their flow will become exponentially bigger (remember that + + + F + ( + + s + 0 + + ) + = + Z + = + + ∑ + x + + R + ( + x + ) + + F(s_0) = Z = \sum_x R(x) + + ), but we care equally about all nodes. Second, we predict + + + + F + θ + log + ⁡ + + ≈ + log + ⁡ + F + + F^{\log}_\theta\approx\log F + + for the same reasons. Finally, we add an + + + ϵ + + \epsilon + + value inside the + + + log + ⁡ + + \log + + ; this doesn't change the minima of the objective, but gives more gradient weight to large values and less to small values.
We show in the paper that a minimizer of this objective achieves our desiderata, which is to have + + + p + ( + x + ) + ∝ + R + ( + x + ) + + p(x)\propto R(x) + + when sampling from + + + π + ( + a + ∣ + s + ) + + \pi(a|s) + + as defined above.

GFlowNet as Amortized Sampling with an OOD Potential

-

It is interesting to compare GFlowNet with Monte-Carlo Markov Chain (MCMC) methods. MCMC methods can be used to sample from a distribution for which there is no analytical sampling formula but an energy function or unnormalized probability function is available. In our context, this unnormalized probability function is our reward function .
Like MCMC methods, GFlowNet can turn a given energy function into samples but it does it in an amortized way, converting the cost a lot of very expensive MCMC trajectories (to obtain each sample) into the cost training a generative model (in our case a generative policy which sequentially builds up ). Sampling from the generative model is then very cheap (e.g. adding one component at a time to a molecule) compared to an MCMC. But the most important gain may not be just computational, but in terms of the ability to discover new modes of the reward function.
MCMC methods are iterative, making many small noisy steps, which can converge in the neighborhood of a mode, and with some probability jump from one mode to a nearby one. However, if two modes are far from each other, MCMC can require exponential time to mix between the two. If in addition the modes occupy a tiny volume of the state space, the chances of initializing a chain near one of the unknown modes is also tiny, and the MCMC approach becomes unsatisfactory. Whereas such a situation seems hopeless with MCMC, GFlowNet has the potential to discover modes and jump there directly, if there is structure that relates the modes that it already knows, and if its inductive biases and training procedure make it possible to generalize there.
GFlowNet does not need to perfectly know where the modes are: it is sufficient to make guesses which occasionally work well. Like for MCMC methods, once a point in the region of new mode is discovered, further training of GFlowNet will sculpt that mode and zoom in on its peak.
Note that we can put to some power , a coefficient which acts like a temperature, and , making it possible to focus more or less on the highest modes (versus spreading probability mass more uniformly).
+

It is interesting to compare GFlowNet with Monte-Carlo Markov Chain (MCMC) methods. MCMC methods can be used to sample from a distribution for which there is no analytical sampling formula but an energy function or unnormalized probability function is available. In our context, this unnormalized probability function is our reward function + + + R + ( + x + ) + = + + e + + − + e + n + e + r + g + y + ( + x + ) + + + + R(x)=e^{-energy(x)} + + .
Like MCMC methods, GFlowNet can turn a given energy function into samples but it does it in an amortized way, converting the cost a lot of very expensive MCMC trajectories (to obtain each sample) into the cost training a generative model (in our case a generative policy which sequentially builds up + + + x + + x + + ). Sampling from the generative model is then very cheap (e.g. adding one component at a time to a molecule) compared to an MCMC. But the most important gain may not be just computational, but in terms of the ability to discover new modes of the reward function.
MCMC methods are iterative, making many small noisy steps, which can converge in the neighborhood of a mode, and with some probability jump from one mode to a nearby one. However, if two modes are far from each other, MCMC can require exponential time to mix between the two. If in addition the modes occupy a tiny volume of the state space, the chances of initializing a chain near one of the unknown modes is also tiny, and the MCMC approach becomes unsatisfactory. Whereas such a situation seems hopeless with MCMC, GFlowNet has the potential to discover modes and jump there directly, if there is structure that relates the modes that it already knows, and if its inductive biases and training procedure make it possible to generalize there.
GFlowNet does not need to perfectly know where the modes are: it is sufficient to make guesses which occasionally work well. Like for MCMC methods, once a point in the region of new mode is discovered, further training of GFlowNet will sculpt that mode and zoom in on its peak.
Note that we can put + + + R + ( + x + ) + + R(x) + + to some power + + + β + + \beta + + , a coefficient which acts like a temperature, and + + + R + ( + x + + ) + β + + = + + e + + − + β +    + e + n + e + r + g + y + ( + x + ) + + + + R(x)^\beta = e^{-\beta\; energy(x)} + + , making it possible to focus more or less on the highest modes (versus spreading probability mass more uniformly).

Generating molecule graphs

-

The motivation for this work is to be able to generate diverse molecules from a proxy reward that is imprecise because it comes from biochemical simulations that have a high uncertainty. As such, we do not care about the maximizer as RL methods would, but rather about a set of ``good enough'' candidates to send to a true biochemical assay.
Another motivation is to have diversity: by fitting the distribution of rewards rather than trying to maximize the expected reward, we're likely to find more modes than if we were being greedy after having found a good enough mode, which again and again we've found RL methods such as PPO to do.
Here we generate molecule graphs via a sequence of additive edits, i.e. we progressively build the graph by adding new leaf nodes to it. We also create molecules block-by-block rather than atom-by-atom.
We find experimentally that we get both good molecules, and diverse ones. We compare ourselves to PPO and MARS (an MCMC-based method).
Figure 3 shows that we're fitting a distribution that makes sense. If we change the reward by exponentiating it as with , this shifts the reward distribution to the right.
Figure 4 shows the top- found as a function of the number of episodes.

+

The motivation for this work is to be able to generate diverse molecules from a proxy reward + + + R + + R + + that is imprecise because it comes from biochemical simulations that have a high uncertainty. As such, we do not care about the maximizer as RL methods would, but rather about a set of ``good enough'' candidates to send to a true biochemical assay.
Another motivation is to have diversity: by fitting the distribution of rewards rather than trying to maximize the expected reward, we're likely to find more modes than if we were being greedy after having found a good enough mode, which again and again we've found RL methods such as PPO to do.
Here we generate molecule graphs via a sequence of additive edits, i.e. we progressively build the graph by adding new leaf nodes to it. We also create molecules block-by-block rather than atom-by-atom.
We find experimentally that we get both good molecules, and diverse ones. We compare ourselves to PPO and MARS (an MCMC-based method).
Figure 3 shows that we're fitting a distribution that makes sense. If we change the reward by exponentiating it as + + + + R + β + + + R^\beta + + with + + + β + > + 1 + + \beta>1 + + , this shifts the reward distribution to the right.
Figure 4 shows the top- + + + k + + k + + found as a function of the number of episodes.

@@ -44,11 +1566,151 @@

Active Learning experiments

-

The above experiments assume access to a reward that is cheap to evaluate. In fact it uses a neural network proxy trained from a large dataset of molecules. This setup isn't quite what we would get when interacting with biochemical assays, where we'd have access to much fewer data. To emulate such a setting, we consider our oracle to be a docking simulation (which is relatively expensive to run, ~30 cpu seconds).
In this setting, there is a limited budget for calls to the true oracle . We use a proxy initialized by training on a limited dataset of pairs , where is the true reward from the oracle. The generative model () is then trained to fit but as predicted by the proxy . We then sample a batch where , which is evaluated with the oracle . The proxy is updated with this newly acquired and labeled batch, and the process is repeated for iterations.
By doing this on the molecule setting we again find that we can generate better molecules. This showcases the importance of having these diverse candidates.

+

The above experiments assume access to a reward + + + R + + R + + that is cheap to evaluate. In fact it uses a neural network proxy trained from a large dataset of molecules. This setup isn't quite what we would get when interacting with biochemical assays, where we'd have access to much fewer data. To emulate such a setting, we consider our oracle to be a docking simulation (which is relatively expensive to run, ~30 cpu seconds).
In this setting, there is a limited budget for calls to the true oracle + + + O + + O + + . We use a proxy + + + M + + M + + initialized by training on a limited dataset of + + + ( + x + , + R + ( + x + ) + ) + + (x, R(x)) + + pairs + + + + D + 0 + + + D_0 + + , where + + + R + ( + x + ) + + R(x) + + is the true reward from the oracle. The generative model ( + + + + π + θ + + + \pi_{\theta} + + ) is then trained to fit + + + R + + R + + but as predicted by the proxy + + + M + + M + + . We then sample a batch + + + B + = + { + + x + 1 + + , + + x + 2 + + , + … + + x + k + + } + + B=\{x_1, x_2, \dots x_k\} + + where + + + + x + i + + ∼ + + π + θ + + + x_i\sim \pi_{\theta} + + , which is evaluated with the oracle + + + O + + O + + . The proxy + + + M + + M + + is updated with this newly acquired and labeled batch, and the process is repeated for + + + N + + N + + iterations.
By doing this on the molecule setting we again find that we can generate better molecules. This showcases the importance of having these diverse candidates.

For more figures, experiments and explanations, check out the paper, or reach out to us!

-
\ No newline at end of file +
\ No newline at end of file From 0984dca18304ee048d070d624798c7fc222e7455 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 11 May 2022 22:04:47 +0800 Subject: [PATCH 21/22] Remove adblocker and block resources by url and also block mathJax script --- packages/content-fetch/fetch-content.js | 68 +++++++++++++++++-------- packages/puppeteer-parse/index.js | 58 +++++++++++++++++++-- packages/puppeteer-parse/package.json | 1 - 3 files changed, 102 insertions(+), 25 deletions(-) diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index c7493e364..38c19fcf1 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -31,9 +31,6 @@ const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/pl // Add stealth plugin to hide puppeteer usage const StealthPlugin = require('puppeteer-extra-plugin-stealth'); puppeteer.use(StealthPlugin()); -// Add adblocker plugin to block ads and trackers -const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const userAgentForUrl = (url) => { @@ -351,6 +348,33 @@ function getUrl(req) { return parsed.href; } + +async function blockResources(page) { + const blockedResources = [ + // Assets + '*/favicon.ico', + '.css', + '.jpg', + '.jpeg', + '.png', + '.svg', + '.woff', + + // Analytics and other fluff + '*.optimizely.com', + 'everesttech.net', + 'userzoom.com', + 'doubleclick.net', + 'googleadservices.com', + 'adservice.google.com/*', + 'connect.facebook.com', + 'connect.facebook.net', + 'sp.analytics.yahoo.com', + ] + + await page._client.send('Network.setBlockedURLs', { urls: blockedResources }); +} + async function retrievePage(url) { validateUrlString(url); @@ -406,6 +430,8 @@ async function retrievePage(url) { } catch {} }); + await blockResources(page); + /* * Disallow MathJax from running in Puppeteer and modifying the document, * we shall instead run it in our frontend application to transform any @@ -413,24 +439,24 @@ async function retrievePage(url) { */ await page.setRequestInterception(true); let requestCount = 0; - // page.on('request', request => { - // if (request.resourceType() === 'font' || request.resourceType() === 'image') { - // request.abort(); - // return; - // } - // if (requestCount++ > 100) { - // request.abort(); - // return; - // } - // if ( - // request.resourceType() === 'script' && - // request.url().toLowerCase().indexOf('mathjax') > -1 - // ) { - // request.abort(); - // } else { - // request.continue(); - // } - // }); + page.on('request', request => { + if (request.resourceType() === 'font' || request.resourceType() === 'image') { + request.abort(); + return; + } + if (requestCount++ > 100) { + request.abort(); + return; + } + if ( + request.resourceType() === 'script' && + request.url().toLowerCase().indexOf('mathjax') > -1 + ) { + request.abort(); + } else { + request.continue(); + } + }); // Puppeteer fails during download of PDf files, // so record the failure and use those items diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 6cb5926ad..56a64afc2 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -29,9 +29,6 @@ const puppeteer = require('puppeteer-extra'); // Add stealth plugin to hide puppeteer usage const StealthPlugin = require('puppeteer-extra-plugin-stealth'); puppeteer.use(StealthPlugin()); -// Add adblocker plugin to block ads and trackers -const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; @@ -554,6 +551,32 @@ function getUrl(req) { } catch (e) {} } +async function blockResources(page) { + const blockedResources = [ + // Assets + '*/favicon.ico', + '.css', + '.jpg', + '.jpeg', + '.png', + '.svg', + '.woff', + + // Analytics and other fluff + '*.optimizely.com', + 'everesttech.net', + 'userzoom.com', + 'doubleclick.net', + 'googleadservices.com', + 'adservice.google.com/*', + 'connect.facebook.com', + 'connect.facebook.net', + 'sp.analytics.yahoo.com', + ] + + await page._client.send('Network.setBlockedURLs', { urls: blockedResources }); +} + async function retrievePage(url) { validateUrlString(url); @@ -609,6 +632,35 @@ async function retrievePage(url) { } catch {} }); + await blockResources(page); + + /* + * Disallow MathJax from running in Puppeteer and modifying the document, + * we shall instead run it in our frontend application to transform any + * mathjax content when present. + */ + await page.setRequestInterception(true); + let requestCount = 0; + page.on('request', request => { + if (request.resourceType() === 'font' || request.resourceType() === 'image') { + request.abort(); + return; + } + if (requestCount++ > 100) { + request.abort(); + return; + } + if ( + request.resourceType() === 'script' && + request.url().toLowerCase().indexOf('mathjax') > -1 + ) { + request.abort(); + } else { + request.continue(); + } + }); + + // Puppeteer fails during download of PDf files, // so record the failure and use those items let lastPdfUrl = undefined; diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 2159015e5..e37537415 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -15,7 +15,6 @@ "luxon": "^2.3.1", "puppeteer-core": "^13.7.0", "puppeteer-extra": "^3.2.3", - "puppeteer-extra-plugin-adblocker": "^2.12.0", "puppeteer-extra-plugin-stealth": "^2.9.0", "winston": "^3.3.3" }, From 602d141decd3d73c79fbfe8e48ae72c919acaf6e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 12 May 2022 11:00:32 +0800 Subject: [PATCH 22/22] Rename doc to dom --- packages/api/src/utils/parser.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 878f801b1..06fe8a433 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -240,12 +240,12 @@ export const parsePreparedContent = async ( // ...details, // }) // }) - const { document: doc } = parseHTML(document) + const dom = parseHTML(document).document - await applyHandlers(url, doc) + await applyHandlers(url, dom) try { - article = getReadabilityResult(url, document, doc, isNewsletter) + article = getReadabilityResult(url, document, dom, isNewsletter) // Format code blocks // TODO: we probably want to move this type of thing @@ -278,7 +278,7 @@ export const parsePreparedContent = async ( const clean = DOMPurify.sanitize(article?.content || '', DOM_PURIFY_CONFIG) const jsonLdLinkMetadata = (async () => { - return getJSONLdLinkMetadata(doc) + return getJSONLdLinkMetadata(dom) })() Object.assign(article, { @@ -311,7 +311,7 @@ export const parsePreparedContent = async ( domContent: preparedDocument.document, parsedContent: article, canonicalUrl, - pageType: parseOriginalContent(doc), + pageType: parseOriginalContent(dom), } } @@ -358,26 +358,26 @@ type Metadata = { export const parsePageMetadata = (html: string): Metadata | undefined => { try { - const window = parseHTML(html).window + const document = parseHTML(html).document // get open graph metadata const description = - window.document + document .querySelector("head meta[property='og:description']") ?.getAttribute('content') || '' const previewImage = - window.document + document .querySelector("head meta[property='og:image']") ?.getAttribute('content') || '' const title = - window.document + document .querySelector("head meta[property='og:title']") ?.getAttribute('content') || undefined const author = - window.document + document .querySelector("head meta[name='author']") ?.getAttribute('content') || undefined