Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix last two tweets not saved in a long thread by waiting for document loaded after scrolling

Browse Source
This commit is contained in:
Hongbo Wu
2023-01-12 22:03:29 +08:00
parent 7bfd36e308
commit 145a8b9730
2 changed files with 16 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/content-handler/src/websites/twitter-handler.ts
View File

@ -274,7 +274,7 @@ const getTweetIds = async (
}
window.scrollBy(0, distance)
await waitFor(100)
await waitFor(500)
currentHeight += distance
}
@ -364,6 +364,7 @@ export class TwitterHandler extends ContentHandler {
`
const content = `
<html>
<head>
<meta property="og:image" content="${authorImage}" />
<meta property="og:image:secure_url" content="${authorImage}" />
@ -375,7 +376,8 @@ export class TwitterHandler extends ContentHandler {
${tweetsContent}
${tweetUrl}
</div>
</body>`
</body>
</html>`
return { content, url, title }
}

13
packages/puppeteer-parse/index.js
View File

@ -348,7 +348,18 @@ async function fetchContent(req, res) {
const content = sbResult.domContent;
logRecord.fetchContentTime = Date.now() - functionStartTime;
const readabilityResult = content ? (await getReadabilityResult(url, content)) : null;
let readabilityResult = null;
if (content) {
let document = parseHTML(content).document;
// preParse content
const preParsedDom = await preParseContent(sbUrl, document)
if (preParsedDom) {
document = preParsedDom
}
readabilityResult = await getReadabilityResult(url, document);
}
const apiResponse = await sendSavePageMutation(userId, {
url: finalUrl,