Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix importer metrics not updated when failed to catch invalid url in the list

Browse Source
This commit is contained in:
Hongbo Wu
2023-11-28 12:14:27 +08:00
parent e20ff74b2d
commit b10b704da3
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/puppeteer-parse/index.js
View File

@ -432,6 +432,7 @@ async function fetchContent(req, res) {
} catch (e) {
logRecord.error = e.message;
console.error(`Error while retrieving page`, logRecord);
statusCode = 500;
// fallback to scrapingbee for non pdf content
if (url && contentType !== 'application/pdf') {
@ -442,6 +443,7 @@ async function fetchContent(req, res) {
content = sbResult.domContent;
title = sbResult.title;
logRecord.fetchContentTime = Date.now() - fetchStartTime;
statusCode = 200;
}
} finally {
// close browser context if it was opened
@ -492,7 +494,7 @@ async function fetchContent(req, res) {
// mark import failed on the last failed retry
const retryCount = req.headers['x-cloudtasks-taskretrycount'];
if (retryCount == MAX_RETRY_COUNT) {
console.debug('max retry count reached');
console.info('max retry count reached');
importStatus = importStatus || 'failed';
}