Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mark import failed on the last failed retry

Browse Source
This commit is contained in:
Hongbo Wu
2023-06-13 17:30:26 +08:00
parent 5d17a4560f
commit 9fbe32b219
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/puppeteer-parse/index.js
View File

@ -47,6 +47,7 @@ const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/pl
const IMPORTER_METRICS_COLLECTOR_URL = process.env.IMPORTER_METRICS_COLLECTOR_URL;
const REQUEST_TIMEOUT = 30000; // 30 seconds
const MAX_RETRY_COUNT = process.env.MAX_RETRY_COUNT || '1';
const userAgentForUrl = (url) => {
try {
@ -457,6 +458,12 @@ async function fetchContent(req, res) {
logRecord.totalTime = Date.now() - functionStartTime;
console.info(`parse-page`, logRecord);
// mark import failed on the last failed retry
const retryCount = req.headers['x-cloudtasks-taskretrycount'];
if (retryCount == MAX_RETRY_COUNT) {
importStatus = importStatus || 'failed';
}
// send import status to update the metrics
if (taskId && importStatus) {
await sendImportStatusUpdate(userId, taskId, importStatus);