Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add timeout and content length to the axios requests

Browse Source
This commit is contained in:
Hongbo Wu
2023-06-12 21:31:35 +08:00
parent a4449e6f93
commit 4a438f45f5
3 changed files with 77 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/import-handler/src/index.ts
View File

@ -223,15 +223,29 @@ const sendSavePageMutation = async (userId: string, input: unknown) => {
})
const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string
const response = await axios.post(`${REST_BACKEND_ENDPOINT}/graphql`, data, {
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
})
try {
const response = await axios.post(
`${REST_BACKEND_ENDPOINT}/graphql`,
data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
timeout: 30000, // 30s
}
)
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
return !!response.data.data.savePage
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
return !!response.data.data.savePage
} catch (error) {
if (axios.isAxiosError(error)) {
console.error('save page mutation error', error.message)
} else {
console.error(error)
}
return false
}
}
const contentHandler = async (

2
packages/puppeteer-parse/index.js
View File

@ -46,7 +46,7 @@ const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/pl
const IMPORTER_METRICS_COLLECTOR_URL = process.env.IMPORTER_METRICS_COLLECTOR_URL;
const REQUEST_TIMEOUT = 30000;
const REQUEST_TIMEOUT = 30000; // 30 seconds
const userAgentForUrl = (url) => {
try {

78
packages/thumbnail-handler/src/index.ts
View File

@ -40,8 +40,12 @@ Sentry.GCPFunction.init({
})
const signToken = promisify(jwt.sign)
const REQUEST_TIMEOUT = 30000 // 30s
const articleQuery = async (userId: string, slug: string): Promise<Page> => {
const articleQuery = async (
userId: string,
slug: string
): Promise<Page | null> => {
const JWT_SECRET = process.env.JWT_SECRET
const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT
@ -71,18 +75,28 @@ const articleQuery = async (userId: string, slug: string): Promise<Page> => {
})
const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string
const response = await axios.post<ArticleResponse>(
`${REST_BACKEND_ENDPOINT}/graphql`,
data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
}
)
try {
const response = await axios.post<ArticleResponse>(
`${REST_BACKEND_ENDPOINT}/graphql`,
data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
timeout: REQUEST_TIMEOUT,
}
)
return response.data.data.article.article
return response.data.data.article.article
} catch (error) {
if (axios.isAxiosError(error)) {
console.error('article query error', error.message)
} else {
console.error(error)
}
return null
}
}
const updatePageMutation = async (
@ -119,18 +133,28 @@ const updatePageMutation = async (
})
const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string
const response = await axios.post<UpdatePageResponse>(
`${REST_BACKEND_ENDPOINT}/graphql`,
data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
}
)
try {
const response = await axios.post<UpdatePageResponse>(
`${REST_BACKEND_ENDPOINT}/graphql`,
data,
{
headers: {
Cookie: `auth=${auth};`,
'Content-Type': 'application/json',
},
timeout: REQUEST_TIMEOUT,
}
)
return !!response.data.data.updatePage
return !!response.data.data.updatePage
} catch (error) {
if (axios.isAxiosError(error)) {
console.error('update page mutation error', error.message)
} else {
console.error(error)
}
return false
}
}
const isThumbnailRequest = (body: any): body is ThumbnailRequest => {
@ -142,6 +166,8 @@ const getImageSize = async (url: string): Promise<[number, number] | null> => {
// get image file by url
const response = await axios.get(url, {
responseType: 'arraybuffer',
timeout: 5000, // 5s
maxContentLength: 10000000, // 10mb
})
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
@ -265,7 +291,11 @@ export const thumbnailHandler = Sentry.GCPFunction.wrapHttpFunction(
}
const page = await articleQuery(uid, slug)
console.debug('find page', page.id)
if (!page) {
console.info('page not found')
return res.status(200).send('NOT_FOUND')
}
// update page with thumbnail if not already set
if (page.image) {
console.debug('thumbnail already set')