Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reduce retry

Browse Source
This commit is contained in:
Hongbo Wu
2023-08-17 17:59:24 +08:00
parent 6f315ef4d9
commit 0609043956
9 changed files with 83 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/api/package.json
View File

@ -61,7 +61,7 @@
"express-rate-limit": "^6.3.0",
"fast-safe-stringify": "^2.1.1",
"firebase-admin": "^11.5.0",
"googleapis": "^105.0.0",
"googleapis": "^125.0.0",
"graphql": "^15.3.0",
"graphql-fields": "^2.0.3",
"graphql-middleware": "^6.0.10",

13
packages/api/src/routers/svc/integrations.ts
View File

@ -45,7 +45,7 @@ export function integrationsServiceRouter() {
const { message: msgStr, expired } = readPushSubscription(req)
if (!msgStr) {
return res.status(400).send('Bad Request')
return res.status(200).send('Bad Request')
}
if (expired) {
@ -58,7 +58,7 @@ export function integrationsServiceRouter() {
const type = data.type
if (!userId) {
logger.info('No userId found in message')
res.status(400).send('Bad Request')
res.status(200).send('Bad Request')
return
}
@ -92,7 +92,7 @@ export function integrationsServiceRouter() {
}
if (!id) {
logger.info('No id found in message')
res.status(400).send('Bad Request')
res.status(200).send('Bad Request')
return
}
const page = await getPageById(id)
@ -159,13 +159,14 @@ export function integrationsServiceRouter() {
res.status(200).send('Unknown action')
return
}
res.status(200).send('OK')
} catch (err) {
logger.error('sync with integrations failed', err)
res.status(500).send(err)
return res.status(500).send(err)
}
res.status(200).send('OK')
})
// import pages from integration task handler
router.post('/import', async (req, res) => {
logger.info('start cloud task to import pages from integration')

14
packages/api/src/routers/svc/newsletters.ts
View File

@ -93,7 +93,7 @@ export function newsletterServiceRouter() {
try {
const { message, expired } = readPushSubscription(req)
if (!message) {
return res.status(400).send('Bad Request')
return res.status(200).send('Bad Request')
}
if (expired) {
@ -104,7 +104,7 @@ export function newsletterServiceRouter() {
const data = JSON.parse(message) as unknown
if (!isNewsletterMessage(data)) {
logger.error('invalid newsletter message', { data })
return res.status(400).send('Bad Request')
return res.status(200).send('Invalid Message')
}
// get user from newsletter email
@ -150,17 +150,17 @@ export function newsletterServiceRouter() {
// update received email type
await updateReceivedEmail(data.receivedEmailId, 'article')
res.status(200).send('newsletter created')
} catch (e) {
logger.error(e)
if (e instanceof SyntaxError) {
// when message is not a valid json string
res.status(400).send(e)
} else {
res.status(500).send(e)
return res.status(400).send(e)
}
return res.status(500).send(e)
}
res.status(200).send('newsletter created')
})
return router

22
packages/api/src/routers/svc/rss_feed.ts
View File

@ -13,15 +13,15 @@ export function rssFeedRouter() {
router.post('/fetchAll', async (req, res) => {
logger.info('fetch all rss feeds')
const { message: msgStr, expired } = readPushSubscription(req)
logger.info('read pubsub message', msgStr, 'has expired', expired)
if (expired) {
logger.info('discarding expired message')
return res.status(200).send('Expired')
}
try {
const { message: msgStr, expired } = readPushSubscription(req)
logger.info('read pubsub message', msgStr, 'has expired', expired)
if (expired) {
logger.info('discarding expired message')
return res.status(200).send('Expired')
}
// get all active rss feed subscriptions
const subscriptions = await getRepository(Subscription).find({
select: ['id', 'url', 'user', 'lastFetchedAt'],
@ -42,12 +42,12 @@ export function rssFeedRouter() {
}
})
)
res.send('OK')
} catch (error) {
logger.info('error fetching rss feeds', error)
res.status(500).send('Internal Server Error')
return res.status(500).send('Internal Server Error')
}
res.send('OK')
})
return router

10
packages/api/src/routers/svc/upload.ts
View File

@ -17,7 +17,7 @@ export function uploadServiceRouter() {
const { message: msgStr, expired } = readPushSubscription(req)
if (!msgStr) {
return res.status(400).send('Bad Request')
return res.status(200).send('Bad Request')
}
if (expired) {
@ -28,7 +28,7 @@ export function uploadServiceRouter() {
const data: { userId: string; type: string } = JSON.parse(msgStr)
if (!data.userId || !data.type) {
logger.info('No userId or type found in message')
return res.status(400).send('Bad Request')
return res.status(200).send('Bad Request')
}
const filePath = `${req.params.folder}/${data.type}/${
@ -42,12 +42,12 @@ export function uploadServiceRouter() {
{ contentType: 'application/json' },
env.fileUpload.gcsUploadPrivateBucket
)
res.status(200).send('OK')
} catch (err) {
logger.error('upload page data failed', err)
res.status(500).send(err)
return res.status(500).send(err)
}
res.status(200).send('OK')
})
return router

33
packages/api/src/routers/svc/webhooks.ts
View File

@ -13,25 +13,26 @@ export function webhooksServiceRouter() {
router.post('/trigger/:action', async (req, res) => {
logger.info('trigger webhook of action', req.params.action)
const { message: msgStr, expired } = readPushSubscription(req)
if (!msgStr) {
res.status(400).send('Bad Request')
return
}
if (expired) {
logger.info('discarding expired message')
res.status(200).send('Expired')
return
}
try {
const { message: msgStr, expired } = readPushSubscription(req)
if (!msgStr) {
res.status(200).send('Bad Request')
return
}
if (expired) {
logger.info('discarding expired message')
res.status(200).send('Expired')
return
}
const data = JSON.parse(msgStr)
const { userId, type } = data as { userId: string; type: string }
if (!userId || !type) {
logger.info('No userId or type found in message')
res.status(400).send('Bad Request')
res.status(200).send('Bad Request')
return
}
@ -90,12 +91,12 @@ export function webhooksServiceRouter() {
})
})
)
res.status(200).send('OK')
} catch (err) {
logger.error('trigger webhook failed', err)
res.status(500).send(err)
return res.status(500).send(err)
}
res.send('OK')
})
return router

11
packages/api/src/utils/logger.ts
View File

@ -61,10 +61,15 @@ const colors = {
debug: 'underline gray',
}
const MAX_LOG_SIZE = 250000
const googleConfigs = {
level: 'info',
logName: 'logger',
levels: config.syslog.levels,
maxEntrySize: MAX_LOG_SIZE,
useMessageField: false,
redirectToStdout: true,
}
function localConfig(id: string): ConsoleTransportOptions {
@ -100,7 +105,7 @@ const truncateObjectDeep = (object: any, length: number): any => {
const truncateDeep = (obj: any, level: number): any => {
// reach maximum call stack size
if (level >= 5) {
return obj
return {}
}
if (isString(obj) && obj.length > length) {
@ -129,8 +134,8 @@ const truncateObjectDeep = (object: any, length: number): any => {
class GcpLoggingTransport extends LoggingWinston {
log(info: any, callback: (err: Error | null, apiResponse?: any) => void) {
const sizeInfo = jsonStringify(info).length
if (sizeInfo > 250000) {
info = truncateObjectDeep(info, 5000) as never // the max length for string values is 5000
if (sizeInfo > MAX_LOG_SIZE) {
info = truncateObjectDeep(info, 500) as never // the max length for string values is 500
}
super.log(info, callback)
}

2
packages/api/src/utils/uploads.ts
View File

@ -129,7 +129,7 @@ export const uploadToBucket = async (
await storage
.bucket(selectedBucket || bucketName)
.file(filePath)
.save(data, options)
.save(data, { ...options, timeout: 30000 })
}
export const createGCSFile = (filename: string): File => {

42
yarn.lock
View File

@ -3174,9 +3174,9 @@
integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==
"@grpc/grpc-js@^1.1.8", "@grpc/grpc-js@~1.8.0":
version "1.8.11"
resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.8.11.tgz#f113f7bc197e8d6f3d3f0c6b02925c7a5da1aec4"
integrity sha512-f/xC+6Z2QKsRJ+VSSFlt4hA5KSRm+PKvMWV8kMPkMgGlFidR6PeIkXrOasIY2roe+WROM6GFQLlgDKfeEZo2YQ==
version "1.8.21"
resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.8.21.tgz#d282b122c71227859bf6c5866f4c40f4a2696513"
integrity sha512-KeyQeZpxeEBSqFVTi3q2K7PiPXmgBfECc4updA1ejCLjYmoAlvvM3ZMp5ztTDUCUQmoY3CpDxvchjO1+rFkoHg==
dependencies:
"@grpc/proto-loader" "^0.7.0"
"@types/node" ">=12.12.47"
@ -15393,6 +15393,16 @@ gaxios@^6.0.0, gaxios@^6.0.2:
is-stream "^2.0.0"
node-fetch "^2.6.9"
gaxios@^6.0.3:
version "6.1.0"
resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-6.1.0.tgz#8ab08adbf9cc600368a57545f58e004ccf831ccb"
integrity sha512-EIHuesZxNyIkUGcTQKQPMICyOpDD/bi+LJIJx+NLsSGmnS7N+xCLRX5bi4e9yAu9AlSZdVq+qlyWWVuTh/483w==
dependencies:
extend "^3.0.2"
https-proxy-agent "^7.0.1"
is-stream "^2.0.0"
node-fetch "^2.6.9"
gcp-metadata@^4.2.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.0.tgz#0423d06becdbfb9cbb8762eaacf14d5324997900"
@ -15926,25 +15936,25 @@ google-proto-files@^3.0.0:
protobufjs "^7.0.0"
walkdir "^0.4.0"
googleapis-common@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/googleapis-common/-/googleapis-common-6.0.0.tgz#cd3f811f18dbc82191530aedaafa6224333a758e"
integrity sha512-ieZouiuoyTHCckOgu7NU+n5UvA8kAzGTRRMQD+3bobCX9npnRvYDmTHZjM5lQLzf0cGz1xQ1ABCxGQ+xSsMkCw==
googleapis-common@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/googleapis-common/-/googleapis-common-7.0.0.tgz#a7b5262e320c922c25b123edea2a3958f15c3edd"
integrity sha512-58iSybJPQZ8XZNMpjrklICefuOuyJ0lMxfKmBqmaC0/xGT4SiOs4BE60LAOOGtBURy1n8fHa2X2YUNFEWWbXyQ==
dependencies:
extend "^3.0.2"
gaxios "^4.0.0"
google-auth-library "^8.0.2"
gaxios "^6.0.3"
google-auth-library "^9.0.0"
qs "^6.7.0"
url-template "^2.0.8"
uuid "^8.0.0"
uuid "^9.0.0"
googleapis@^105.0.0:
version "105.0.0"
resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-105.0.0.tgz#65ff8969cf837a08ee127ae56468ac3d96ddc9d5"
integrity sha512-wH/jU/6QpqwsjTKj4vfKZz97ne7xT7BBbKwzQEwnbsG8iH9Seyw19P+AuLJcxNNrmgblwLqfr3LORg4Okat1BQ==
googleapis@^125.0.0:
version "125.0.0"
resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-125.0.0.tgz#d084565e567081dceaf5aade0c4f5122ae53e083"
integrity sha512-KsMe3gdbiI6bj4M+Zuwcl7xL0Koz8m0kaq0XQj99YT/4zHsZdaLJqGmYMDyWI4SAScVqkW7TvQftzL7L74x1uQ==
dependencies:
google-auth-library "^8.0.2"
googleapis-common "^6.0.0"
google-auth-library "^9.0.0"
googleapis-common "^7.0.0"
got@^9.6.0:
version "9.6.0"