diff --git a/packages/api/package.json b/packages/api/package.json index 6db8e39cf..f6d274271 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -61,7 +61,7 @@ "express-rate-limit": "^6.3.0", "fast-safe-stringify": "^2.1.1", "firebase-admin": "^11.5.0", - "googleapis": "^105.0.0", + "googleapis": "^125.0.0", "graphql": "^15.3.0", "graphql-fields": "^2.0.3", "graphql-middleware": "^6.0.10", diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index bc5468981..bb09b9d8f 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -45,7 +45,7 @@ export function integrationsServiceRouter() { const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { - return res.status(400).send('Bad Request') + return res.status(200).send('Bad Request') } if (expired) { @@ -58,7 +58,7 @@ export function integrationsServiceRouter() { const type = data.type if (!userId) { logger.info('No userId found in message') - res.status(400).send('Bad Request') + res.status(200).send('Bad Request') return } @@ -92,7 +92,7 @@ export function integrationsServiceRouter() { } if (!id) { logger.info('No id found in message') - res.status(400).send('Bad Request') + res.status(200).send('Bad Request') return } const page = await getPageById(id) @@ -159,13 +159,14 @@ export function integrationsServiceRouter() { res.status(200).send('Unknown action') return } - - res.status(200).send('OK') } catch (err) { logger.error('sync with integrations failed', err) - res.status(500).send(err) + return res.status(500).send(err) } + + res.status(200).send('OK') }) + // import pages from integration task handler router.post('/import', async (req, res) => { logger.info('start cloud task to import pages from integration') diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index 243fb7f6b..d39ea8280 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -93,7 +93,7 @@ export function newsletterServiceRouter() { try { const { message, expired } = readPushSubscription(req) if (!message) { - return res.status(400).send('Bad Request') + return res.status(200).send('Bad Request') } if (expired) { @@ -104,7 +104,7 @@ export function newsletterServiceRouter() { const data = JSON.parse(message) as unknown if (!isNewsletterMessage(data)) { logger.error('invalid newsletter message', { data }) - return res.status(400).send('Bad Request') + return res.status(200).send('Invalid Message') } // get user from newsletter email @@ -150,17 +150,17 @@ export function newsletterServiceRouter() { // update received email type await updateReceivedEmail(data.receivedEmailId, 'article') - - res.status(200).send('newsletter created') } catch (e) { logger.error(e) if (e instanceof SyntaxError) { // when message is not a valid json string - res.status(400).send(e) - } else { - res.status(500).send(e) + return res.status(400).send(e) } + + return res.status(500).send(e) } + + res.status(200).send('newsletter created') }) return router diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 72834d774..c53ee726e 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -13,15 +13,15 @@ export function rssFeedRouter() { router.post('/fetchAll', async (req, res) => { logger.info('fetch all rss feeds') - const { message: msgStr, expired } = readPushSubscription(req) - logger.info('read pubsub message', msgStr, 'has expired', expired) - - if (expired) { - logger.info('discarding expired message') - return res.status(200).send('Expired') - } - try { + const { message: msgStr, expired } = readPushSubscription(req) + logger.info('read pubsub message', msgStr, 'has expired', expired) + + if (expired) { + logger.info('discarding expired message') + return res.status(200).send('Expired') + } + // get all active rss feed subscriptions const subscriptions = await getRepository(Subscription).find({ select: ['id', 'url', 'user', 'lastFetchedAt'], @@ -42,12 +42,12 @@ export function rssFeedRouter() { } }) ) - - res.send('OK') } catch (error) { logger.info('error fetching rss feeds', error) - res.status(500).send('Internal Server Error') + return res.status(500).send('Internal Server Error') } + + res.send('OK') }) return router diff --git a/packages/api/src/routers/svc/upload.ts b/packages/api/src/routers/svc/upload.ts index 1a42fba7d..9d3167832 100644 --- a/packages/api/src/routers/svc/upload.ts +++ b/packages/api/src/routers/svc/upload.ts @@ -17,7 +17,7 @@ export function uploadServiceRouter() { const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { - return res.status(400).send('Bad Request') + return res.status(200).send('Bad Request') } if (expired) { @@ -28,7 +28,7 @@ export function uploadServiceRouter() { const data: { userId: string; type: string } = JSON.parse(msgStr) if (!data.userId || !data.type) { logger.info('No userId or type found in message') - return res.status(400).send('Bad Request') + return res.status(200).send('Bad Request') } const filePath = `${req.params.folder}/${data.type}/${ @@ -42,12 +42,12 @@ export function uploadServiceRouter() { { contentType: 'application/json' }, env.fileUpload.gcsUploadPrivateBucket ) - - res.status(200).send('OK') } catch (err) { logger.error('upload page data failed', err) - res.status(500).send(err) + return res.status(500).send(err) } + + res.status(200).send('OK') }) return router diff --git a/packages/api/src/routers/svc/webhooks.ts b/packages/api/src/routers/svc/webhooks.ts index dd596e74e..c2cdd26e0 100644 --- a/packages/api/src/routers/svc/webhooks.ts +++ b/packages/api/src/routers/svc/webhooks.ts @@ -13,25 +13,26 @@ export function webhooksServiceRouter() { router.post('/trigger/:action', async (req, res) => { logger.info('trigger webhook of action', req.params.action) - const { message: msgStr, expired } = readPushSubscription(req) - - if (!msgStr) { - res.status(400).send('Bad Request') - return - } - - if (expired) { - logger.info('discarding expired message') - res.status(200).send('Expired') - return - } try { + const { message: msgStr, expired } = readPushSubscription(req) + + if (!msgStr) { + res.status(200).send('Bad Request') + return + } + + if (expired) { + logger.info('discarding expired message') + res.status(200).send('Expired') + return + } + const data = JSON.parse(msgStr) const { userId, type } = data as { userId: string; type: string } if (!userId || !type) { logger.info('No userId or type found in message') - res.status(400).send('Bad Request') + res.status(200).send('Bad Request') return } @@ -90,12 +91,12 @@ export function webhooksServiceRouter() { }) }) ) - - res.status(200).send('OK') } catch (err) { logger.error('trigger webhook failed', err) - res.status(500).send(err) + return res.status(500).send(err) } + + res.send('OK') }) return router diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 4e198cb21..e8e592e2a 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -61,10 +61,15 @@ const colors = { debug: 'underline gray', } +const MAX_LOG_SIZE = 250000 + const googleConfigs = { level: 'info', logName: 'logger', levels: config.syslog.levels, + maxEntrySize: MAX_LOG_SIZE, + useMessageField: false, + redirectToStdout: true, } function localConfig(id: string): ConsoleTransportOptions { @@ -100,7 +105,7 @@ const truncateObjectDeep = (object: any, length: number): any => { const truncateDeep = (obj: any, level: number): any => { // reach maximum call stack size if (level >= 5) { - return obj + return {} } if (isString(obj) && obj.length > length) { @@ -129,8 +134,8 @@ const truncateObjectDeep = (object: any, length: number): any => { class GcpLoggingTransport extends LoggingWinston { log(info: any, callback: (err: Error | null, apiResponse?: any) => void) { const sizeInfo = jsonStringify(info).length - if (sizeInfo > 250000) { - info = truncateObjectDeep(info, 5000) as never // the max length for string values is 5000 + if (sizeInfo > MAX_LOG_SIZE) { + info = truncateObjectDeep(info, 500) as never // the max length for string values is 500 } super.log(info, callback) } diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index f4b709fb0..7ba3f4613 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -129,7 +129,7 @@ export const uploadToBucket = async ( await storage .bucket(selectedBucket || bucketName) .file(filePath) - .save(data, options) + .save(data, { ...options, timeout: 30000 }) } export const createGCSFile = (filename: string): File => { diff --git a/yarn.lock b/yarn.lock index 36a39832d..312bcd169 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3174,9 +3174,9 @@ integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ== "@grpc/grpc-js@^1.1.8", "@grpc/grpc-js@~1.8.0": - version "1.8.11" - resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.8.11.tgz#f113f7bc197e8d6f3d3f0c6b02925c7a5da1aec4" - integrity sha512-f/xC+6Z2QKsRJ+VSSFlt4hA5KSRm+PKvMWV8kMPkMgGlFidR6PeIkXrOasIY2roe+WROM6GFQLlgDKfeEZo2YQ== + version "1.8.21" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.8.21.tgz#d282b122c71227859bf6c5866f4c40f4a2696513" + integrity sha512-KeyQeZpxeEBSqFVTi3q2K7PiPXmgBfECc4updA1ejCLjYmoAlvvM3ZMp5ztTDUCUQmoY3CpDxvchjO1+rFkoHg== dependencies: "@grpc/proto-loader" "^0.7.0" "@types/node" ">=12.12.47" @@ -15393,6 +15393,16 @@ gaxios@^6.0.0, gaxios@^6.0.2: is-stream "^2.0.0" node-fetch "^2.6.9" +gaxios@^6.0.3: + version "6.1.0" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-6.1.0.tgz#8ab08adbf9cc600368a57545f58e004ccf831ccb" + integrity sha512-EIHuesZxNyIkUGcTQKQPMICyOpDD/bi+LJIJx+NLsSGmnS7N+xCLRX5bi4e9yAu9AlSZdVq+qlyWWVuTh/483w== + dependencies: + extend "^3.0.2" + https-proxy-agent "^7.0.1" + is-stream "^2.0.0" + node-fetch "^2.6.9" + gcp-metadata@^4.2.0: version "4.3.0" resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.0.tgz#0423d06becdbfb9cbb8762eaacf14d5324997900" @@ -15926,25 +15936,25 @@ google-proto-files@^3.0.0: protobufjs "^7.0.0" walkdir "^0.4.0" -googleapis-common@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/googleapis-common/-/googleapis-common-6.0.0.tgz#cd3f811f18dbc82191530aedaafa6224333a758e" - integrity sha512-ieZouiuoyTHCckOgu7NU+n5UvA8kAzGTRRMQD+3bobCX9npnRvYDmTHZjM5lQLzf0cGz1xQ1ABCxGQ+xSsMkCw== +googleapis-common@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/googleapis-common/-/googleapis-common-7.0.0.tgz#a7b5262e320c922c25b123edea2a3958f15c3edd" + integrity sha512-58iSybJPQZ8XZNMpjrklICefuOuyJ0lMxfKmBqmaC0/xGT4SiOs4BE60LAOOGtBURy1n8fHa2X2YUNFEWWbXyQ== dependencies: extend "^3.0.2" - gaxios "^4.0.0" - google-auth-library "^8.0.2" + gaxios "^6.0.3" + google-auth-library "^9.0.0" qs "^6.7.0" url-template "^2.0.8" - uuid "^8.0.0" + uuid "^9.0.0" -googleapis@^105.0.0: - version "105.0.0" - resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-105.0.0.tgz#65ff8969cf837a08ee127ae56468ac3d96ddc9d5" - integrity sha512-wH/jU/6QpqwsjTKj4vfKZz97ne7xT7BBbKwzQEwnbsG8iH9Seyw19P+AuLJcxNNrmgblwLqfr3LORg4Okat1BQ== +googleapis@^125.0.0: + version "125.0.0" + resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-125.0.0.tgz#d084565e567081dceaf5aade0c4f5122ae53e083" + integrity sha512-KsMe3gdbiI6bj4M+Zuwcl7xL0Koz8m0kaq0XQj99YT/4zHsZdaLJqGmYMDyWI4SAScVqkW7TvQftzL7L74x1uQ== dependencies: - google-auth-library "^8.0.2" - googleapis-common "^6.0.0" + google-auth-library "^9.0.0" + googleapis-common "^7.0.0" got@^9.6.0: version "9.6.0"