From 5f239d2dcbb10dcfdf2baa4230bfe23e54ecff27 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 9 Apr 2024 16:49:03 +0800 Subject: [PATCH] update graceful shutdown --- packages/api/src/apollo.ts | 14 ++++++++++- packages/api/src/queue-processor.ts | 11 +++++++++ packages/api/src/server.ts | 36 ++++++++++------------------ packages/api/test/global-teardown.ts | 12 +++++----- packages/api/test/util.ts | 4 +++- 5 files changed, 46 insertions(+), 31 deletions(-) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index 2559197ba..38420daa1 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -28,6 +28,7 @@ import { SetClaimsRole } from './utils/dictionary' import { logger } from './utils/logger' import { ReadingProgressDataSource } from './datasources/reading_progress_data_source' import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter' +import { ApolloServerPlugin } from 'apollo-server-plugin-base' const signToken = promisify(jwt.sign) const pubsub = createPubSubClient() @@ -112,10 +113,20 @@ export function makeApolloServer(app: Express): ApolloServer { }, }) + // enforce usage limits for the API + const usageLimitPlugin = (): ApolloServerPlugin => { + return { + async requestDidStart(contextValue) { + // get graphql query from the request + console.log(contextValue) + }, + } + } + const apollo = new ApolloServer({ schema: schema, context: contextFunc, - plugins: [promExporter], + plugins: [promExporter, usageLimitPlugin], formatError: (err) => { logger.info('server error', err) Sentry.captureException(err) @@ -124,6 +135,7 @@ export function makeApolloServer(app: Express): ApolloServer { }, introspection: env.dev.isLocal, persistedQueries: false, + stopOnTerminationSignals: false, // we handle this ourselves }) return apollo diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index aab69d3dc..d13e934de 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -304,8 +304,19 @@ const main = async () => { const gracefulShutdown = async (signal: string) => { console.log(`[queue-processor]: Received ${signal}, closing server...`) + await new Promise((resolve) => { + server.close((err) => { + console.log('[queue-processor]: Express server closed') + if (err) { + console.log('[queue-processor]: error stopping server', { err }) + } + + resolve() + }) + }) await worker.close() await redisDataSource.shutdown() + await appDataSource.destroy() process.exit(0) } diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 473df7a79..2c945581c 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -47,11 +47,7 @@ import { apiLimiter, authLimiter } from './utils/rate_limit' const PORT = process.env.PORT || 4000 -export const createApp = (): { - app: Express - apollo: ApolloServer - httpServer: Server -} => { +export const createApp = (): Express => { const app = express() // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -136,10 +132,7 @@ export const createApp = (): { res.end(await prom.register.metrics()) }) - const apollo = makeApolloServer(app) - const httpServer = createServer(app) - - return { app, apollo, httpServer } + return app } const main = async (): Promise => { @@ -154,19 +147,17 @@ const main = async (): Promise => { await redisDataSource.initialize() } - const { app, apollo, httpServer } = createApp() - + const app = createApp() + const apollo = makeApolloServer(app) await apollo.start() apollo.applyMiddleware({ app, path: '/api/graphql', cors: corsConfig }) - if (!env.dev.isLocal) { - const mwLogger = loggers.get('express', { levels: config.syslog.levels }) - const transport = buildLoggerTransport('express') - const mw = await lw.express.makeMiddleware(mwLogger, transport) - app.use(mw) - } + const mwLogger = loggers.get('express', { levels: config.syslog.levels }) + const transport = buildLoggerTransport('express') + const mw = await lw.express.makeMiddleware(mwLogger, transport) + app.use(mw) - const listener = httpServer.listen({ port: PORT }, async () => { + const listener = app.listen({ port: PORT }, async () => { const logger = buildLogger('app.dispatch') logger.notice(`🚀 Server ready at ${apollo.graphqlPath}`) }) @@ -181,15 +172,14 @@ const main = async (): Promise => { listener.timeout = 640 * 1000 // match headersTimeout const gracefulShutdown = async (signal: string) => { + console.log(`[api]: Received ${signal}, closing server...`) + await apollo.stop() + console.log('[api]: Apollo server stopped') + console.log('[posthog]: flushing events') await analytics.shutdownAsync() console.log('[posthog]: events flushed') - console.log(`[api]: Received ${signal}, closing server...`) - - await apollo.stop() - console.log('[api]: Apollo server stopped') - await new Promise((resolve) => { listener.close((err) => { console.log('[api]: Express listener closed') diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts index 922d5db46..383fd6e36 100644 --- a/packages/api/test/global-teardown.ts +++ b/packages/api/test/global-teardown.ts @@ -7,16 +7,16 @@ export const mochaGlobalTeardown = async () => { await stopApolloServer() console.log('apollo server stopped') - await appDataSource.destroy() - console.log('db connection closed') - if (env.redis.cache.url) { - await redisDataSource.shutdown() - console.log('redis connection closed') - if (redisDataSource.workerRedisClient) { await stopWorker() console.log('worker closed') } + + await redisDataSource.shutdown() + console.log('redis connection closed') } + + await appDataSource.destroy() + console.log('db connection closed') } diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index 6717fdbe1..ee1c30bdc 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -2,11 +2,13 @@ import { ConnectionOptions, Job, QueueEvents, Worker } from 'bullmq' import { nanoid } from 'nanoid' import supertest from 'supertest' import { v4 } from 'uuid' +import { makeApolloServer } from '../src/apollo' import { createWorker, QUEUE_NAME } from '../src/queue-processor' import { createApp } from '../src/server' import { corsConfig } from '../src/utils/corsConfig' -const { app, apollo } = createApp() +const app = createApp() +const apollo = makeApolloServer(app) export const request = supertest(app) let worker: Worker let queueEvents: QueueEvents