From be0a2e8c52e7123c96153fd7058e5a8481304c94 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 25 Jan 2024 17:20:26 +0800
Subject: [PATCH] use different redis server for mq and cache

---
 packages/api/src/queue-processor.ts   |  4 +--
 packages/api/src/redis_data_source.ts | 35 +++++++++++++++++----------
 packages/api/src/server.ts            |  2 +-
 packages/api/src/util.ts              | 21 +++++++++++++---
 4 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 2810ac6d6..d99d79f10 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -40,8 +40,8 @@ const main = async () => {
   const port = process.env.PORT || 3002
 
   redisDataSource.setOptions({
-    REDIS_URL: env.redis.url,
-    REDIS_CERT: env.redis.cert,
+    cache: env.redis.cache,
+    mq: env.redis.mq,
   })
 
   appDataSource.setOptions({
diff --git a/packages/api/src/redis_data_source.ts b/packages/api/src/redis_data_source.ts
index ef7cf55e3..26da7d9fe 100644
--- a/packages/api/src/redis_data_source.ts
+++ b/packages/api/src/redis_data_source.ts
@@ -1,9 +1,14 @@
 import Redis, { RedisOptions } from 'ioredis'
 import { env } from './env'
+import { logger } from './utils/logger'
 
+type RedisClientType = 'cache' | 'mq'
+type RedisDataSourceOption = {
+  url?: string
+  cert?: string
+}
 export type RedisDataSourceOptions = {
-  REDIS_URL?: string
-  REDIS_CERT?: string
+  [key in RedisClientType]: RedisDataSourceOption
 }
 
 export class RedisDataSource {
@@ -22,8 +27,9 @@ export class RedisDataSource {
   async initialize(): Promise<this> {
     if (this.isInitialized) throw 'Error already initialized'
 
-    this.redisClient = createIORedisClient('app', this.options)
-    this.workerRedisClient = createIORedisClient('worker', this.options)
+    this.redisClient = createIORedisClient('cache', this.options)
+    this.workerRedisClient =
+      createIORedisClient('mq', this.options) || this.redisClient // if mq is not defined, use cache
     this.isInitialized = true
 
     return Promise.resolve(this)
@@ -45,17 +51,21 @@ export class RedisDataSource {
 }
 
 const createIORedisClient = (
-  name: string,
+  name: RedisClientType,
   options: RedisDataSourceOptions
 ): Redis | undefined => {
-  const redisURL = options.REDIS_URL
+  const option = options[name]
+  const redisURL = option.url
   if (!redisURL) {
-    throw 'Error: no redisURL supplied'
+    logger.info(`no redisURL supplied: ${name}`)
+    return undefined
   }
+
+  const redisCert = option.cert
   const tls =
-    redisURL.startsWith('rediss://') && options.REDIS_CERT
+    redisURL.startsWith('rediss://') && redisCert
       ? {
-          ca: options.REDIS_CERT,
+          ca: redisCert,
           rejectUnauthorized: false,
         }
       : undefined
@@ -92,7 +102,6 @@ const createIORedisClient = (
   return new Redis(redisURL, redisOptions)
 }
 
-export const redisDataSource = new RedisDataSource({
-  REDIS_URL: env.redis.url,
-  REDIS_CERT: env.redis.cert,
-})
+export const redisDataSource = new RedisDataSource(
+  env.redis as RedisDataSourceOptions
+)
diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts
index e52b0312b..58a7e4d2b 100755
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@@ -159,7 +159,7 @@ const main = async (): Promise<void> => {
   await appDataSource.initialize()
 
   // redis is optional for the API server
-  if (env.redis.url) {
+  if (env.redis.cache.url) {
     await redisDataSource.initialize()
   }
 
diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts
index 691b58063..3f1570c26 100755
--- a/packages/api/src/util.ts
+++ b/packages/api/src/util.ts
@@ -4,6 +4,11 @@
 import * as dotenv from 'dotenv'
 import os from 'os'
 
+interface redisConfig {
+  url?: string
+  cert?: string
+}
+
 export interface BackendEnv {
   pg: {
     host: string
@@ -105,8 +110,8 @@ export interface BackendEnv {
     }
   }
   redis: {
-    url?: string
-    cert?: string
+    mq: redisConfig
+    cache: redisConfig
   }
 }
 
@@ -156,6 +161,8 @@ const nullableEnvVars = [
   'SUBSCRIPTION_FEED_MAX',
   'REDIS_URL',
   'REDIS_CERT',
+  'MQ_REDIS_URL',
+  'MQ_REDIS_CERT',
   'IMPORTER_METRICS_COLLECTOR_URL',
   'INTERNAL_API_URL',
 ] // Allow some vars to be null/empty
@@ -295,8 +302,14 @@ export function getEnv(): BackendEnv {
     },
   }
   const redis = {
-    url: parse('REDIS_URL'),
-    cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    mq: {
+      url: parse('MQ_REDIS_URL'),
+      cert: parse('MQ_REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
+    cache: {
+      url: parse('REDIS_URL'),
+      cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
   }
 
   return {