use different redis server for mq and cache

packages/api/src/queue-processor.ts

@@ -40,8 +40,8 @@ const main = async () => {
   const port = process.env.PORT || 3002
 
   redisDataSource.setOptions({
-    REDIS_URL: env.redis.url,
-    REDIS_CERT: env.redis.cert,
+    cache: env.redis.cache,
+    mq: env.redis.mq,
   })
 
   appDataSource.setOptions({

packages/api/src/redis_data_source.ts

@@ -1,9 +1,14 @@
 import Redis, { RedisOptions } from 'ioredis'
 import { env } from './env'
+import { logger } from './utils/logger'
 
+type RedisClientType = 'cache' | 'mq'
+type RedisDataSourceOption = {
+  url?: string
+  cert?: string
+}
 export type RedisDataSourceOptions = {
-  REDIS_URL?: string
-  REDIS_CERT?: string
+  [key in RedisClientType]: RedisDataSourceOption
 }
 
 export class RedisDataSource {
@@ -22,8 +27,9 @@ export class RedisDataSource {
   async initialize(): Promise<this> {
     if (this.isInitialized) throw 'Error already initialized'
 
-    this.redisClient = createIORedisClient('app', this.options)
-    this.workerRedisClient = createIORedisClient('worker', this.options)
+    this.redisClient = createIORedisClient('cache', this.options)
+    this.workerRedisClient =
+      createIORedisClient('mq', this.options) || this.redisClient // if mq is not defined, use cache
     this.isInitialized = true
 
     return Promise.resolve(this)
@@ -45,17 +51,21 @@ export class RedisDataSource {
 }
 
 const createIORedisClient = (
-  name: string,
+  name: RedisClientType,
   options: RedisDataSourceOptions
 ): Redis | undefined => {
-  const redisURL = options.REDIS_URL
+  const option = options[name]
+  const redisURL = option.url
   if (!redisURL) {
-    throw 'Error: no redisURL supplied'
+    logger.info(`no redisURL supplied: ${name}`)
+    return undefined
   }
+
+  const redisCert = option.cert
   const tls =
-    redisURL.startsWith('rediss://') && options.REDIS_CERT
+    redisURL.startsWith('rediss://') && redisCert
       ? {
-          ca: options.REDIS_CERT,
+          ca: redisCert,
           rejectUnauthorized: false,
         }
       : undefined
@@ -92,7 +102,6 @@ const createIORedisClient = (
   return new Redis(redisURL, redisOptions)
 }
 
-export const redisDataSource = new RedisDataSource({
-  REDIS_URL: env.redis.url,
-  REDIS_CERT: env.redis.cert,
-})
+export const redisDataSource = new RedisDataSource(
+  env.redis as RedisDataSourceOptions
+)

packages/api/src/server.ts

@@ -159,7 +159,7 @@ const main = async (): Promise<void> => {
   await appDataSource.initialize()
 
   // redis is optional for the API server
-  if (env.redis.url) {
+  if (env.redis.cache.url) {
     await redisDataSource.initialize()
   }
 

packages/api/src/util.ts

@@ -4,6 +4,11 @@
 import * as dotenv from 'dotenv'
 import os from 'os'
 
+interface redisConfig {
+  url?: string
+  cert?: string
+}
+
 export interface BackendEnv {
   pg: {
     host: string
@@ -105,8 +110,8 @@ export interface BackendEnv {
     }
   }
   redis: {
-    url?: string
-    cert?: string
+    mq: redisConfig
+    cache: redisConfig
   }
 }
 
@@ -156,6 +161,8 @@ const nullableEnvVars = [
   'SUBSCRIPTION_FEED_MAX',
   'REDIS_URL',
   'REDIS_CERT',
+  'MQ_REDIS_URL',
+  'MQ_REDIS_CERT',
   'IMPORTER_METRICS_COLLECTOR_URL',
   'INTERNAL_API_URL',
 ] // Allow some vars to be null/empty
@@ -295,8 +302,14 @@ export function getEnv(): BackendEnv {
     },
   }
   const redis = {
-    url: parse('REDIS_URL'),
-    cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    mq: {
+      url: parse('MQ_REDIS_URL'),
+      cert: parse('MQ_REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
+    cache: {
+      url: parse('REDIS_URL'),
+      cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
+    },
   }
 
   return {