use @omnivore/utils in text-to-speech-handler

packages/text-to-speech/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16-alpine
+FROM node:18.16
 
 # Run everything after as non-privileged user.
 WORKDIR /app
@@ -10,9 +10,12 @@ COPY .prettierrc .
 COPY .eslintrc .
 
 COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/utils/package.json ./packages/utils/package.json
 
 RUN yarn install --pure-lockfile
 
+ADD /packages/utils ./packages/utils
+RUN yarn workspace @omnivore/utils build
 ADD /packages/text-to-speech ./packages/text-to-speech
 RUN yarn workspace @omnivore/text-to-speech-handler build
 

packages/text-to-speech/package.json

@@ -36,6 +36,7 @@
   "dependencies": {
     "@google-cloud/functions-framework": "3.1.2",
     "@google-cloud/storage": "^7.0.1",
+    "@omnivore/utils": "1.0.0",
     "@sentry/serverless": "^7.77.0",
     "axios": "^0.27.2",
     "dotenv": "^16.0.1",

packages/text-to-speech/src/index.ts

@@ -4,17 +4,16 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 
 import { File, Storage } from '@google-cloud/storage'
+import { RedisDataSource } from '@omnivore/utils'
 import * as Sentry from '@sentry/serverless'
 import axios from 'axios'
 import crypto from 'crypto'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
-import Redis from 'ioredis'
 import * as jwt from 'jsonwebtoken'
 import { AzureTextToSpeech } from './azureTextToSpeech'
 import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml'
 import { OpenAITextToSpeech } from './openaiTextToSpeech'
 import { RealisticTextToSpeech } from './realisticTextToSpeech'
-import { createRedisClient } from './redis'
 import {
   SpeechMark,
   TextToSpeechInput,
@@ -115,10 +114,10 @@ const updateSpeech = async (
 }
 
 const getCharacterCountFromRedis = async (
-  redisClient: Redis,
+  redisClient: RedisDataSource,
   uid: string
 ): Promise<number> => {
-  const wordCount = await redisClient.get(`tts:charCount:${uid}`)
+  const wordCount = await redisClient.cacheClient.get(`tts:charCount:${uid}`)
   return wordCount ? parseInt(wordCount) : 0
 }
 
@@ -126,11 +125,11 @@ const getCharacterCountFromRedis = async (
 // which will be used to rate limit the request
 // expires after 1 day
 const updateCharacterCountInRedis = async (
-  redisClient: Redis,
+  redisClient: RedisDataSource,
   uid: string,
   wordCount: number
 ) => {
-  await redisClient.set(
+  await redisClient.cacheClient.set(
     `tts:charCount:${uid}`,
     wordCount.toString(),
     'EX',
@@ -241,11 +240,17 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       return res.status(401).send({ errorCode: 'UNAUTHENTICATED' })
     }
 
-    // create redis client
-    const redisClient = createRedisClient(
-      process.env.REDIS_TTS_URL,
-      process.env.REDIS_TTS_CERT
-    )
+    // create redis source
+    const redisDataSource = new RedisDataSource({
+      cache: {
+        url: process.env.REDIS_URL,
+        cert: process.env.REDIS_CERT,
+      },
+      mq: {
+        url: process.env.MQ_REDIS_URL,
+        cert: process.env.MQ_REDIS_CERT,
+      },
+    })
 
     try {
       const utteranceInput = req.body as UtteranceInput
@@ -267,7 +272,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // validate character count
       const characterCount =
-        (await getCharacterCountFromRedis(redisClient, claim.uid)) +
+        (await getCharacterCountFromRedis(redisDataSource, claim.uid)) +
         utteranceInput.text.length
       if (characterCount > MAX_CHARACTER_COUNT) {
         return res.status(429).send('RATE_LIMITED')
@@ -284,7 +289,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       // hash ssml to get the cache key
       const cacheKey = crypto.createHash('md5').update(ssml).digest('hex')
       // find audio data in cache
-      const cacheResult = await redisClient.get(cacheKey)
+      const cacheResult = await redisDataSource.cacheClient.get(cacheKey)
       if (cacheResult) {
         console.log('Cache hit')
         const { audioDataString, speechMarks }: CacheResult =
@@ -352,7 +357,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       const audioDataString = audioData.toString('hex')
       // save audio data to cache for 72 hours for mainly the newsletters
-      await redisClient.set(
+      await redisDataSource.cacheClient.set(
         cacheKey,
         JSON.stringify({ audioDataString, speechMarks }),
         'EX',
@@ -362,7 +367,11 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       console.log('Cache saved')
 
       // update character count
-      await updateCharacterCountInRedis(redisClient, claim.uid, characterCount)
+      await updateCharacterCountInRedis(
+        redisDataSource,
+        claim.uid,
+        characterCount
+      )
 
       res.send({
         idx: utteranceInput.idx,
@@ -373,7 +382,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
       console.error('Text to speech streaming error:', e)
       return res.status(500).send({ errorCodes: 'SYNTHESIZER_ERROR' })
     } finally {
-      await redisClient.quit()
+      await redisDataSource.cacheClient.quit()
       console.log('Redis Client Disconnected')
     }
   }

packages/text-to-speech/src/redis.ts

@@ -1,34 +0,0 @@
-import { Redis } from 'ioredis'
-
-export const createRedisClient = (url?: string, cert?: string) => {
-  return new Redis(url || 'redis://localhost:6379', {
-    connectTimeout: 10000, // 10 seconds
-    tls: cert
-      ? {
-          cert: cert.replace(/\\n/g, '\n'), // replace \n with new line
-          rejectUnauthorized: false, // for self-signed certs
-        }
-      : undefined,
-    reconnectOnError: (err) => {
-      const targetErrors = [/READONLY/, /ETIMEDOUT/]
-
-      targetErrors.forEach((targetError) => {
-        if (targetError.test(err.message)) {
-          // Only reconnect when the error contains the keyword
-          return true
-        }
-      })
-
-      return false
-    },
-    retryStrategy: (times) => {
-      if (times > 10) {
-        // End reconnecting after a specific number of tries and flush all commands with a individual error
-        return null
-      }
-
-      // reconnect after
-      return Math.min(times * 50, 2000)
-    },
-  })
-}