diff --git a/packages/api/package.json b/packages/api/package.json index 75cecca83..432de1b5d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -69,6 +69,7 @@ "highlightjs": "^9.16.2", "html-entities": "^2.3.2", "intercom-client": "^3.1.4", + "ioredis": "^5.3.2", "jsonwebtoken": "^8.5.1", "jwks-rsa": "^2.0.3", "linkedom": "^0.14.9", @@ -82,7 +83,6 @@ "pg": "^8.3.3", "postgrator": "^4.2.0", "private-ip": "^2.3.3", - "redis": "^4.3.1", "rss-parser": "^3.13.0", "sanitize-html": "^2.3.2", "sax": "^1.3.0", diff --git a/packages/api/src/redis.ts b/packages/api/src/redis.ts index 14559c0be..ebc2b75d6 100644 --- a/packages/api/src/redis.ts +++ b/packages/api/src/redis.ts @@ -1,18 +1,39 @@ -import { createClient } from 'redis' +import { Redis } from 'ioredis' import { env } from './env' -export const redisClient = createClient({ - url: env.redis.url, - socket: { - tls: env.redis.url?.startsWith('rediss://'), // rediss:// is the protocol for TLS - cert: env.redis.cert?.replace(/\\n/g, '\n'), // replace \n with new line - rejectUnauthorized: false, // for self-signed certs - connectTimeout: 10000, // 10 seconds - reconnectStrategy(retries: number): number | Error { - if (retries > 10) { - return new Error('Retries exhausted') - } - return 1000 - }, - }, -}) +const url = env.redis.url +const cert = env.redis.cert + +export const redisClient = url + ? new Redis(url, { + connectTimeout: 10000, // 10 seconds + tls: cert + ? { + cert, + rejectUnauthorized: false, // for self-signed certs + } + : undefined, + reconnectOnError: (err) => { + const targetErrors = [/READONLY/, /ETIMEDOUT/] + + targetErrors.forEach((targetError) => { + if (targetError.test(err.message)) { + // Only reconnect when the error contains the keyword + return true + } + }) + + return false + }, + retryStrategy: (times) => { + if (times > 10) { + // End reconnecting after a specific number of tries and flush all commands with a individual error + return null + } + + // reconnect after + return Math.min(times * 50, 2000) + }, + maxRetriesPerRequest: 1, + }) + : null diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index a6458730f..4f1fe52bb 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -159,12 +159,7 @@ const main = async (): Promise => { await appDataSource.initialize() // redis is optional - if (env.redis.url) { - redisClient.on('error', (err) => { - console.error('Redis Client Error', err) - }) - - await redisClient.connect() + if (redisClient) { console.log('Redis Client Connected:', env.redis.url) } @@ -193,6 +188,18 @@ const main = async (): Promise => { // And a workaround for node.js bug: https://github.com/nodejs/node/issues/27363 listener.headersTimeout = 640 * 1000 // 10s more than above listener.timeout = 640 * 1000 // match headersTimeout + + process.on('SIGINT', async () => { + if (redisClient) { + await redisClient.quit() + console.log('Redis client closed.') + } + + await appDataSource.destroy() + console.log('DB connection closed.') + + process.exit(0) + }) } // only call main if the file was called from the CLI and wasn't required from another module diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 339c7a26d..ddfa15cd1 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -6,9 +6,9 @@ import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' -import { env } from '../env' import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' import { createPubSubClient, EntityType } from '../pubsub' +import { redisClient } from '../redis' import { authTrx, getColumns, @@ -824,8 +824,12 @@ export const createLibraryItem = async ( ) // set recently saved item in redis if redis is enabled - if (env.redis.url) { - await setRecentlySavedItemInRedis(userId, newLibraryItem.originalUrl) + if (redisClient) { + await setRecentlySavedItemInRedis( + redisClient, + userId, + newLibraryItem.originalUrl + ) } if (skipPubSub) { diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index f37b51587..7f665fd97 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -324,7 +324,7 @@ export function getEnv(): BackendEnv { } const redis = { url: parse('REDIS_URL'), - cert: parse('REDIS_CERT'), + cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line } return { diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 304553eba..58de3d455 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import crypto from 'crypto' +import Redis from 'ioredis' import normalizeUrl from 'normalize-url' import path from 'path' import _ from 'underscore' @@ -410,6 +411,7 @@ export const getAbsoluteUrl = (url: string, baseUrl: string): string => { } export const setRecentlySavedItemInRedis = async ( + redisClient: Redis, userId: string, url: string ) => { @@ -417,10 +419,7 @@ export const setRecentlySavedItemInRedis = async ( const redisKey = `recent-saved-item:${userId}:${url}` const ttlInSeconds = 60 * 60 * 26 try { - return redisClient.set(redisKey, 1, { - EX: ttlInSeconds, - NX: true, - }) + return redisClient.set(redisKey, 1, 'EX', ttlInSeconds, 'NX') } catch (error) { logger.error('error setting recently saved item in redis', { redisKey, diff --git a/packages/api/test/global-setup.ts b/packages/api/test/global-setup.ts index b36cfa15b..b9ecb507a 100644 --- a/packages/api/test/global-setup.ts +++ b/packages/api/test/global-setup.ts @@ -1,4 +1,3 @@ -import { env } from '../src/env' import { redisClient } from '../src/redis' import { createTestConnection } from './db' import { startApolloServer } from './util' @@ -7,8 +6,7 @@ export const mochaGlobalSetup = async () => { await createTestConnection() console.log('db connection created') - if (env.redis.url) { - await redisClient.connect() + if (redisClient) { console.log('redis connection created') } diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts index 2d11b2a62..578dc8d64 100644 --- a/packages/api/test/global-teardown.ts +++ b/packages/api/test/global-teardown.ts @@ -1,5 +1,4 @@ import { appDataSource } from '../src/data_source' -import { env } from '../src/env' import { redisClient } from '../src/redis' import { stopApolloServer } from './util' @@ -10,8 +9,8 @@ export const mochaGlobalTeardown = async () => { await appDataSource.destroy() console.log('db connection closed') - if (env.redis.url) { - await redisClient.disconnect() + if (redisClient) { + await redisClient.quit() console.log('redis connection closed') } } diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index 55ead0886..ae5017f0a 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -32,11 +32,11 @@ "dependencies": { "addressparser": "^1.0.1", "axios": "^0.27.2", + "ioredis": "^5.3.2", "linkedom": "^0.14.16", "lodash": "^4.17.21", "luxon": "^3.0.4", "puppeteer-core": "^19.1.1", - "redis": "^4.3.1", "underscore": "^1.13.6", "uuid": "^9.0.0" }, diff --git a/packages/content-handler/src/redis.ts b/packages/content-handler/src/redis.ts index 2ecf068c6..e6e657e2b 100644 --- a/packages/content-handler/src/redis.ts +++ b/packages/content-handler/src/redis.ts @@ -1,32 +1,34 @@ -import { createClient } from 'redis' +import { Redis } from 'ioredis' -// explicitly create the return type of RedisClient -export type RedisClient = ReturnType - -export const createRedisClient = async ( - url?: string, - cert?: string -): Promise => { - const redisClient = createClient({ - url, - socket: { - tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS - cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line - rejectUnauthorized: false, // for self-signed certs - connectTimeout: 10000, // 10 seconds - reconnectStrategy(retries: number): number | Error { - if (retries > 10) { - return new Error('Retries exhausted') +export const createRedisClient = (url?: string, cert?: string) => { + return new Redis(url || 'redis://localhost:6379', { + connectTimeout: 10000, // 10 seconds + tls: cert + ? { + cert, + rejectUnauthorized: false, // for self-signed certs } - return 1000 - }, + : undefined, + reconnectOnError: (err) => { + const targetErrors = [/READONLY/, /ETIMEDOUT/] + + targetErrors.forEach((targetError) => { + if (targetError.test(err.message)) { + // Only reconnect when the error contains the keyword + return true + } + }) + + return false + }, + retryStrategy: (times) => { + if (times > 10) { + // End reconnecting after a specific number of tries and flush all commands with a individual error + return null + } + + // reconnect after + return Math.min(times * 50, 2000) }, }) - - redisClient.on('error', (err) => console.error('Redis Client Error', err)) - - await redisClient.connect() - console.log('Redis Client Connected:', url) - - return redisClient } diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index d5a1a4e83..36ebfd8c2 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -1,9 +1,10 @@ import axios from 'axios' +import Redis from 'ioredis' import { parseHTML } from 'linkedom' import _, { truncate } from 'lodash' import { DateTime } from 'luxon' import { ContentHandler, PreHandleResult } from '../content-handler' -import { createRedisClient, RedisClient } from '../redis' +import { createRedisClient } from '../redis' interface Tweet { url: string @@ -50,18 +51,24 @@ export class NitterHandler extends ContentHandler { this.instance = '' } - async getInstances(redisClient: RedisClient) { + async getInstances(redisClient: Redis) { // get instances by score in ascending order - const instances = await redisClient.zRange(this.REDIS_KEY, '-inf', '+inf', { - BY: 'SCORE', - }) + const instances = await redisClient.zrange( + this.REDIS_KEY, + '-inf', + '+inf', + 'BYSCORE' + ) console.debug('instances', instances) // if no instance is found, save the default instances if (instances.length === 0) { - const result = await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES, { - NX: true, // only add if the key does not exist - }) + // only add if the key does not exist + const result = await redisClient.zadd( + this.REDIS_KEY, + 'NX', + ...this.INSTANCES.map((i) => [i.score, i.value]).flat() + ) console.debug('add instances', result) // expire the key after 1 day @@ -75,11 +82,11 @@ export class NitterHandler extends ContentHandler { } async incrementInstanceScore( - redisClient: RedisClient, + redisClient: Redis, instance: string, score = 1 ) { - await redisClient.zIncrBy(this.REDIS_KEY, score, instance) + await redisClient.zincrby(this.REDIS_KEY, score, instance) } async getTweets(username: string, tweetId: string) { @@ -177,7 +184,7 @@ export class NitterHandler extends ContentHandler { } } - const redisClient = await createRedisClient( + const redisClient = createRedisClient( process.env.REDIS_URL, process.env.REDIS_CERT ) diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index b202b4e9c..013a7ad94 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -46,10 +46,10 @@ "dompurify": "^2.4.3", "fs-extra": "^11.1.0", "glob": "^8.1.0", + "ioredis": "^5.3.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.21", "nodemon": "^2.0.15", - "redis": "^4.3.1", "unzip-stream": "^0.3.1", "urlsafe-base64": "^1.0.0", "uuid": "^9.0.0" diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 599219168..d95751b47 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -2,10 +2,10 @@ import { Storage } from '@google-cloud/storage' import { Readability } from '@omnivore/readability' import * as Sentry from '@sentry/serverless' import axios from 'axios' +import Redis from 'ioredis' import * as jwt from 'jsonwebtoken' import { Stream } from 'node:stream' import * as path from 'path' -import { createClient } from 'redis' import { promisify } from 'util' import { v4 as uuid } from 'uuid' import { importCsv } from './csv' @@ -14,9 +14,6 @@ import { ImportStatus, updateMetrics } from './metrics' import { createRedisClient } from './redis' import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' -// explicitly create the return type of RedisClient -type RedisClient = ReturnType - export enum ArticleSavingRequestStatus { Failed = 'FAILED', Processing = 'PROCESSING', @@ -59,7 +56,7 @@ export type ImportContext = { countFailed: number urlHandler: UrlHandler contentHandler: ContentHandler - redisClient: RedisClient + redisClient: Redis taskId: string source: string } @@ -300,7 +297,7 @@ const contentHandler = async ( return Promise.resolve() } -const handleEvent = async (data: StorageEvent, redisClient: RedisClient) => { +const handleEvent = async (data: StorageEvent, redisClient: Redis) => { if (shouldHandle(data)) { const handler = handlerForFile(data.name) if (!handler) { @@ -367,7 +364,7 @@ export const importHandler = Sentry.GCPFunction.wrapHttpFunction( const obj = getStorageEvent(pubSubMessage) if (obj) { // create redis client - const redisClient = await createRedisClient( + const redisClient = createRedisClient( process.env.REDIS_URL, process.env.REDIS_CERT ) @@ -416,7 +413,7 @@ export const importMetricsCollector = Sentry.GCPFunction.wrapHttpFunction( return res.status(400).send('Bad Request') } - const redisClient = await createRedisClient( + const redisClient = createRedisClient( process.env.REDIS_URL, process.env.REDIS_CERT ) diff --git a/packages/import-handler/src/metrics.ts b/packages/import-handler/src/metrics.ts index 953a8a543..8b6145c15 100644 --- a/packages/import-handler/src/metrics.ts +++ b/packages/import-handler/src/metrics.ts @@ -1,10 +1,7 @@ -import { createClient } from 'redis' +import Redis from 'ioredis' import { sendImportCompletedEmail } from '.' import { lua } from './redis' -// explicitly create the return type of RedisClient -type RedisClient = ReturnType - export enum ImportStatus { STARTED = 'started', INVALID = 'invalid', @@ -31,7 +28,7 @@ interface ImportMetrics { } export const createMetrics = async ( - redisClient: RedisClient, + redisClient: Redis, userId: string, taskId: string, source: string @@ -39,7 +36,7 @@ export const createMetrics = async ( const key = `import:${userId}:${taskId}` try { // set multiple fields - await redisClient.hSet(key, { + await redisClient.hset(key, { ['start_time']: Date.now(), ['source']: source, ['state']: ImportTaskState.STARTED, @@ -50,7 +47,7 @@ export const createMetrics = async ( } export const updateMetrics = async ( - redisClient: RedisClient, + redisClient: Redis, userId: string, taskId: string, status: ImportStatus @@ -59,10 +56,13 @@ export const updateMetrics = async ( try { // use lua script to increment hash field - const state = await redisClient.evalSha(lua.sha, { - keys: [key], - arguments: [status, Date.now().toString()], - }) + const state = await redisClient.evalsha( + lua.sha, + 1, + key, + status, + Date.now().toString() + ) // if the task is finished, send email if (state == ImportTaskState.FINISHED) { @@ -77,13 +77,13 @@ export const updateMetrics = async ( } export const getMetrics = async ( - redisClient: RedisClient, + redisClient: Redis, userId: string, taskId: string ): Promise => { const key = `import:${userId}:${taskId}` try { - const metrics = await redisClient.hGetAll(key) + const metrics = await redisClient.hgetall(key) return { // convert to integer diff --git a/packages/import-handler/src/redis.ts b/packages/import-handler/src/redis.ts index 481f136b9..89798243d 100644 --- a/packages/import-handler/src/redis.ts +++ b/packages/import-handler/src/redis.ts @@ -1,6 +1,6 @@ import { readFileSync } from 'fs' +import { Redis } from 'ioredis' import path from 'path' -import { createClient } from 'redis' // load lua script export const lua = { @@ -11,31 +11,35 @@ export const lua = { sha: '', } -export const createRedisClient = async (url?: string, cert?: string) => { - const redisClient = createClient({ - url, - socket: { - tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS - cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line - rejectUnauthorized: false, // for self-signed certs - connectTimeout: 10000, // 10 seconds - reconnectStrategy(retries: number): number | Error { - if (retries > 10) { - return new Error('Retries exhausted') +export const createRedisClient = (url?: string, cert?: string) => { + return new Redis(url || 'redis://localhost:6379', { + connectTimeout: 10000, // 10 seconds + tls: cert + ? { + cert, + rejectUnauthorized: false, // for self-signed certs } - return 1000 - }, + : undefined, + reconnectOnError: (err) => { + const targetErrors = [/READONLY/, /ETIMEDOUT/] + + targetErrors.forEach((targetError) => { + if (targetError.test(err.message)) { + // Only reconnect when the error contains the keyword + return true + } + }) + + return false + }, + retryStrategy: (times) => { + if (times > 10) { + // End reconnecting after a specific number of tries and flush all commands with a individual error + return null + } + + // reconnect after + return Math.min(times * 50, 2000) }, }) - - redisClient.on('error', (err) => console.error('Redis Client Error', err)) - - await redisClient.connect() - console.log('Redis Client Connected:', url) - - // load script to redis - lua.sha = await redisClient.scriptLoad(lua.script) - console.log('Redis Lua Script Loaded', lua.sha) - - return redisClient } diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index 728cd9b82..c1debcd0f 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -27,9 +27,9 @@ "@sentry/serverless": "^7.77.0", "axios": "^1.4.0", "dotenv": "^16.0.1", + "ioredis": "^5.3.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.16.4", - "redis": "^4.3.1", "rss-parser": "^3.13.0" }, "volta": { diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 624674598..74c7130bf 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -2,17 +2,15 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' import crypto from 'crypto' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import Redis from 'ioredis' import * as jwt from 'jsonwebtoken' import { parseHTML } from 'linkedom' -import { createClient } from 'redis' import Parser, { Item } from 'rss-parser' import { promisify } from 'util' import { createRedisClient } from './redis' import { CONTENT_FETCH_URL, createCloudTask } from './task' type FolderType = 'following' | 'inbox' -// explicitly create the return type of RedisClient -type RedisClient = ReturnType interface RssFeedRequest { subscriptionIds: string[] @@ -61,7 +59,7 @@ export const isOldItem = (item: RssFeedItem, lastFetchedAt: number) => { const feedFetchFailedRedisKey = (feedUrl: string) => `feed-fetch-failure:${feedUrl}` -const isFeedBlocked = async (feedUrl: string, redisClient: RedisClient) => { +const isFeedBlocked = async (feedUrl: string, redisClient: Redis) => { const key = feedFetchFailedRedisKey(feedUrl) try { const result = await redisClient.get(key) @@ -78,10 +76,7 @@ const isFeedBlocked = async (feedUrl: string, redisClient: RedisClient) => { return false } -const incrementFeedFailure = async ( - feedUrl: string, - redisClient: RedisClient -) => { +const incrementFeedFailure = async (feedUrl: string, redisClient: Redis) => { const key = feedFetchFailedRedisKey(feedUrl) try { const result = await redisClient.incr(key) @@ -259,7 +254,7 @@ const sendUpdateSubscriptionMutation = async ( } const isItemRecentlySaved = async ( - redisClient: RedisClient, + redisClient: Redis, userId: string, url: string ) => { @@ -274,7 +269,7 @@ const createTask = async ( item: RssFeedItem, fetchContent: boolean, folder: FolderType, - redisClient: RedisClient + redisClient: Redis ) => { const isRecentlySaved = await isItemRecentlySaved( redisClient, @@ -464,7 +459,7 @@ const processSubscription = async ( fetchContent: boolean, folder: FolderType, feed: RssFeed, - redisClient: RedisClient + redisClient: Redis ) => { let lastItemFetchedAt: Date | null = null let lastValidItem: RssFeedItem | null = null @@ -606,7 +601,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( } // create redis client - const redisClient = await createRedisClient( + const redisClient = createRedisClient( process.env.REDIS_URL, process.env.REDIS_CERT ) diff --git a/packages/rss-handler/src/redis.ts b/packages/rss-handler/src/redis.ts index 350cb09f6..e6e657e2b 100644 --- a/packages/rss-handler/src/redis.ts +++ b/packages/rss-handler/src/redis.ts @@ -1,26 +1,34 @@ -import { createClient } from 'redis' +import { Redis } from 'ioredis' -export const createRedisClient = async (url?: string, cert?: string) => { - const redisClient = createClient({ - url, - socket: { - tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS - cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line - rejectUnauthorized: false, // for self-signed certs - connectTimeout: 10000, // 10 seconds - reconnectStrategy(retries: number): number | Error { - if (retries > 10) { - return new Error('Retries exhausted') +export const createRedisClient = (url?: string, cert?: string) => { + return new Redis(url || 'redis://localhost:6379', { + connectTimeout: 10000, // 10 seconds + tls: cert + ? { + cert, + rejectUnauthorized: false, // for self-signed certs } - return 1000 - }, + : undefined, + reconnectOnError: (err) => { + const targetErrors = [/READONLY/, /ETIMEDOUT/] + + targetErrors.forEach((targetError) => { + if (targetError.test(err.message)) { + // Only reconnect when the error contains the keyword + return true + } + }) + + return false + }, + retryStrategy: (times) => { + if (times > 10) { + // End reconnecting after a specific number of tries and flush all commands with a individual error + return null + } + + // reconnect after + return Math.min(times * 50, 2000) }, }) - - redisClient.on('error', (err) => console.error('Redis Client Error', err)) - - await redisClient.connect() - console.log('Redis Client Connected:', url) - - return redisClient } diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index 5cf3e209e..c240ea2bc 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -40,11 +40,11 @@ "dotenv": "^16.0.1", "fluent-ffmpeg": "^2.1.2", "html-to-text": "^8.2.1", + "ioredis": "^5.3.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.12", "microsoft-cognitiveservices-speech-sdk": "1.30", "natural": "^6.2.0", - "redis": "^4.3.1", "underscore": "^1.13.4" }, "volta": { diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index 9ff7abf4a..31afb853b 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -3,26 +3,23 @@ /* eslint-disable @typescript-eslint/no-unsafe-argument */ /* eslint-disable @typescript-eslint/no-unused-vars */ +import { File, Storage } from '@google-cloud/storage' import * as Sentry from '@sentry/serverless' import axios from 'axios' -import * as jwt from 'jsonwebtoken' -import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { AzureTextToSpeech } from './azureTextToSpeech' -import { File, Storage } from '@google-cloud/storage' -import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml' import crypto from 'crypto' +import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import Redis from 'ioredis' +import * as jwt from 'jsonwebtoken' +import { AzureTextToSpeech } from './azureTextToSpeech' +import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml' +import { OpenAITextToSpeech } from './openaiTextToSpeech' +import { RealisticTextToSpeech } from './realisticTextToSpeech' import { createRedisClient } from './redis' import { SpeechMark, TextToSpeechInput, TextToSpeechOutput, } from './textToSpeech' -import { createClient } from 'redis' -import { RealisticTextToSpeech } from './realisticTextToSpeech' -import { OpenAITextToSpeech } from './openaiTextToSpeech' - -// explicitly create the return type of RedisClient -type RedisClient = ReturnType interface UtteranceInput { text: string @@ -118,7 +115,7 @@ const updateSpeech = async ( } const getCharacterCountFromRedis = async ( - redisClient: RedisClient, + redisClient: Redis, uid: string ): Promise => { const wordCount = await redisClient.get(`tts:charCount:${uid}`) @@ -129,14 +126,17 @@ const getCharacterCountFromRedis = async ( // which will be used to rate limit the request // expires after 1 day const updateCharacterCountInRedis = async ( - redisClient: RedisClient, + redisClient: Redis, uid: string, wordCount: number -): Promise => { - await redisClient.set(`tts:charCount:${uid}`, wordCount.toString(), { - EX: 3600 * 24, // in seconds - NX: true, - }) +) => { + await redisClient.set( + `tts:charCount:${uid}`, + wordCount.toString(), + 'EX', + 86400, // 1 day in seconds + 'NX' + ) } export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( @@ -240,7 +240,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( } // create redis client - const redisClient = await createRedisClient( + const redisClient = createRedisClient( process.env.REDIS_URL, process.env.REDIS_CERT ) @@ -353,10 +353,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( await redisClient.set( cacheKey, JSON.stringify({ audioDataString, speechMarks }), - { - EX: 3600 * 72, // in seconds - NX: true, - } + 'EX', + 3600 * 72, + 'NX' ) console.log('Cache saved') diff --git a/packages/text-to-speech/src/redis.ts b/packages/text-to-speech/src/redis.ts index 350cb09f6..e6e657e2b 100644 --- a/packages/text-to-speech/src/redis.ts +++ b/packages/text-to-speech/src/redis.ts @@ -1,26 +1,34 @@ -import { createClient } from 'redis' +import { Redis } from 'ioredis' -export const createRedisClient = async (url?: string, cert?: string) => { - const redisClient = createClient({ - url, - socket: { - tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS - cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line - rejectUnauthorized: false, // for self-signed certs - connectTimeout: 10000, // 10 seconds - reconnectStrategy(retries: number): number | Error { - if (retries > 10) { - return new Error('Retries exhausted') +export const createRedisClient = (url?: string, cert?: string) => { + return new Redis(url || 'redis://localhost:6379', { + connectTimeout: 10000, // 10 seconds + tls: cert + ? { + cert, + rejectUnauthorized: false, // for self-signed certs } - return 1000 - }, + : undefined, + reconnectOnError: (err) => { + const targetErrors = [/READONLY/, /ETIMEDOUT/] + + targetErrors.forEach((targetError) => { + if (targetError.test(err.message)) { + // Only reconnect when the error contains the keyword + return true + } + }) + + return false + }, + retryStrategy: (times) => { + if (times > 10) { + // End reconnecting after a specific number of tries and flush all commands with a individual error + return null + } + + // reconnect after + return Math.min(times * 50, 2000) }, }) - - redisClient.on('error', (err) => console.error('Redis Client Error', err)) - - await redisClient.connect() - console.log('Redis Client Connected:', url) - - return redisClient } diff --git a/yarn.lock b/yarn.lock index af6a98e0d..4512cf5f5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3257,6 +3257,11 @@ resolved "https://registry.yarnpkg.com/@icons/material/-/material-0.2.4.tgz#e90c9f71768b3736e76d7dd6783fc6c2afa88bc8" integrity sha512-QPcGmICAPbGLGb6F/yNf/KzKqvFx8z5qx3D1yFqVAjoFmXK35EgyW+cJ57Te3CNsmzblwtzakLGFqHPqrfb4Tw== +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" @@ -5424,40 +5429,6 @@ resolved "https://registry.yarnpkg.com/@reach/observe-rect/-/observe-rect-1.2.0.tgz#d7a6013b8aafcc64c778a0ccb83355a11204d3b2" integrity sha512-Ba7HmkFgfQxZqqaeIWWkNK0rEhpxVQHIoVyW1YDSkGsGIXzcaW4deC8B0pZrNSSyLTdIk7y+5olKt5+g0GmFIQ== -"@redis/bloom@1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf" - integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw== - -"@redis/client@1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77" - integrity sha512-XCFV60nloXAefDsPnYMjHGtvbtHR8fV5Om8cQ0JYqTNbWcQo/4AryzJ2luRj4blveWazRK/j40gES8M7Cp6cfQ== - dependencies: - cluster-key-slot "1.1.0" - generic-pool "3.8.2" - yallist "4.0.0" - -"@redis/graph@1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99" - integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ== - -"@redis/json@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" - integrity sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw== - -"@redis/search@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df" - integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ== - -"@redis/time-series@1.0.3": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" - integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== - "@remusao/guess-url-type@^1.1.2": version "1.2.1" resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" @@ -11624,10 +11595,10 @@ clsx@^1.1.1: resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188" integrity sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA== -cluster-key-slot@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" - integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== +cluster-key-slot@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== cmd-shim@6.0.1: version "6.0.1" @@ -13055,6 +13026,11 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -15803,11 +15779,6 @@ gcp-metadata@^6.0.0: gaxios "^6.0.0" json-bigint "^1.0.0" -generic-pool@3.8.2: - version "3.8.2" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" - integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== - gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -17627,6 +17598,21 @@ invariant@^2.2.4: dependencies: loose-envify "^1.0.0" +ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ip-regex@^4.0.0, ip-regex@^4.1.0, ip-regex@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.3.0.tgz#687275ab0f57fa76978ff8f4dddc8a23d5990db5" @@ -20196,6 +20182,11 @@ lodash.debounce@^4.0.8: resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + lodash.escaperegexp@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347" @@ -20221,6 +20212,11 @@ lodash.includes@^4.3.0: resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8= +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + lodash.isboolean@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" @@ -26093,17 +26089,17 @@ redeyed@~2.1.0: dependencies: esprima "~4.0.0" -redis@^4.3.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.3.1.tgz#290532a0c22221e05e991162ac4dca1e1b2ff6da" - integrity sha512-cM7yFU5CA6zyCF7N/+SSTcSJQSRMEKN0k0Whhu6J7n9mmXRoXugfWDBo5iOzGwABmsWKSwGPTU5J4Bxbl+0mrA== +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== dependencies: - "@redis/bloom" "1.0.2" - "@redis/client" "1.3.0" - "@redis/graph" "1.0.1" - "@redis/json" "1.0.4" - "@redis/search" "1.1.0" - "@redis/time-series" "1.0.3" + redis-errors "^1.0.0" reflect-metadata@^0.1.13: version "0.1.13" @@ -27803,6 +27799,11 @@ stacktrace-parser@^0.1.10: dependencies: type-fest "^0.7.1" +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + state-toggle@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe" @@ -30940,11 +30941,6 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yallist@4.0.0, yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yallist@^2.0.0, yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" @@ -30955,6 +30951,11 @@ yallist@^3.0.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yaml-ast-parser@^0.0.43: version "0.0.43" resolved "https://registry.yarnpkg.com/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb"