Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'omnivore-app:main' into main

Browse Source
This commit is contained in:
Sebastian Jäger
2024-01-17 15:35:22 +01:00
committed by GitHub
parent e44616b01d 9c3f241c24
commit bce26b9e54
22 changed files with 309 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/api/package.json
View File

@ -69,6 +69,7 @@
"highlightjs": "^9.16.2",
"html-entities": "^2.3.2",
"intercom-client": "^3.1.4",
"ioredis": "^5.3.2",
"jsonwebtoken": "^8.5.1",
"jwks-rsa": "^2.0.3",
"linkedom": "^0.14.9",
@ -82,7 +83,6 @@
"pg": "^8.3.3",
"postgrator": "^4.2.0",
"private-ip": "^2.3.3",
"redis": "^4.3.1",
"rss-parser": "^3.13.0",
"sanitize-html": "^2.3.2",
"sax": "^1.3.0",

53
packages/api/src/redis.ts
View File

@ -1,18 +1,39 @@
import { createClient } from 'redis'
import { Redis } from 'ioredis'
import { env } from './env'
export const redisClient = createClient({
url: env.redis.url,
socket: {
tls: env.redis.url?.startsWith('rediss://'), // rediss:// is the protocol for TLS
cert: env.redis.cert?.replace(/\\n/g, '\n'), // replace \n with new line
rejectUnauthorized: false, // for self-signed certs
connectTimeout: 10000, // 10 seconds
reconnectStrategy(retries: number): number | Error {
if (retries > 10) {
return new Error('Retries exhausted')
}
return 1000
},
},
})
const url = env.redis.url
const cert = env.redis.cert
export const redisClient = url
? new Redis(url, {
connectTimeout: 10000, // 10 seconds
tls: cert
? {
cert,
rejectUnauthorized: false, // for self-signed certs
}
: undefined,
reconnectOnError: (err) => {
const targetErrors = [/READONLY/, /ETIMEDOUT/]
targetErrors.forEach((targetError) => {
if (targetError.test(err.message)) {
// Only reconnect when the error contains the keyword
return true
}
})
return false
},
retryStrategy: (times) => {
if (times > 10) {
// End reconnecting after a specific number of tries and flush all commands with a individual error
return null
}
// reconnect after
return Math.min(times * 50, 2000)
},
maxRetriesPerRequest: 1,
})
: null

19
packages/api/src/server.ts
View File

@ -159,12 +159,7 @@ const main = async (): Promise<void> => {
await appDataSource.initialize()
// redis is optional
if (env.redis.url) {
redisClient.on('error', (err) => {
console.error('Redis Client Error', err)
})
await redisClient.connect()
if (redisClient) {
console.log('Redis Client Connected:', env.redis.url)
}
@ -193,6 +188,18 @@ const main = async (): Promise<void> => {
// And a workaround for node.js bug: https://github.com/nodejs/node/issues/27363
listener.headersTimeout = 640 * 1000 // 10s more than above
listener.timeout = 640 * 1000 // match headersTimeout
process.on('SIGINT', async () => {
if (redisClient) {
await redisClient.quit()
console.log('Redis client closed.')
}
await appDataSource.destroy()
console.log('DB connection closed.')
process.exit(0)
})
}
// only call main if the file was called from the CLI and wasn't required from another module

10
packages/api/src/services/library_item.ts
View File

@ -6,9 +6,9 @@ import { EntityLabel } from '../entity/entity_label'
import { Highlight } from '../entity/highlight'
import { Label } from '../entity/label'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { env } from '../env'
import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql'
import { createPubSubClient, EntityType } from '../pubsub'
import { redisClient } from '../redis'
import {
authTrx,
getColumns,
@ -824,8 +824,12 @@ export const createLibraryItem = async (
)
// set recently saved item in redis if redis is enabled
if (env.redis.url) {
await setRecentlySavedItemInRedis(userId, newLibraryItem.originalUrl)
if (redisClient) {
await setRecentlySavedItemInRedis(
redisClient,
userId,
newLibraryItem.originalUrl
)
}
if (skipPubSub) {

2
packages/api/src/util.ts
View File

@ -324,7 +324,7 @@ export function getEnv(): BackendEnv {
}
const redis = {
url: parse('REDIS_URL'),
cert: parse('REDIS_CERT'),
cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
}
return {

7
packages/api/src/utils/helpers.ts
View File

@ -1,5 +1,6 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import crypto from 'crypto'
import Redis from 'ioredis'
import normalizeUrl from 'normalize-url'
import path from 'path'
import _ from 'underscore'
@ -410,6 +411,7 @@ export const getAbsoluteUrl = (url: string, baseUrl: string): string => {
}
export const setRecentlySavedItemInRedis = async (
redisClient: Redis,
userId: string,
url: string
) => {
@ -417,10 +419,7 @@ export const setRecentlySavedItemInRedis = async (
const redisKey = `recent-saved-item:${userId}:${url}`
const ttlInSeconds = 60 * 60 * 26
try {
return redisClient.set(redisKey, 1, {
EX: ttlInSeconds,
NX: true,
})
return redisClient.set(redisKey, 1, 'EX', ttlInSeconds, 'NX')
} catch (error) {
logger.error('error setting recently saved item in redis', {
redisKey,

4
packages/api/test/global-setup.ts
View File

@ -1,4 +1,3 @@
import { env } from '../src/env'
import { redisClient } from '../src/redis'
import { createTestConnection } from './db'
import { startApolloServer } from './util'
@ -7,8 +6,7 @@ export const mochaGlobalSetup = async () => {
await createTestConnection()
console.log('db connection created')
if (env.redis.url) {
await redisClient.connect()
if (redisClient) {
console.log('redis connection created')
}

5
packages/api/test/global-teardown.ts
View File

@ -1,5 +1,4 @@
import { appDataSource } from '../src/data_source'
import { env } from '../src/env'
import { redisClient } from '../src/redis'
import { stopApolloServer } from './util'
@ -10,8 +9,8 @@ export const mochaGlobalTeardown = async () => {
await appDataSource.destroy()
console.log('db connection closed')
if (env.redis.url) {
await redisClient.disconnect()
if (redisClient) {
await redisClient.quit()
console.log('redis connection closed')
}
}

2
packages/content-handler/package.json
View File

@ -32,11 +32,11 @@
"dependencies": {
"addressparser": "^1.0.1",
"axios": "^0.27.2",
"ioredis": "^5.3.2",
"linkedom": "^0.14.16",
"lodash": "^4.17.21",
"luxon": "^3.0.4",
"puppeteer-core": "^19.1.1",
"redis": "^4.3.1",
"underscore": "^1.13.6",
"uuid": "^9.0.0"
},

56
packages/content-handler/src/redis.ts
View File

@ -1,32 +1,34 @@
import { createClient } from 'redis'
import { Redis } from 'ioredis'
// explicitly create the return type of RedisClient
export type RedisClient = ReturnType<typeof createClient>
export const createRedisClient = async (
url?: string,
cert?: string
): Promise<RedisClient> => {
const redisClient = createClient({
url,
socket: {
tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS
cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line
rejectUnauthorized: false, // for self-signed certs
connectTimeout: 10000, // 10 seconds
reconnectStrategy(retries: number): number | Error {
if (retries > 10) {
return new Error('Retries exhausted')
export const createRedisClient = (url?: string, cert?: string) => {
return new Redis(url || 'redis://localhost:6379', {
connectTimeout: 10000, // 10 seconds
tls: cert
? {
cert,
rejectUnauthorized: false, // for self-signed certs
}
return 1000
},
: undefined,
reconnectOnError: (err) => {
const targetErrors = [/READONLY/, /ETIMEDOUT/]
targetErrors.forEach((targetError) => {
if (targetError.test(err.message)) {
// Only reconnect when the error contains the keyword
return true
}
})
return false
},
retryStrategy: (times) => {
if (times > 10) {
// End reconnecting after a specific number of tries and flush all commands with a individual error
return null
}
// reconnect after
return Math.min(times * 50, 2000)
},
})
redisClient.on('error', (err) => console.error('Redis Client Error', err))
await redisClient.connect()
console.log('Redis Client Connected:', url)
return redisClient
}

29
packages/content-handler/src/websites/nitter-handler.ts
View File

@ -1,9 +1,10 @@
import axios from 'axios'
import Redis from 'ioredis'
import { parseHTML } from 'linkedom'
import _, { truncate } from 'lodash'
import { DateTime } from 'luxon'
import { ContentHandler, PreHandleResult } from '../content-handler'
import { createRedisClient, RedisClient } from '../redis'
import { createRedisClient } from '../redis'
interface Tweet {
url: string
@ -50,18 +51,24 @@ export class NitterHandler extends ContentHandler {
this.instance = ''
}
async getInstances(redisClient: RedisClient) {
async getInstances(redisClient: Redis) {
// get instances by score in ascending order
const instances = await redisClient.zRange(this.REDIS_KEY, '-inf', '+inf', {
BY: 'SCORE',
})
const instances = await redisClient.zrange(
this.REDIS_KEY,
'-inf',
'+inf',
'BYSCORE'
)
console.debug('instances', instances)
// if no instance is found, save the default instances
if (instances.length === 0) {
const result = await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES, {
NX: true, // only add if the key does not exist
})
// only add if the key does not exist
const result = await redisClient.zadd(
this.REDIS_KEY,
'NX',
...this.INSTANCES.map((i) => [i.score, i.value]).flat()
)
console.debug('add instances', result)
// expire the key after 1 day
@ -75,11 +82,11 @@ export class NitterHandler extends ContentHandler {
}
async incrementInstanceScore(
redisClient: RedisClient,
redisClient: Redis,
instance: string,
score = 1
) {
await redisClient.zIncrBy(this.REDIS_KEY, score, instance)
await redisClient.zincrby(this.REDIS_KEY, score, instance)
}
async getTweets(username: string, tweetId: string) {
@ -177,7 +184,7 @@ export class NitterHandler extends ContentHandler {
}
}
const redisClient = await createRedisClient(
const redisClient = createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)

2
packages/import-handler/package.json
View File

@ -46,10 +46,10 @@
"dompurify": "^2.4.3",
"fs-extra": "^11.1.0",
"glob": "^8.1.0",
"ioredis": "^5.3.2",
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.14.21",
"nodemon": "^2.0.15",
"redis": "^4.3.1",
"unzip-stream": "^0.3.1",
"urlsafe-base64": "^1.0.0",
"uuid": "^9.0.0"

13
packages/import-handler/src/index.ts
View File

@ -2,10 +2,10 @@ import { Storage } from '@google-cloud/storage'
import { Readability } from '@omnivore/readability'
import * as Sentry from '@sentry/serverless'
import axios from 'axios'
import Redis from 'ioredis'
import * as jwt from 'jsonwebtoken'
import { Stream } from 'node:stream'
import * as path from 'path'
import { createClient } from 'redis'
import { promisify } from 'util'
import { v4 as uuid } from 'uuid'
import { importCsv } from './csv'
@ -14,9 +14,6 @@ import { ImportStatus, updateMetrics } from './metrics'
import { createRedisClient } from './redis'
import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task'
// explicitly create the return type of RedisClient
type RedisClient = ReturnType<typeof createClient>
export enum ArticleSavingRequestStatus {
Failed = 'FAILED',
Processing = 'PROCESSING',
@ -59,7 +56,7 @@ export type ImportContext = {
countFailed: number
urlHandler: UrlHandler
contentHandler: ContentHandler
redisClient: RedisClient
redisClient: Redis
taskId: string
source: string
}
@ -300,7 +297,7 @@ const contentHandler = async (
return Promise.resolve()
}
const handleEvent = async (data: StorageEvent, redisClient: RedisClient) => {
const handleEvent = async (data: StorageEvent, redisClient: Redis) => {
if (shouldHandle(data)) {
const handler = handlerForFile(data.name)
if (!handler) {
@ -367,7 +364,7 @@ export const importHandler = Sentry.GCPFunction.wrapHttpFunction(
const obj = getStorageEvent(pubSubMessage)
if (obj) {
// create redis client
const redisClient = await createRedisClient(
const redisClient = createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)
@ -416,7 +413,7 @@ export const importMetricsCollector = Sentry.GCPFunction.wrapHttpFunction(
return res.status(400).send('Bad Request')
}
const redisClient = await createRedisClient(
const redisClient = createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)

26
packages/import-handler/src/metrics.ts
View File

@ -1,10 +1,7 @@
import { createClient } from 'redis'
import Redis from 'ioredis'
import { sendImportCompletedEmail } from '.'
import { lua } from './redis'
// explicitly create the return type of RedisClient
type RedisClient = ReturnType<typeof createClient>
export enum ImportStatus {
STARTED = 'started',
INVALID = 'invalid',
@ -31,7 +28,7 @@ interface ImportMetrics {
}
export const createMetrics = async (
redisClient: RedisClient,
redisClient: Redis,
userId: string,
taskId: string,
source: string
@ -39,7 +36,7 @@ export const createMetrics = async (
const key = `import:${userId}:${taskId}`
try {
// set multiple fields
await redisClient.hSet(key, {
await redisClient.hset(key, {
['start_time']: Date.now(),
['source']: source,
['state']: ImportTaskState.STARTED,
@ -50,7 +47,7 @@ export const createMetrics = async (
}
export const updateMetrics = async (
redisClient: RedisClient,
redisClient: Redis,
userId: string,
taskId: string,
status: ImportStatus
@ -59,10 +56,13 @@ export const updateMetrics = async (
try {
// use lua script to increment hash field
const state = await redisClient.evalSha(lua.sha, {
keys: [key],
arguments: [status, Date.now().toString()],
})
const state = await redisClient.evalsha(
lua.sha,
1,
key,
status,
Date.now().toString()
)
// if the task is finished, send email
if (state == ImportTaskState.FINISHED) {
@ -77,13 +77,13 @@ export const updateMetrics = async (
}
export const getMetrics = async (
redisClient: RedisClient,
redisClient: Redis,
userId: string,
taskId: string
): Promise<ImportMetrics | null> => {
const key = `import:${userId}:${taskId}`
try {
const metrics = await redisClient.hGetAll(key)
const metrics = await redisClient.hgetall(key)
return {
// convert to integer

54
packages/import-handler/src/redis.ts
View File

@ -1,6 +1,6 @@
import { readFileSync } from 'fs'
import { Redis } from 'ioredis'
import path from 'path'
import { createClient } from 'redis'
// load lua script
export const lua = {
@ -11,31 +11,35 @@ export const lua = {
sha: '',
}
export const createRedisClient = async (url?: string, cert?: string) => {
const redisClient = createClient({
url,
socket: {
tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS
cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line
rejectUnauthorized: false, // for self-signed certs
connectTimeout: 10000, // 10 seconds
reconnectStrategy(retries: number): number | Error {
if (retries > 10) {
return new Error('Retries exhausted')
export const createRedisClient = (url?: string, cert?: string) => {
return new Redis(url || 'redis://localhost:6379', {
connectTimeout: 10000, // 10 seconds
tls: cert
? {
cert,
rejectUnauthorized: false, // for self-signed certs
}
return 1000
},
: undefined,
reconnectOnError: (err) => {
const targetErrors = [/READONLY/, /ETIMEDOUT/]
targetErrors.forEach((targetError) => {
if (targetError.test(err.message)) {
// Only reconnect when the error contains the keyword
return true
}
})
return false
},
retryStrategy: (times) => {
if (times > 10) {
// End reconnecting after a specific number of tries and flush all commands with a individual error
return null
}
// reconnect after
return Math.min(times * 50, 2000)
},
})
redisClient.on('error', (err) => console.error('Redis Client Error', err))
await redisClient.connect()
console.log('Redis Client Connected:', url)
// load script to redis
lua.sha = await redisClient.scriptLoad(lua.script)
console.log('Redis Lua Script Loaded', lua.sha)
return redisClient
}

2
packages/rss-handler/package.json
View File

@ -27,9 +27,9 @@
"@sentry/serverless": "^7.77.0",
"axios": "^1.4.0",
"dotenv": "^16.0.1",
"ioredis": "^5.3.2",
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.16.4",
"redis": "^4.3.1",
"rss-parser": "^3.13.0"
},
"volta": {

19
packages/rss-handler/src/index.ts
View File

@ -2,17 +2,15 @@ import * as Sentry from '@sentry/serverless'
import axios from 'axios'
import crypto from 'crypto'
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import Redis from 'ioredis'
import * as jwt from 'jsonwebtoken'
import { parseHTML } from 'linkedom'
import { createClient } from 'redis'
import Parser, { Item } from 'rss-parser'
import { promisify } from 'util'
import { createRedisClient } from './redis'
import { CONTENT_FETCH_URL, createCloudTask } from './task'
type FolderType = 'following' | 'inbox'
// explicitly create the return type of RedisClient
type RedisClient = ReturnType<typeof createClient>
interface RssFeedRequest {
subscriptionIds: string[]
@ -61,7 +59,7 @@ export const isOldItem = (item: RssFeedItem, lastFetchedAt: number) => {
const feedFetchFailedRedisKey = (feedUrl: string) =>
`feed-fetch-failure:${feedUrl}`
const isFeedBlocked = async (feedUrl: string, redisClient: RedisClient) => {
const isFeedBlocked = async (feedUrl: string, redisClient: Redis) => {
const key = feedFetchFailedRedisKey(feedUrl)
try {
const result = await redisClient.get(key)
@ -78,10 +76,7 @@ const isFeedBlocked = async (feedUrl: string, redisClient: RedisClient) => {
return false
}
const incrementFeedFailure = async (
feedUrl: string,
redisClient: RedisClient
) => {
const incrementFeedFailure = async (feedUrl: string, redisClient: Redis) => {
const key = feedFetchFailedRedisKey(feedUrl)
try {
const result = await redisClient.incr(key)
@ -259,7 +254,7 @@ const sendUpdateSubscriptionMutation = async (
}
const isItemRecentlySaved = async (
redisClient: RedisClient,
redisClient: Redis,
userId: string,
url: string
) => {
@ -274,7 +269,7 @@ const createTask = async (
item: RssFeedItem,
fetchContent: boolean,
folder: FolderType,
redisClient: RedisClient
redisClient: Redis
) => {
const isRecentlySaved = await isItemRecentlySaved(
redisClient,
@ -464,7 +459,7 @@ const processSubscription = async (
fetchContent: boolean,
folder: FolderType,
feed: RssFeed,
redisClient: RedisClient
redisClient: Redis
) => {
let lastItemFetchedAt: Date | null = null
let lastValidItem: RssFeedItem | null = null
@ -606,7 +601,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
}
// create redis client
const redisClient = await createRedisClient(
const redisClient = createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)

50
packages/rss-handler/src/redis.ts
View File

@ -1,26 +1,34 @@
import { createClient } from 'redis'
import { Redis } from 'ioredis'
export const createRedisClient = async (url?: string, cert?: string) => {
const redisClient = createClient({
url,
socket: {
tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS
cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line
rejectUnauthorized: false, // for self-signed certs
connectTimeout: 10000, // 10 seconds
reconnectStrategy(retries: number): number | Error {
if (retries > 10) {
return new Error('Retries exhausted')
export const createRedisClient = (url?: string, cert?: string) => {
return new Redis(url || 'redis://localhost:6379', {
connectTimeout: 10000, // 10 seconds
tls: cert
? {
cert,
rejectUnauthorized: false, // for self-signed certs
}
return 1000
},
: undefined,
reconnectOnError: (err) => {
const targetErrors = [/READONLY/, /ETIMEDOUT/]
targetErrors.forEach((targetError) => {
if (targetError.test(err.message)) {
// Only reconnect when the error contains the keyword
return true
}
})
return false
},
retryStrategy: (times) => {
if (times > 10) {
// End reconnecting after a specific number of tries and flush all commands with a individual error
return null
}
// reconnect after
return Math.min(times * 50, 2000)
},
})
redisClient.on('error', (err) => console.error('Redis Client Error', err))
await redisClient.connect()
console.log('Redis Client Connected:', url)
return redisClient
}

2
packages/text-to-speech/package.json
View File

@ -40,11 +40,11 @@
"dotenv": "^16.0.1",
"fluent-ffmpeg": "^2.1.2",
"html-to-text": "^8.2.1",
"ioredis": "^5.3.2",
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.14.12",
"microsoft-cognitiveservices-speech-sdk": "1.30",
"natural": "^6.2.0",
"redis": "^4.3.1",
"underscore": "^1.13.4"
},
"volta": {

45
packages/text-to-speech/src/index.ts
View File

@ -3,26 +3,23 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable @typescript-eslint/no-unused-vars */
import { File, Storage } from '@google-cloud/storage'
import * as Sentry from '@sentry/serverless'
import axios from 'axios'
import * as jwt from 'jsonwebtoken'
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import { AzureTextToSpeech } from './azureTextToSpeech'
import { File, Storage } from '@google-cloud/storage'
import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml'
import crypto from 'crypto'
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import Redis from 'ioredis'
import * as jwt from 'jsonwebtoken'
import { AzureTextToSpeech } from './azureTextToSpeech'
import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml'
import { OpenAITextToSpeech } from './openaiTextToSpeech'
import { RealisticTextToSpeech } from './realisticTextToSpeech'
import { createRedisClient } from './redis'
import {
SpeechMark,
TextToSpeechInput,
TextToSpeechOutput,
} from './textToSpeech'
import { createClient } from 'redis'
import { RealisticTextToSpeech } from './realisticTextToSpeech'
import { OpenAITextToSpeech } from './openaiTextToSpeech'
// explicitly create the return type of RedisClient
type RedisClient = ReturnType<typeof createClient>
interface UtteranceInput {
text: string
@ -118,7 +115,7 @@ const updateSpeech = async (
}
const getCharacterCountFromRedis = async (
redisClient: RedisClient,
redisClient: Redis,
uid: string
): Promise<number> => {
const wordCount = await redisClient.get(`tts:charCount:${uid}`)
@ -129,14 +126,17 @@ const getCharacterCountFromRedis = async (
// which will be used to rate limit the request
// expires after 1 day
const updateCharacterCountInRedis = async (
redisClient: RedisClient,
redisClient: Redis,
uid: string,
wordCount: number
): Promise<void> => {
await redisClient.set(`tts:charCount:${uid}`, wordCount.toString(), {
EX: 3600 * 24, // in seconds
NX: true,
})
) => {
await redisClient.set(
`tts:charCount:${uid}`,
wordCount.toString(),
'EX',
86400, // 1 day in seconds
'NX'
)
}
export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
@ -240,7 +240,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
}
// create redis client
const redisClient = await createRedisClient(
const redisClient = createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)
@ -353,10 +353,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
await redisClient.set(
cacheKey,
JSON.stringify({ audioDataString, speechMarks }),
{
EX: 3600 * 72, // in seconds
NX: true,
}
'EX',
3600 * 72,
'NX'
)
console.log('Cache saved')

50
packages/text-to-speech/src/redis.ts
View File

@ -1,26 +1,34 @@
import { createClient } from 'redis'
import { Redis } from 'ioredis'
export const createRedisClient = async (url?: string, cert?: string) => {
const redisClient = createClient({
url,
socket: {
tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS
cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line
rejectUnauthorized: false, // for self-signed certs
connectTimeout: 10000, // 10 seconds
reconnectStrategy(retries: number): number | Error {
if (retries > 10) {
return new Error('Retries exhausted')
export const createRedisClient = (url?: string, cert?: string) => {
return new Redis(url || 'redis://localhost:6379', {
connectTimeout: 10000, // 10 seconds
tls: cert
? {
cert,
rejectUnauthorized: false, // for self-signed certs
}
return 1000
},
: undefined,
reconnectOnError: (err) => {
const targetErrors = [/READONLY/, /ETIMEDOUT/]
targetErrors.forEach((targetError) => {
if (targetError.test(err.message)) {
// Only reconnect when the error contains the keyword
return true
}
})
return false
},
retryStrategy: (times) => {
if (times > 10) {
// End reconnecting after a specific number of tries and flush all commands with a individual error
return null
}
// reconnect after
return Math.min(times * 50, 2000)
},
})
redisClient.on('error', (err) => console.error('Redis Client Error', err))
await redisClient.connect()
console.log('Redis Client Connected:', url)
return redisClient
}

117
yarn.lock
View File

@ -3257,6 +3257,11 @@
resolved "https://registry.yarnpkg.com/@icons/material/-/material-0.2.4.tgz#e90c9f71768b3736e76d7dd6783fc6c2afa88bc8"
integrity sha512-QPcGmICAPbGLGb6F/yNf/KzKqvFx8z5qx3D1yFqVAjoFmXK35EgyW+cJ57Te3CNsmzblwtzakLGFqHPqrfb4Tw==
"@ioredis/commands@^1.1.1":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
"@isaacs/cliui@^8.0.2":
version "8.0.2"
resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550"
@ -5424,40 +5429,6 @@
resolved "https://registry.yarnpkg.com/@reach/observe-rect/-/observe-rect-1.2.0.tgz#d7a6013b8aafcc64c778a0ccb83355a11204d3b2"
integrity sha512-Ba7HmkFgfQxZqqaeIWWkNK0rEhpxVQHIoVyW1YDSkGsGIXzcaW4deC8B0pZrNSSyLTdIk7y+5olKt5+g0GmFIQ==
"@redis/bloom@1.0.2":
version "1.0.2"
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf"
integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==
"@redis/client@1.3.0":
version "1.3.0"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77"
integrity sha512-XCFV60nloXAefDsPnYMjHGtvbtHR8fV5Om8cQ0JYqTNbWcQo/4AryzJ2luRj4blveWazRK/j40gES8M7Cp6cfQ==
dependencies:
cluster-key-slot "1.1.0"
generic-pool "3.8.2"
yallist "4.0.0"
"@redis/graph@1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99"
integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==
"@redis/json@1.0.4":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1"
integrity sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==
"@redis/search@1.1.0":
version "1.1.0"
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df"
integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==
"@redis/time-series@1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4"
integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==
"@remusao/guess-url-type@^1.1.2":
version "1.2.1"
resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57"
@ -11624,10 +11595,10 @@ clsx@^1.1.1:
resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188"
integrity sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==
cluster-key-slot@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
cluster-key-slot@^1.1.0:
version "1.1.2"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
cmd-shim@6.0.1:
version "6.0.1"
@ -13055,6 +13026,11 @@ delegates@^1.0.0:
resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=
denque@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
depd@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df"
@ -15803,11 +15779,6 @@ gcp-metadata@^6.0.0:
gaxios "^6.0.0"
json-bigint "^1.0.0"
generic-pool@3.8.2:
version "3.8.2"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@ -17627,6 +17598,21 @@ invariant@^2.2.4:
dependencies:
loose-envify "^1.0.0"
ioredis@^5.3.2:
version "5.3.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7"
integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==
dependencies:
"@ioredis/commands" "^1.1.1"
cluster-key-slot "^1.1.0"
debug "^4.3.4"
denque "^2.1.0"
lodash.defaults "^4.2.0"
lodash.isarguments "^3.1.0"
redis-errors "^1.2.0"
redis-parser "^3.0.0"
standard-as-callback "^2.1.0"
ip-regex@^4.0.0, ip-regex@^4.1.0, ip-regex@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.3.0.tgz#687275ab0f57fa76978ff8f4dddc8a23d5990db5"
@ -20196,6 +20182,11 @@ lodash.debounce@^4.0.8:
resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168=
lodash.defaults@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==
lodash.escaperegexp@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347"
@ -20221,6 +20212,11 @@ lodash.includes@^4.3.0:
resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=
lodash.isarguments@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==
lodash.isboolean@^3.0.3:
version "3.0.3"
resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
@ -26093,17 +26089,17 @@ redeyed@~2.1.0:
dependencies:
esprima "~4.0.0"
redis@^4.3.1:
version "4.3.1"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.3.1.tgz#290532a0c22221e05e991162ac4dca1e1b2ff6da"
integrity sha512-cM7yFU5CA6zyCF7N/+SSTcSJQSRMEKN0k0Whhu6J7n9mmXRoXugfWDBo5iOzGwABmsWKSwGPTU5J4Bxbl+0mrA==
redis-errors@^1.0.0, redis-errors@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==
redis-parser@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==
dependencies:
"@redis/bloom" "1.0.2"
"@redis/client" "1.3.0"
"@redis/graph" "1.0.1"
"@redis/json" "1.0.4"
"@redis/search" "1.1.0"
"@redis/time-series" "1.0.3"
redis-errors "^1.0.0"
reflect-metadata@^0.1.13:
version "0.1.13"
@ -27803,6 +27799,11 @@ stacktrace-parser@^0.1.10:
dependencies:
type-fest "^0.7.1"
standard-as-callback@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
state-toggle@^1.0.0:
version "1.0.3"
resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe"
@ -30940,11 +30941,6 @@ y18n@^5.0.5:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
yallist@4.0.0, yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yallist@^2.0.0, yallist@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
@ -30955,6 +30951,11 @@ yallist@^3.0.2:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yaml-ast-parser@^0.0.43:
version "0.0.43"
resolved "https://registry.yarnpkg.com/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb"