diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index 239e31376..7de5e338b 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -35,6 +35,7 @@ "lodash": "^4.17.21", "luxon": "^3.0.4", "puppeteer-core": "^19.1.1", + "redis": "^4.6.7", "underscore": "^1.13.6", "uuid": "^9.0.0" } diff --git a/packages/content-handler/src/redis.ts b/packages/content-handler/src/redis.ts new file mode 100644 index 000000000..350cb09f6 --- /dev/null +++ b/packages/content-handler/src/redis.ts @@ -0,0 +1,26 @@ +import { createClient } from 'redis' + +export const createRedisClient = async (url?: string, cert?: string) => { + const redisClient = createClient({ + url, + socket: { + tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS + cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line + rejectUnauthorized: false, // for self-signed certs + connectTimeout: 10000, // 10 seconds + reconnectStrategy(retries: number): number | Error { + if (retries > 10) { + return new Error('Retries exhausted') + } + return 1000 + }, + }, + }) + + redisClient.on('error', (err) => console.error('Redis Client Error', err)) + + await redisClient.connect() + console.log('Redis Client Connected:', url) + + return redisClient +} diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 423cbc65a..6d1c68d8b 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -2,7 +2,12 @@ import axios from 'axios' import { parseHTML } from 'linkedom' import _, { truncate } from 'lodash' import { DateTime } from 'luxon' +import { createClient } from 'redis' import { ContentHandler, PreHandleResult } from '../content-handler' +import { createRedisClient } from '../redis' + +// explicitly create the return type of RedisClient +type RedisClient = ReturnType interface Tweet { url: string @@ -31,14 +36,15 @@ export class NitterHandler extends ContentHandler { URL_MATCH = /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ INSTANCES = [ - 'https://nitter.net', - 'https://nitter.lacontrevoie.fr', - 'https://nitter.1d4.us', - 'https://nitter.kavin.rocks', - 'https://nitter.it', - 'https://twitter.owacon.moe', - 'https://singapore.unofficialbird.com', + { value: 'https://nitter.net', score: 0 }, + { value: 'https://nitter.lacontrevoie.fr', score: 0 }, + { value: 'https://nitter.1d4.us', score: 0 }, + { value: 'https://nitter.kavin.rocks', score: 0 }, + { value: 'https://nitter.it', score: 0 }, + { value: 'https://twitter.owacon.moe', score: 0 }, + { value: 'https://singapore.unofficialbird.com', score: 0 }, ] + REDIS_KEY = 'nitter-instances' private instance: string @@ -48,6 +54,44 @@ export class NitterHandler extends ContentHandler { this.instance = '' } + async getInstance(redisClient: RedisClient) { + const instances = await redisClient.zRangeByScore( + this.REDIS_KEY, + '-inf', + '+inf', + { + LIMIT: { + count: 1, + offset: 0, + }, + } + ) + + // if no instance is found, save the default instances + if (instances.length === 0) { + await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES) + return this.INSTANCES[0].value + } + + return instances[0] + } + + async incrementInstanceScore( + redisClient: RedisClient, + instance: string, + score = 1 + ) { + await redisClient.zIncrBy(this.REDIS_KEY, score, instance) + } + + async decrementInstanceScore( + redisClient: RedisClient, + instance: string, + score = 1 + ) { + await redisClient.zIncrBy(this.REDIS_KEY, score, instance) + } + async getTweets(username: string, tweetId: string) { function authorParser(header: Element) { const profileImageUrl = @@ -139,22 +183,30 @@ export class NitterHandler extends ContentHandler { } } + const redisClient = await createRedisClient() + try { const tweets: Tweet[] = [] const option = { timeout: 20000, // 20 seconds } let html: any - // use the first instance that works - for (const instance of this.INSTANCES) { + // get instance from redis + for (let i = 0; i < this.INSTANCES.length; i++) { + const instance = await this.getInstance(redisClient) + try { const url = `${instance}/${username}/status/${tweetId}` const response = await axios.get(url, option) // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment html = response.data this.instance = instance + + await this.incrementInstanceScore(redisClient, instance) break } catch (error) { + await this.decrementInstanceScore(redisClient, instance) + if (axios.isAxiosError(error)) { console.info(`Error getting tweets from ${instance}`, error.message) } else { @@ -214,6 +266,8 @@ export class NitterHandler extends ContentHandler { console.error('Error getting tweets', error) return [] + } finally { + await redisClient?.quit() } } diff --git a/yarn.lock b/yarn.lock index 025cd3b97..178efa557 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5436,6 +5436,11 @@ resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf" integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw== +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + "@redis/client@1.3.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77" @@ -5445,11 +5450,25 @@ generic-pool "3.8.2" yallist "4.0.0" +"@redis/client@1.5.8": + version "1.5.8" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1" + integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + "@redis/graph@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99" integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ== +"@redis/graph@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" + integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== + "@redis/json@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" @@ -5460,11 +5479,21 @@ resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df" integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ== +"@redis/search@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b" + integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng== + "@redis/time-series@1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== +"@redis/time-series@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" + integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== + "@remusao/guess-url-type@^1.1.2": version "1.2.1" resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" @@ -11860,6 +11889,11 @@ cluster-key-slot@1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + cmd-shim@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -15355,6 +15389,11 @@ generic-pool@3.8.2: resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -24194,6 +24233,18 @@ redis@^4.3.1: "@redis/search" "1.1.0" "@redis/time-series" "1.0.3" +redis@^4.6.7: + version "4.6.7" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.7.tgz#c73123ad0b572776223f172ec78185adb72a6b57" + integrity sha512-KrkuNJNpCwRm5vFJh0tteMxW8SaUzkm5fBH7eL5hd/D0fAkzvapxbfGPP/r+4JAXdQuX7nebsBkBqA2RHB7Usw== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.5.8" + "@redis/graph" "1.1.0" + "@redis/json" "1.0.4" + "@redis/search" "1.1.3" + "@redis/time-series" "1.0.4" + reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"