Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save instances in redis as a sorted set

Browse Source
This commit is contained in:
Hongbo Wu
2023-06-23 12:17:41 +08:00
parent 4607754f58
commit ae504a970b
4 changed files with 141 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/content-handler/package.json
View File

@ -35,6 +35,7 @@
"lodash": "^4.17.21",
"luxon": "^3.0.4",
"puppeteer-core": "^19.1.1",
"redis": "^4.6.7",
"underscore": "^1.13.6",
"uuid": "^9.0.0"
}

26
packages/content-handler/src/redis.ts Normal file
View File

@ -0,0 +1,26 @@
import { createClient } from 'redis'
export const createRedisClient = async (url?: string, cert?: string) => {
const redisClient = createClient({
url,
socket: {
tls: url?.startsWith('rediss://'), // rediss:// is the protocol for TLS
cert: cert?.replace(/\\n/g, '\n'), // replace \n with new line
rejectUnauthorized: false, // for self-signed certs
connectTimeout: 10000, // 10 seconds
reconnectStrategy(retries: number): number | Error {
if (retries > 10) {
return new Error('Retries exhausted')
}
return 1000
},
},
})
redisClient.on('error', (err) => console.error('Redis Client Error', err))
await redisClient.connect()
console.log('Redis Client Connected:', url)
return redisClient
}

72
packages/content-handler/src/websites/nitter-handler.ts
View File

@ -2,7 +2,12 @@ import axios from 'axios'
import { parseHTML } from 'linkedom'
import _, { truncate } from 'lodash'
import { DateTime } from 'luxon'
import { createClient } from 'redis'
import { ContentHandler, PreHandleResult } from '../content-handler'
import { createRedisClient } from '../redis'
// explicitly create the return type of RedisClient
type RedisClient = ReturnType<typeof createClient>
interface Tweet {
url: string
@ -31,14 +36,15 @@ export class NitterHandler extends ContentHandler {
URL_MATCH =
/((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
INSTANCES = [
'https://nitter.net',
'https://nitter.lacontrevoie.fr',
'https://nitter.1d4.us',
'https://nitter.kavin.rocks',
'https://nitter.it',
'https://twitter.owacon.moe',
'https://singapore.unofficialbird.com',
{ value: 'https://nitter.net', score: 0 },
{ value: 'https://nitter.lacontrevoie.fr', score: 0 },
{ value: 'https://nitter.1d4.us', score: 0 },
{ value: 'https://nitter.kavin.rocks', score: 0 },
{ value: 'https://nitter.it', score: 0 },
{ value: 'https://twitter.owacon.moe', score: 0 },
{ value: 'https://singapore.unofficialbird.com', score: 0 },
]
REDIS_KEY = 'nitter-instances'
private instance: string
@ -48,6 +54,44 @@ export class NitterHandler extends ContentHandler {
this.instance = ''
}
async getInstance(redisClient: RedisClient) {
const instances = await redisClient.zRangeByScore(
this.REDIS_KEY,
'-inf',
'+inf',
{
LIMIT: {
count: 1,
offset: 0,
},
}
)
// if no instance is found, save the default instances
if (instances.length === 0) {
await redisClient.zAdd(this.REDIS_KEY, this.INSTANCES)
return this.INSTANCES[0].value
}
return instances[0]
}
async incrementInstanceScore(
redisClient: RedisClient,
instance: string,
score = 1
) {
await redisClient.zIncrBy(this.REDIS_KEY, score, instance)
}
async decrementInstanceScore(
redisClient: RedisClient,
instance: string,
score = 1
) {
await redisClient.zIncrBy(this.REDIS_KEY, score, instance)
}
async getTweets(username: string, tweetId: string) {
function authorParser(header: Element) {
const profileImageUrl =
@ -139,22 +183,30 @@ export class NitterHandler extends ContentHandler {
}
}
const redisClient = await createRedisClient()
try {
const tweets: Tweet[] = []
const option = {
timeout: 20000, // 20 seconds
}
let html: any
// use the first instance that works
for (const instance of this.INSTANCES) {
// get instance from redis
for (let i = 0; i < this.INSTANCES.length; i++) {
const instance = await this.getInstance(redisClient)
try {
const url = `${instance}/${username}/status/${tweetId}`
const response = await axios.get(url, option)
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
html = response.data
this.instance = instance
await this.incrementInstanceScore(redisClient, instance)
break
} catch (error) {
await this.decrementInstanceScore(redisClient, instance)
if (axios.isAxiosError(error)) {
console.info(`Error getting tweets from ${instance}`, error.message)
} else {
@ -214,6 +266,8 @@ export class NitterHandler extends ContentHandler {
console.error('Error getting tweets', error)
return []
} finally {
await redisClient?.quit()
}
}

51
yarn.lock
View File

@ -5436,6 +5436,11 @@
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf"
integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==
"@redis/bloom@1.2.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==
"@redis/client@1.3.0":
version "1.3.0"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77"
@ -5445,11 +5450,25 @@
generic-pool "3.8.2"
yallist "4.0.0"
"@redis/client@1.5.8":
version "1.5.8"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1"
integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw==
dependencies:
cluster-key-slot "1.1.2"
generic-pool "3.9.0"
yallist "4.0.0"
"@redis/graph@1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99"
integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==
"@redis/graph@1.1.0":
version "1.1.0"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519"
integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==
"@redis/json@1.0.4":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1"
@ -5460,11 +5479,21 @@
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df"
integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==
"@redis/search@1.1.3":
version "1.1.3"
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b"
integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng==
"@redis/time-series@1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4"
integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==
"@redis/time-series@1.0.4":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717"
integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==
"@remusao/guess-url-type@^1.1.2":
version "1.2.1"
resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57"
@ -11860,6 +11889,11 @@ cluster-key-slot@1.1.0:
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
cluster-key-slot@1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
cmd-shim@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd"
@ -15355,6 +15389,11 @@ generic-pool@3.8.2:
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
generic-pool@3.9.0:
version "3.9.0"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==
gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@ -24194,6 +24233,18 @@ redis@^4.3.1:
"@redis/search" "1.1.0"
"@redis/time-series" "1.0.3"
redis@^4.6.7:
version "4.6.7"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.7.tgz#c73123ad0b572776223f172ec78185adb72a6b57"
integrity sha512-KrkuNJNpCwRm5vFJh0tteMxW8SaUzkm5fBH7eL5hd/D0fAkzvapxbfGPP/r+4JAXdQuX7nebsBkBqA2RHB7Usw==
dependencies:
"@redis/bloom" "1.2.0"
"@redis/client" "1.5.8"
"@redis/graph" "1.1.0"
"@redis/json" "1.0.4"
"@redis/search" "1.1.3"
"@redis/time-series" "1.0.4"
reflect-metadata@^0.1.13:
version "0.1.13"
resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"