diff --git a/packages/discover/src/index.ts b/packages/discover/src/index.ts index 04e2a2a1c..2f6e403f3 100644 --- a/packages/discover/src/index.ts +++ b/packages/discover/src/index.ts @@ -22,5 +22,7 @@ const enrichedArticles$ = (): Observable => { putImageInProxy$, insertArticleToStore$ ) - .subscribe((_it) => {}) + .subscribe((it) => { + console.log('enriched: ', it) + }) })() diff --git a/packages/discover/src/lib/ai/embedding.ts b/packages/discover/src/lib/ai/embedding.ts index 4d845baf6..2ce21c448 100644 --- a/packages/discover/src/lib/ai/embedding.ts +++ b/packages/discover/src/lib/ai/embedding.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ import { mergeMap } from 'rxjs/operators' import { OmnivoreArticle } from '../../types/OmnivoreArticle' import { OperatorFunction, pipe, share } from 'rxjs' diff --git a/packages/discover/src/lib/ai/enrich.ts b/packages/discover/src/lib/ai/enrich.ts index 0e949ecad..943c38f74 100644 --- a/packages/discover/src/lib/ai/enrich.ts +++ b/packages/discover/src/lib/ai/enrich.ts @@ -16,7 +16,7 @@ export const needsPopulating = (article: OmnivoreArticle) => { } const setArticleDescription = async ( - article: OmnivoreArticle, + article: OmnivoreArticle ): Promise => { const client = await omnivoreClient const { content } = await client.fetchPage(article.slug) @@ -31,7 +31,7 @@ export const setArticleDescriptionAsSubsetOfContent: OperatorFunction< OmnivoreArticle > = mergeMap( (it: OmnivoreArticle) => fromPromise(setArticleDescription(it)), - 10, + 10 ) const enrichArticleWithAiSummary = (it: OmnivoreArticle) => @@ -43,7 +43,7 @@ const enrichArticleWithAiSummary = (it: OmnivoreArticle) => try { const tokens = convert(content).slice( 0, - Math.floor(client.tokenLimit * 0.75), + Math.floor(client.tokenLimit * 0.75) ) const description = await client.summarizeText(tokens) return { ...article, description } @@ -52,7 +52,7 @@ const enrichArticleWithAiSummary = (it: OmnivoreArticle) => console.log(e) throw e } - })(it), + })(it) ) export const enrichArticleWithAiGeneratedDescription: OperatorFunction< @@ -61,6 +61,6 @@ export const enrichArticleWithAiGeneratedDescription: OperatorFunction< > = pipe( rateLimiter({ resetLimit: 50, timeMs: 60_000 }), mergeMap((it: OmnivoreArticle) => - enrichArticleWithAiSummary(it).pipe(exponentialBackOff(30)), - ), + enrichArticleWithAiSummary(it).pipe(exponentialBackOff(30)) + ) ) diff --git a/packages/discover/src/lib/clients/ai/bedrock.ts b/packages/discover/src/lib/clients/ai/bedrock.ts index 603f1635e..c2ffc5cc6 100644 --- a/packages/discover/src/lib/clients/ai/bedrock.ts +++ b/packages/discover/src/lib/clients/ai/bedrock.ts @@ -16,7 +16,7 @@ export class BedrockClient implements AiClient { params: BedrockClientParams = { region: 'us-west-2', endpoint: 'https://bedrock-runtime.us-west-2.amazonaws.com', - }, + } ) { this.client = axios.create({ baseURL: params.endpoint, @@ -34,7 +34,7 @@ export class BedrockClient implements AiClient { } _extractHttpBody( - invokeParams: BedrockInvokeParams, + invokeParams: BedrockInvokeParams ): Partial { const { model: _, prompt, ...httpCommands } = invokeParams return { ...httpCommands, prompt: this._wrapPrompt(prompt) } @@ -47,7 +47,7 @@ export class BedrockClient implements AiClient { async getEmbeddings(text: string): Promise { const { data } = await this.client.post( `/model/cohere.embed-english-v3/invoke`, - { texts: [text], input_type: 'clustering' }, + { texts: [text], input_type: 'clustering' } ) return data.embeddings![0] } @@ -65,7 +65,7 @@ export class BedrockClient implements AiClient { const { data } = await this.client.post( `/model/${summariseParams.model}/invoke`, - this._extractHttpBody(summariseParams), + this._extractHttpBody(summariseParams) ) return data.completion } diff --git a/packages/discover/src/lib/clients/omnivore/imageProxy.ts b/packages/discover/src/lib/clients/omnivore/imageProxy.ts index 1ca55b660..c14d5622a 100644 --- a/packages/discover/src/lib/clients/omnivore/imageProxy.ts +++ b/packages/discover/src/lib/clients/omnivore/imageProxy.ts @@ -23,6 +23,6 @@ export const putImageInProxy$ = pipe( image: it.article.image && addImageToProxy(it.article.image), }, } - }), - ), + }) + ) ) diff --git a/packages/discover/src/lib/clients/omnivore/omnivore.ts b/packages/discover/src/lib/clients/omnivore/omnivore.ts index 53b7abf1f..4ebfb72c4 100644 --- a/packages/discover/src/lib/clients/omnivore/omnivore.ts +++ b/packages/discover/src/lib/clients/omnivore/omnivore.ts @@ -220,7 +220,7 @@ export class OmnivoreClient { Cookie: `auth=${this.token};`, 'Content-Type': 'application/json', }, - }, + } ) .then((_) => true) } diff --git a/packages/discover/src/lib/inputSources/articles/communityArticles.ts b/packages/discover/src/lib/inputSources/articles/communityArticles.ts index 213ea1f6e..5dfaafc88 100644 --- a/packages/discover/src/lib/inputSources/articles/communityArticles.ts +++ b/packages/discover/src/lib/inputSources/articles/communityArticles.ts @@ -10,7 +10,7 @@ export const COMMUNITY = 'OMNIVORE_COMMUNITY' const extractArticleFromMessage = (message: Message): OmnivoreArticle => { const parsedMessage: OmnivoreArticle = JSON.parse( - message.data.toString(), + message.data.toString() ) as OmnivoreArticle return { @@ -49,10 +49,10 @@ export const communityArticles$ = new Observable( .catch((e) => { console.error( 'Error creating Subscription, continuing without community articles...', - e, + e ) }) - }, + } ).pipe( catchError((err) => { console.log('Caught Error, continuing') @@ -60,5 +60,5 @@ export const communityArticles$ = new Observable( // Return an empty Observable which gets collapsed in the output return EMPTY - }), + }) ) diff --git a/packages/discover/src/lib/inputSources/articles/rss/newFeedIngestor.ts b/packages/discover/src/lib/inputSources/articles/rss/newFeedIngestor.ts index 4121f6fa8..dd2a1a240 100644 --- a/packages/discover/src/lib/inputSources/articles/rss/newFeedIngestor.ts +++ b/packages/discover/src/lib/inputSources/articles/rss/newFeedIngestor.ts @@ -60,10 +60,10 @@ export const newFeeds$ = new Observable( .catch((e) => { console.error( 'Error creating Subscription, continuing without new feed parsing...', - e, + e ) }) - }, + } ).pipe( catchError((err) => { console.log('Caught Error, continuing') @@ -71,5 +71,5 @@ export const newFeeds$ = new Observable( // Return an empty Observable which gets collapsed in the output return EMPTY - }), + }) ) diff --git a/packages/discover/src/lib/inputSources/articles/rss/rssConverters/atom.ts b/packages/discover/src/lib/inputSources/articles/rss/rssConverters/atom.ts index 5cd6d5578..74fbe6616 100644 --- a/packages/discover/src/lib/inputSources/articles/rss/rssConverters/atom.ts +++ b/packages/discover/src/lib/inputSources/articles/rss/rssConverters/atom.ts @@ -1,3 +1,8 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ + import { OmnivoreArticle } from '../../../../../types/OmnivoreArticle' import { slugify } from 'voca' import { Observable, tap } from 'rxjs' @@ -63,6 +68,6 @@ export const convertAtomStream = (feed: OmnivoreFeed) => (parsedXml: any) => { type: 'rss', feedId: feed.title, } - }), + }) ) } diff --git a/packages/discover/src/lib/inputSources/articles/rss/rssConverters/generic.ts b/packages/discover/src/lib/inputSources/articles/rss/rssConverters/generic.ts index 00d83ccd4..a784d6cb2 100644 --- a/packages/discover/src/lib/inputSources/articles/rss/rssConverters/generic.ts +++ b/packages/discover/src/lib/inputSources/articles/rss/rssConverters/generic.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ + import { OmnivoreArticle } from '../../../../../types/OmnivoreArticle' import { XMLParser } from 'fast-xml-parser' import { Observable } from 'rxjs' @@ -92,7 +96,7 @@ export const parseAtomOrRss = (contentFeed: OmnivoreContentFeed) => { ? parseRss(contentFeed.feed)( parsedXml.rss?.channel?.item || parsedXml['rdf:RDF'].channel?.item || - parsedXml['rdf:RDF'].item, + parsedXml['rdf:RDF'].item ) : convertAtomStream(contentFeed.feed)(parsedXml) } diff --git a/packages/discover/src/lib/inputSources/articles/rss/rssConverters/rss.ts b/packages/discover/src/lib/inputSources/articles/rss/rssConverters/rss.ts index 1e29b2754..c672a707b 100644 --- a/packages/discover/src/lib/inputSources/articles/rss/rssConverters/rss.ts +++ b/packages/discover/src/lib/inputSources/articles/rss/rssConverters/rss.ts @@ -1,3 +1,8 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ + import { JSDOM } from 'jsdom' import { get } from 'lodash' import { fromArrayLike } from 'rxjs/internal/observable/innerFrom' @@ -103,11 +108,11 @@ export const parseRss = (feed: OmnivoreFeed) => (parsedXml: any) => { image: image ?? '', site: new URL(article.link).host, publishedAt: new Date( - article.pubDate ?? article['dc:date'] ?? Date.now(), + article.pubDate ?? article['dc:date'] ?? Date.now() ), type: 'rss', feedId: feed.id, } - }), + }) ) } diff --git a/packages/discover/src/lib/store/articles.ts b/packages/discover/src/lib/store/articles.ts index 0cc6b0601..a31aa1f2d 100644 --- a/packages/discover/src/lib/store/articles.ts +++ b/packages/discover/src/lib/store/articles.ts @@ -1,3 +1,8 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ + import { EmbeddedOmnivoreArticle } from '../ai/embedding' import { filter, map, mergeMap, bufferTime } from 'rxjs/operators' import { toSql } from 'pgvector/pg' @@ -12,7 +17,7 @@ import { onErrorContinue } from '../utils/reactive' const hasStoredInDatabase = async (articleSlug: string, feedId: string) => { const { rows } = await sqlClient.query( 'SELECT slug FROM omnivore.discover_feed_articles WHERE slug = $1 and feed_id = $2', - [articleSlug, feedId], + [articleSlug, feedId] ) return rows && rows.length === 0 } @@ -21,13 +26,13 @@ export const removeDuplicateArticles$ = onErrorContinue( mergeMap((x: OmnivoreArticle) => fromPromise(hasStoredInDatabase(x.slug, x.feedId)).pipe( filter(Boolean), - map(() => x), - ), - ), + map(() => x) + ) + ) ) export const batchInsertArticlesSql = async ( - articles: EmbeddedOmnivoreArticle[], + articles: EmbeddedOmnivoreArticle[] ) => { const params = articles.map((embedded) => [ v4(), @@ -45,7 +50,7 @@ export const batchInsertArticlesSql = async ( if (articles.length > 0) { const formattedMultiInsert = pgformat( `INSERT INTO omnivore.discover_feed_articles(id, title, feed_id, slug, description, url, author, image, published_at, embedding) VALUES %L ON CONFLICT DO NOTHING`, - params, + params ) await sqlClient.query(formattedMultiInsert) @@ -57,7 +62,7 @@ export const batchInsertArticlesSql = async ( const formattedTopicInsert = pgformat( `INSERT INTO omnivore.discover_feed_article_topic_link(discover_topic_name, discover_feed_article_id) VALUES %L ON CONFLICT DO NOTHING`, - topicLinks, + topicLinks ) await sqlClient.query(formattedTopicInsert) @@ -71,8 +76,8 @@ export const insertArticleToStore$ = pipe( bufferTime(5000, null, 100), onErrorContinue( mergeMap((x: EmbeddedOmnivoreArticle[]) => - fromPromise(batchInsertArticlesSql(x)), - ), + fromPromise(batchInsertArticlesSql(x)) + ) ), - mergeMap((it: EmbeddedOmnivoreArticle[]) => from(it)), + mergeMap((it: EmbeddedOmnivoreArticle[]) => from(it)) ) diff --git a/packages/discover/src/lib/store/feeds.ts b/packages/discover/src/lib/store/feeds.ts index dc9b188c2..1bb730acf 100644 --- a/packages/discover/src/lib/store/feeds.ts +++ b/packages/discover/src/lib/store/feeds.ts @@ -6,9 +6,9 @@ import { fromPromise } from 'rxjs/internal/observable/innerFrom' export const getRssFeeds$ = fromPromise( (async (): Promise => { const { rows } = (await sqlClient.query( - `SELECT * FROM omnivore.discover_feed WHERE title != 'OMNIVORE_COMMUNITY'`, + `SELECT * FROM omnivore.discover_feed WHERE title != 'OMNIVORE_COMMUNITY'` )) as { rows: OmnivoreFeed[] } return rows - })(), + })() ).pipe(mergeMap((it) => it)) diff --git a/packages/discover/src/lib/store/labels.ts b/packages/discover/src/lib/store/labels.ts index 12fadfa9e..81a186840 100644 --- a/packages/discover/src/lib/store/labels.ts +++ b/packages/discover/src/lib/store/labels.ts @@ -1,3 +1,5 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ + import { EmbeddedOmnivoreLabel } from '../ai/embedding' import { filter, map, mergeMap } from 'rxjs/operators' import { toSql } from 'pgvector/pg' @@ -9,7 +11,7 @@ import { Label } from '../../types/OmnivoreSchema' const hasLabelsStoredInDatabase = async (label: string) => { const { rows } = await sqlClient.query( `SELECT label FROM label_embeddings where label = $1`, - [label], + [label] ) return rows && rows.length === 0 } @@ -17,16 +19,16 @@ const hasLabelsStoredInDatabase = async (label: string) => { export const removeDuplicateLabels = mergeMap((x: Label) => fromPromise(hasLabelsStoredInDatabase(x.name)).pipe( filter(Boolean), - map(() => x), - ), + map(() => x) + ) ) export const insertLabels = async ( - label: EmbeddedOmnivoreLabel, + label: EmbeddedOmnivoreLabel ): Promise => { await sqlClient.query( 'INSERT INTO omnivore.discover_topic_embedding_link(discover_topic_name, embedding_description, embedding) VALUES($1, $2, $3)', - [label.label.name, label.label.description, toSql(label.embedding)], + [label.label.name, label.label.description, toSql(label.embedding)] ) return label } diff --git a/packages/discover/src/lib/utils/imageproxy.ts b/packages/discover/src/lib/utils/imageproxy.ts index 2f6af34c8..d1c383062 100644 --- a/packages/discover/src/lib/utils/imageproxy.ts +++ b/packages/discover/src/lib/utils/imageproxy.ts @@ -4,14 +4,14 @@ import { env } from '../../env' function signImageProxyUrl(url: string): string { return encode( - crypto.createHmac('sha256', env.imageProxy.secretKey!).update(url).digest(), + crypto.createHmac('sha256', env.imageProxy.secretKey!).update(url).digest() ) } export function createImageProxyUrl( url: string, width = 0, - height = 0, + height = 0 ): string { if (!env.imageProxy.url || !env.imageProxy.secretKey) { return url diff --git a/packages/discover/src/lib/utils/reactive.ts b/packages/discover/src/lib/utils/reactive.ts index eb3b92816..e3d468ba4 100644 --- a/packages/discover/src/lib/utils/reactive.ts +++ b/packages/discover/src/lib/utils/reactive.ts @@ -16,14 +16,14 @@ import { OmnivoreArticle } from '../../types/OmnivoreArticle' import { fromPromise } from 'rxjs/internal/observable/innerFrom' export const exponentialBackOff = ( - count: number, + count: number ): MonoTypeOperatorFunction => retry({ count, delay: (error, retryIndex, interval = 200) => { const delay = Math.pow(2, retryIndex - 1) * interval console.log( - `Backing off: attempt ${retryIndex}, Trying again in: ${delay}ms`, + `Backing off: attempt ${retryIndex}, Trying again in: ${delay}ms` ) return timer(delay) @@ -41,7 +41,7 @@ export const onErrorContinue = (...pipes: OperatorFunction[]) => catchError((e) => { console.error('Error caught in pipe, skipping', e) return EMPTY - }), + }) ) }) @@ -59,11 +59,11 @@ export function mapOrNull(project: (article: any) => Promise) { concatMap((item: any, _value: number) => { try { return fromPromise(project(item).catch((_e) => null)).pipe( - filter((it) => !!it), + filter((it) => !!it) ) as Observable } catch (e) { return EMPTY } - }), + }) ) }