diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts index df82e1713..0bf5c7862 100644 --- a/packages/api/src/entity/public_item.ts +++ b/packages/api/src/entity/public_item.ts @@ -6,7 +6,7 @@ import { UpdateDateColumn, } from 'typeorm' -@Entity() +@Entity({ name: 'public_item' }) export class PublicItem { @PrimaryGeneratedColumn('uuid') id!: string diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index a3d09f2eb..514a7d2ba 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1378,21 +1378,22 @@ export type JustReadFeedItem = { author?: Maybe; broadcastCount?: Maybe; comments?: Maybe>; - createdAt?: Maybe; + createdAt: Scalars['Date']; dir?: Maybe; highlights?: Maybe; id: Scalars['ID']; languageCode?: Maybe; likedCount?: Maybe; previewContent?: Maybe; - publishedAt?: Maybe; savedCount?: Maybe; seen_at?: Maybe; - source?: Maybe; + siteName?: Maybe; + sourceIcon?: Maybe; + sourceName: Scalars['String']; thumbnail?: Maybe; title: Scalars['String']; topic?: Maybe; - updatedAt?: Maybe; + updatedAt: Scalars['Date']; url: Scalars['String']; wordCount?: Maybe; }; @@ -1406,7 +1407,7 @@ export type JustReadFeedSource = { name: Scalars['String']; thumbnail?: Maybe; topics?: Maybe>; - url?: Maybe; + url: Scalars['String']; }; export type JustReadFeedSuccess = { @@ -2075,7 +2076,7 @@ export type MutationUploadImportFileArgs = { export type MySubscriptionRootType = { __typename?: 'MySubscriptionRootType'; - justReadFeed: JustReadFeedResult; + hello?: Maybe; }; export type NewsletterEmail = { @@ -2224,6 +2225,7 @@ export type Query = { hello?: Maybe; integration: IntegrationResult; integrations: IntegrationsResult; + justReadFeed: JustReadFeedResult; labels: LabelsResult; me?: Maybe; newsletterEmails: NewsletterEmailsResult; @@ -6075,21 +6077,22 @@ export type JustReadFeedItemResolvers, ParentType, ContextType>; broadcastCount?: Resolver, ParentType, ContextType>; comments?: Resolver>, ParentType, ContextType>; - createdAt?: Resolver, ParentType, ContextType>; + createdAt?: Resolver; dir?: Resolver, ParentType, ContextType>; highlights?: Resolver, ParentType, ContextType>; id?: Resolver; languageCode?: Resolver, ParentType, ContextType>; likedCount?: Resolver, ParentType, ContextType>; previewContent?: Resolver, ParentType, ContextType>; - publishedAt?: Resolver, ParentType, ContextType>; savedCount?: Resolver, ParentType, ContextType>; seen_at?: Resolver, ParentType, ContextType>; - source?: Resolver, ParentType, ContextType>; + siteName?: Resolver, ParentType, ContextType>; + sourceIcon?: Resolver, ParentType, ContextType>; + sourceName?: Resolver; thumbnail?: Resolver, ParentType, ContextType>; title?: Resolver; topic?: Resolver, ParentType, ContextType>; - updatedAt?: Resolver, ParentType, ContextType>; + updatedAt?: Resolver; url?: Resolver; wordCount?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -6105,7 +6108,7 @@ export type JustReadFeedSourceResolvers; thumbnail?: Resolver, ParentType, ContextType>; topics?: Resolver>, ParentType, ContextType>; - url?: Resolver, ParentType, ContextType>; + url?: Resolver; __isTypeOf?: IsTypeOfResolverFn; }; @@ -6358,7 +6361,7 @@ export type MutationResolvers = { - justReadFeed?: SubscriptionResolver; + hello?: SubscriptionResolver, "hello", ParentType, ContextType>; }; export type NewsletterEmailResolvers = { @@ -6452,6 +6455,7 @@ export type QueryResolvers, ParentType, ContextType>; integration?: Resolver>; integrations?: Resolver; + justReadFeed?: Resolver; labels?: Resolver; me?: Resolver, ParentType, ContextType>; newsletterEmails?: Resolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 0474deea6..c64e7c328 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1241,21 +1241,22 @@ type JustReadFeedItem { author: String broadcastCount: Int comments: [String!] - createdAt: Date + createdAt: Date! dir: String highlights: String id: ID! languageCode: String likedCount: Int previewContent: String - publishedAt: Date savedCount: Int seen_at: Date - source: JustReadFeedSource + siteName: String + sourceIcon: String + sourceName: String! thumbnail: String title: String! topic: String - updatedAt: Date + updatedAt: Date! url: String! wordCount: Int } @@ -1268,7 +1269,7 @@ type JustReadFeedSource { name: String! thumbnail: String topics: [String!] - url: String + url: String! } type JustReadFeedSuccess { @@ -1563,7 +1564,7 @@ type Mutation { } type MySubscriptionRootType { - justReadFeed: JustReadFeedResult! + hello: String } type NewsletterEmail { @@ -1703,6 +1704,7 @@ type Query { hello: String integration(name: String!): IntegrationResult! integrations: IntegrationsResult! + justReadFeed: JustReadFeedResult! labels: LabelsResult! me: User newsletterEmails: NewsletterEmailsResult! diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts index a72cf61de..54340d169 100644 --- a/packages/api/src/jobs/update_just_read_feed.ts +++ b/packages/api/src/jobs/update_just_read_feed.ts @@ -18,6 +18,7 @@ interface JustReadFeedItem { title: string url: string createdAt: Date + updatedAt: Date sourceName: string siteName?: string @@ -59,6 +60,7 @@ const libraryItemToFeedItem = ( sourceIcon: user.profile.pictureUrl || undefined, sourceName: user.name, siteName: item.siteName || undefined, + updatedAt: item.updatedAt, }) const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({ @@ -76,6 +78,7 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({ sourceIcon: item.sourceIcon, sourceName: item.sourceName, siteName: item.siteName, + updatedAt: item.updatedAt, }) interface FeedItemScore { @@ -97,20 +100,28 @@ const selectCandidates = async ( userId ) + logger.info(`Found ${libraryItems.length} library items`) + // map library items to candidates const privateCandidates: Array = libraryItems.map( (libraryItem) => libraryItemToFeedItem(user, libraryItem) ) + logger.info(`Found ${privateCandidates.length} private candidates`) + // get candidates from public inventory const publicItems = await findUnseenPublicItems(userId, { limit: 100, }) + logger.info(`Found ${publicItems.length} public items`) + // map public items to candidates const publicCandidates: Array = publicItems.map(publicItemToFeedItem) + logger.info(`Found ${publicCandidates.length} public candidates`) + // combine candidates return [...privateCandidates, ...publicCandidates] } @@ -124,29 +135,37 @@ const rankFeedItems = async ( } // TODO: get score of candidates - const API_URL = 'https://score.omnivore.app' // fake URL + // const API_URL = 'https://score.omnivore.app' // fake URL - const response = await fetch(API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ candidates: feedItems }), - }) + // const response = await fetch(API_URL, { + // method: 'POST', + // headers: { + // 'Content-Type': 'application/json', + // }, + // body: JSON.stringify({ candidates: feedItems }), + // }) - if (!response.ok) { - throw new Error(`Failed to score candidates: ${response.statusText}`) - } + // if (!response.ok) { + // throw new Error(`Failed to score candidates: ${response.statusText}`) + // } - const scores = (await response.json()) as Array + // const scores = (await response.json()) as Array + + // fake scores + const scores: Array = feedItems.map((item) => ({ + id: item.id, + score: Math.random(), + })) // rank candidates by score in ascending order - return feedItems.sort((a, b) => { + feedItems.sort((a, b) => { const scoreA = scores.find((score) => score.id === a.id)?.score || 0 const scoreB = scores.find((score) => score.id === b.id)?.score || 0 return scoreA - scoreB }) + + return Promise.resolve(feedItems) } const redisKey = (userId: string) => `just-read-feed:${userId}` @@ -209,6 +228,7 @@ const appendItemsToFeed = async ( pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1)) pipeline.zremrangebyscore(key, '-inf', Date.now()) + logger.info('Adding feed items to redis') await pipeline.exec() } @@ -227,11 +247,18 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => { // TODO: integrity check on candidates? + logger.info('Ranking feed items') const rankedFeedItems = await rankFeedItems(feedItems) + if (rankedFeedItems.length === 0) { + logger.info('No feed items to append') + return + } + logger.info('Filtering feed items') // TODO: filtering // get top 100 ranked feed items const filteredFeedItems = rankedFeedItems.slice(0, 100) + logger.info('Appending feed items to feed') await appendItemsToFeed(filteredFeedItems, userId) } diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts index c907d1ddb..2343bd6b6 100644 --- a/packages/api/src/resolvers/just_read_feed/index.ts +++ b/packages/api/src/resolvers/just_read_feed/index.ts @@ -1,5 +1,6 @@ import { JustReadFeedError, JustReadFeedSuccess } from '../../generated/graphql' import { getJustReadFeed } from '../../jobs/update_just_read_feed' +import { enqueueUpdateJustReadFeed } from '../../utils/createTask' import { authorized } from '../../utils/gql-utils' export const justReadFeedResolver = authorized< @@ -9,5 +10,11 @@ export const justReadFeedResolver = authorized< const feed = await getJustReadFeed(uid) log.info('Just read feed fetched') + if (feed.topics.length === 0) { + await enqueueUpdateJustReadFeed({ + userId: uid, + }) + } + return feed }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index ad6d04cbc..9220d141d 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -3107,7 +3107,6 @@ const schema = gql` url: String! topics: [String!] thumbnail: String - url: String languageCodes: [String!] } @@ -3117,21 +3116,22 @@ const schema = gql` topic: String url: String! thumbnail: String - publishedAt: Date - source: JustReadFeedSource previewContent: String highlights: String savedCount: Int likedCount: Int broadcastCount: Int - createdAt: Date - updatedAt: Date + createdAt: Date! + updatedAt: Date! comments: [String!] author: String languageCode: String dir: String seen_at: Date wordCount: Int + sourceName: String! + sourceIcon: String + siteName: String } type JustReadFeedTopic { @@ -3156,7 +3156,7 @@ const schema = gql` union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError type MySubscriptionRootType { - justReadFeed: JustReadFeedResult! + hello: String # for testing only } # Mutations @@ -3354,6 +3354,7 @@ const schema = gql` feeds(input: FeedsInput!): FeedsResult! discoverFeeds: DiscoverFeedResult! scanFeeds(input: ScanFeedsInput!): ScanFeedsResult! + justReadFeed: JustReadFeedResult! } schema { diff --git a/packages/api/src/services/just_read_feed.ts b/packages/api/src/services/just_read_feed.ts index c34fc7151..7d8a18103 100644 --- a/packages/api/src/services/just_read_feed.ts +++ b/packages/api/src/services/just_read_feed.ts @@ -1,5 +1,5 @@ import { PublicItem } from '../entity/public_item' -import { getRepository } from '../repository' +import { authTrx } from '../repository' export const findUnseenPublicItems = async ( userId: string, @@ -7,17 +7,29 @@ export const findUnseenPublicItems = async ( limit?: number offset?: number } -): Promise> => - getRepository(PublicItem) - .createQueryBuilder('public_item') - .leftJoin( - 'omnivore.public_item_interactions', - 'interaction', - 'interaction.public_item_id = public_item.id' - ) - .where('interaction.user_id = :userId', { userId }) - .andWhere('interaction.seen_at IS NULL') - .orderBy('public_item.created_at', 'DESC') - .limit(options.limit) - .offset(options.offset) - .getMany() +): Promise> => { + return authTrx( + async (tx) => + tx + .getRepository(PublicItem) + .createQueryBuilder('public_item') + .leftJoin( + 'public_item_interactions', + 'interaction', + 'interaction.public_item_id = public_item.id' + ) + .innerJoin( + 'public_item_stats', + 'stats', + 'stats.public_item_id = public_item.id' + ) + .where('interaction.user_id = :userId', { userId }) + .andWhere('interaction.seen_at IS NULL') + .orderBy('public_item.createdAt', 'DESC') + .take(options.limit) + .skip(options.offset) + .getMany(), + undefined, + userId + ) +} diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql index 59df6453a..32a618660 100755 --- a/packages/db/migrations/0177.do.public_item.sql +++ b/packages/db/migrations/0177.do.public_item.sql @@ -17,6 +17,7 @@ CREATE TABLE omnivore.public_item_source ( ); CREATE TRIGGER update_public_item_source_modtime BEFORE UPDATE ON omnivore.public_item_source FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +GRANT SELECT ON omnivore.public_item_source TO omnivore_user; CREATE TABLE omnivore.public_item ( @@ -41,6 +42,7 @@ CREATE TABLE omnivore.public_item ( ); CREATE TRIGGER update_public_item_modtime BEFORE UPDATE ON omnivore.public_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +GRANT SELECT ON omnivore.public_item TO omnivore_user; CREATE TABLE omnivore.public_item_stats ( @@ -55,6 +57,7 @@ CREATE TABLE omnivore.public_item_stats ( CREATE INDEX public_item_stats_public_item_id_idx ON omnivore.public_item_stats(public_item_id); CREATE TRIGGER update_public_item_stats_modtime BEFORE UPDATE ON omnivore.public_item_stats FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +GRANT SELECT ON omnivore.public_item_stats TO omnivore_user; CREATE TABLE omnivore.public_item_interactions ( @@ -66,13 +69,14 @@ CREATE TABLE omnivore.public_item_interactions ( broadcasted_at TIMESTAMPTZ, seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, digested_at TIMESTAMPTZ, - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX public_item_interaction_user_id_idx ON omnivore.public_item_interactions(user_id); CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_interactions(public_item_id); CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore.public_item_interactions FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +GRANT SELECT, INSERT, UPDATE ON omnivore.public_item_interactions TO omnivore_user; ALTER TABLE omnivore.library_item