Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

grant db permission

Browse Source
This commit is contained in:
Hongbo Wu
2024-05-24 15:32:14 +08:00
parent a48a614676
commit f2d2e852e1
8 changed files with 111 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/api/src/entity/public_item.ts
View File

@ -6,7 +6,7 @@ import {
UpdateDateColumn,
} from 'typeorm'
@Entity()
@Entity({ name: 'public_item' })
export class PublicItem {
@PrimaryGeneratedColumn('uuid')
id!: string

28
packages/api/src/generated/graphql.ts
View File

@ -1378,21 +1378,22 @@ export type JustReadFeedItem = {
author?: Maybe<Scalars['String']>;
broadcastCount?: Maybe<Scalars['Int']>;
comments?: Maybe<Array<Scalars['String']>>;
createdAt?: Maybe<Scalars['Date']>;
createdAt: Scalars['Date'];
dir?: Maybe<Scalars['String']>;
highlights?: Maybe<Scalars['String']>;
id: Scalars['ID'];
languageCode?: Maybe<Scalars['String']>;
likedCount?: Maybe<Scalars['Int']>;
previewContent?: Maybe<Scalars['String']>;
publishedAt?: Maybe<Scalars['Date']>;
savedCount?: Maybe<Scalars['Int']>;
seen_at?: Maybe<Scalars['Date']>;
source?: Maybe<JustReadFeedSource>;
siteName?: Maybe<Scalars['String']>;
sourceIcon?: Maybe<Scalars['String']>;
sourceName: Scalars['String'];
thumbnail?: Maybe<Scalars['String']>;
title: Scalars['String'];
topic?: Maybe<Scalars['String']>;
updatedAt?: Maybe<Scalars['Date']>;
updatedAt: Scalars['Date'];
url: Scalars['String'];
wordCount?: Maybe<Scalars['Int']>;
};
@ -1406,7 +1407,7 @@ export type JustReadFeedSource = {
name: Scalars['String'];
thumbnail?: Maybe<Scalars['String']>;
topics?: Maybe<Array<Scalars['String']>>;
url?: Maybe<Scalars['String']>;
url: Scalars['String'];
};
export type JustReadFeedSuccess = {
@ -2075,7 +2076,7 @@ export type MutationUploadImportFileArgs = {
export type MySubscriptionRootType = {
__typename?: 'MySubscriptionRootType';
justReadFeed: JustReadFeedResult;
hello?: Maybe<Scalars['String']>;
};
export type NewsletterEmail = {
@ -2224,6 +2225,7 @@ export type Query = {
hello?: Maybe<Scalars['String']>;
integration: IntegrationResult;
integrations: IntegrationsResult;
justReadFeed: JustReadFeedResult;
labels: LabelsResult;
me?: Maybe<User>;
newsletterEmails: NewsletterEmailsResult;
@ -6075,21 +6077,22 @@ export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType
author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
broadcastCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
comments?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
createdAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
highlights?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
languageCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
likedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
publishedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
savedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
source?: Resolver<Maybe<ResolversTypes['JustReadFeedSource']>, ParentType, ContextType>;
siteName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
sourceIcon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
sourceName?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
topic?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
updatedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
wordCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@ -6105,7 +6108,7 @@ export type JustReadFeedSourceResolvers<ContextType = ResolverContext, ParentTyp
name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
topics?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
@ -6358,7 +6361,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
};
export type MySubscriptionRootTypeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MySubscriptionRootType'] = ResolversParentTypes['MySubscriptionRootType']> = {
justReadFeed?: SubscriptionResolver<ResolversTypes['JustReadFeedResult'], "justReadFeed", ParentType, ContextType>;
hello?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "hello", ParentType, ContextType>;
};
export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['NewsletterEmail'] = ResolversParentTypes['NewsletterEmail']> = {
@ -6452,6 +6455,7 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
hello?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
integration?: Resolver<ResolversTypes['IntegrationResult'], ParentType, ContextType, RequireFields<QueryIntegrationArgs, 'name'>>;
integrations?: Resolver<ResolversTypes['IntegrationsResult'], ParentType, ContextType>;
justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType>;
labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;

14
packages/api/src/generated/schema.graphql
View File

@ -1241,21 +1241,22 @@ type JustReadFeedItem {
author: String
broadcastCount: Int
comments: [String!]
createdAt: Date
createdAt: Date!
dir: String
highlights: String
id: ID!
languageCode: String
likedCount: Int
previewContent: String
publishedAt: Date
savedCount: Int
seen_at: Date
source: JustReadFeedSource
siteName: String
sourceIcon: String
sourceName: String!
thumbnail: String
title: String!
topic: String
updatedAt: Date
updatedAt: Date!
url: String!
wordCount: Int
}
@ -1268,7 +1269,7 @@ type JustReadFeedSource {
name: String!
thumbnail: String
topics: [String!]
url: String
url: String!
}
type JustReadFeedSuccess {
@ -1563,7 +1564,7 @@ type Mutation {
}
type MySubscriptionRootType {
justReadFeed: JustReadFeedResult!
hello: String
}
type NewsletterEmail {
@ -1703,6 +1704,7 @@ type Query {
hello: String
integration(name: String!): IntegrationResult!
integrations: IntegrationsResult!
justReadFeed: JustReadFeedResult!
labels: LabelsResult!
me: User
newsletterEmails: NewsletterEmailsResult!

53
packages/api/src/jobs/update_just_read_feed.ts
View File

@ -18,6 +18,7 @@ interface JustReadFeedItem {
title: string
url: string
createdAt: Date
updatedAt: Date
sourceName: string
siteName?: string
@ -59,6 +60,7 @@ const libraryItemToFeedItem = (
sourceIcon: user.profile.pictureUrl || undefined,
sourceName: user.name,
siteName: item.siteName || undefined,
updatedAt: item.updatedAt,
})
const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
@ -76,6 +78,7 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
sourceIcon: item.sourceIcon,
sourceName: item.sourceName,
siteName: item.siteName,
updatedAt: item.updatedAt,
})
interface FeedItemScore {
@ -97,20 +100,28 @@ const selectCandidates = async (
userId
)
logger.info(`Found ${libraryItems.length} library items`)
// map library items to candidates
const privateCandidates: Array<JustReadFeedItem> = libraryItems.map(
(libraryItem) => libraryItemToFeedItem(user, libraryItem)
)
logger.info(`Found ${privateCandidates.length} private candidates`)
// get candidates from public inventory
const publicItems = await findUnseenPublicItems(userId, {
limit: 100,
})
logger.info(`Found ${publicItems.length} public items`)
// map public items to candidates
const publicCandidates: Array<JustReadFeedItem> =
publicItems.map(publicItemToFeedItem)
logger.info(`Found ${publicCandidates.length} public candidates`)
// combine candidates
return [...privateCandidates, ...publicCandidates]
}
@ -124,29 +135,37 @@ const rankFeedItems = async (
}
// TODO: get score of candidates
const API_URL = 'https://score.omnivore.app' // fake URL
// const API_URL = 'https://score.omnivore.app' // fake URL
const response = await fetch(API_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ candidates: feedItems }),
})
// const response = await fetch(API_URL, {
// method: 'POST',
// headers: {
// 'Content-Type': 'application/json',
// },
// body: JSON.stringify({ candidates: feedItems }),
// })
if (!response.ok) {
throw new Error(`Failed to score candidates: ${response.statusText}`)
}
// if (!response.ok) {
// throw new Error(`Failed to score candidates: ${response.statusText}`)
// }
const scores = (await response.json()) as Array<FeedItemScore>
// const scores = (await response.json()) as Array<FeedItemScore>
// fake scores
const scores: Array<FeedItemScore> = feedItems.map((item) => ({
id: item.id,
score: Math.random(),
}))
// rank candidates by score in ascending order
return feedItems.sort((a, b) => {
feedItems.sort((a, b) => {
const scoreA = scores.find((score) => score.id === a.id)?.score || 0
const scoreB = scores.find((score) => score.id === b.id)?.score || 0
return scoreA - scoreB
})
return Promise.resolve(feedItems)
}
const redisKey = (userId: string) => `just-read-feed:${userId}`
@ -209,6 +228,7 @@ const appendItemsToFeed = async (
pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1))
pipeline.zremrangebyscore(key, '-inf', Date.now())
logger.info('Adding feed items to redis')
await pipeline.exec()
}
@ -227,11 +247,18 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
// TODO: integrity check on candidates?
logger.info('Ranking feed items')
const rankedFeedItems = await rankFeedItems(feedItems)
if (rankedFeedItems.length === 0) {
logger.info('No feed items to append')
return
}
logger.info('Filtering feed items')
// TODO: filtering
// get top 100 ranked feed items
const filteredFeedItems = rankedFeedItems.slice(0, 100)
logger.info('Appending feed items to feed')
await appendItemsToFeed(filteredFeedItems, userId)
}

7
packages/api/src/resolvers/just_read_feed/index.ts
View File

@ -1,5 +1,6 @@
import { JustReadFeedError, JustReadFeedSuccess } from '../../generated/graphql'
import { getJustReadFeed } from '../../jobs/update_just_read_feed'
import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
import { authorized } from '../../utils/gql-utils'
export const justReadFeedResolver = authorized<
@ -9,5 +10,11 @@ export const justReadFeedResolver = authorized<
const feed = await getJustReadFeed(uid)
log.info('Just read feed fetched')
if (feed.topics.length === 0) {
await enqueueUpdateJustReadFeed({
userId: uid,
})
}
return feed
})

13
packages/api/src/schema.ts
View File

@ -3107,7 +3107,6 @@ const schema = gql`
url: String!
topics: [String!]
thumbnail: String
url: String
languageCodes: [String!]
}
@ -3117,21 +3116,22 @@ const schema = gql`
topic: String
url: String!
thumbnail: String
publishedAt: Date
source: JustReadFeedSource
previewContent: String
highlights: String
savedCount: Int
likedCount: Int
broadcastCount: Int
createdAt: Date
updatedAt: Date
createdAt: Date!
updatedAt: Date!
comments: [String!]
author: String
languageCode: String
dir: String
seen_at: Date
wordCount: Int
sourceName: String!
sourceIcon: String
siteName: String
}
type JustReadFeedTopic {
@ -3156,7 +3156,7 @@ const schema = gql`
union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError
type MySubscriptionRootType {
justReadFeed: JustReadFeedResult!
hello: String # for testing only
}
# Mutations
@ -3354,6 +3354,7 @@ const schema = gql`
feeds(input: FeedsInput!): FeedsResult!
discoverFeeds: DiscoverFeedResult!
scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
justReadFeed: JustReadFeedResult!
}
schema {

42
packages/api/src/services/just_read_feed.ts
View File

@ -1,5 +1,5 @@
import { PublicItem } from '../entity/public_item'
import { getRepository } from '../repository'
import { authTrx } from '../repository'
export const findUnseenPublicItems = async (
userId: string,
@ -7,17 +7,29 @@ export const findUnseenPublicItems = async (
limit?: number
offset?: number
}
): Promise<Array<PublicItem>> =>
getRepository(PublicItem)
.createQueryBuilder('public_item')
.leftJoin(
'omnivore.public_item_interactions',
'interaction',
'interaction.public_item_id = public_item.id'
)
.where('interaction.user_id = :userId', { userId })
.andWhere('interaction.seen_at IS NULL')
.orderBy('public_item.created_at', 'DESC')
.limit(options.limit)
.offset(options.offset)
.getMany()
): Promise<Array<PublicItem>> => {
return authTrx(
async (tx) =>
tx
.getRepository(PublicItem)
.createQueryBuilder('public_item')
.leftJoin(
'public_item_interactions',
'interaction',
'interaction.public_item_id = public_item.id'
)
.innerJoin(
'public_item_stats',
'stats',
'stats.public_item_id = public_item.id'
)
.where('interaction.user_id = :userId', { userId })
.andWhere('interaction.seen_at IS NULL')
.orderBy('public_item.createdAt', 'DESC')
.take(options.limit)
.skip(options.offset)
.getMany(),
undefined,
userId
)
}

6
packages/db/migrations/0177.do.public_item.sql
View File

@ -17,6 +17,7 @@ CREATE TABLE omnivore.public_item_source (
);
CREATE TRIGGER update_public_item_source_modtime BEFORE UPDATE ON omnivore.public_item_source FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
GRANT SELECT ON omnivore.public_item_source TO omnivore_user;
CREATE TABLE omnivore.public_item (
@ -41,6 +42,7 @@ CREATE TABLE omnivore.public_item (
);
CREATE TRIGGER update_public_item_modtime BEFORE UPDATE ON omnivore.public_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
GRANT SELECT ON omnivore.public_item TO omnivore_user;
CREATE TABLE omnivore.public_item_stats (
@ -55,6 +57,7 @@ CREATE TABLE omnivore.public_item_stats (
CREATE INDEX public_item_stats_public_item_id_idx ON omnivore.public_item_stats(public_item_id);
CREATE TRIGGER update_public_item_stats_modtime BEFORE UPDATE ON omnivore.public_item_stats FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
GRANT SELECT ON omnivore.public_item_stats TO omnivore_user;
CREATE TABLE omnivore.public_item_interactions (
@ -66,13 +69,14 @@ CREATE TABLE omnivore.public_item_interactions (
broadcasted_at TIMESTAMPTZ,
seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
digested_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX public_item_interaction_user_id_idx ON omnivore.public_item_interactions(user_id);
CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_interactions(public_item_id);
CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore.public_item_interactions FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
GRANT SELECT, INSERT, UPDATE ON omnivore.public_item_interactions TO omnivore_user;
ALTER TABLE omnivore.library_item