From 9b0b378c3905fc28ef0ad12de600db00f438e45d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Nov 2023 16:26:03 +0800 Subject: [PATCH] check is_fetching_content and only fetch content if required for rss feed items --- packages/api/src/entity/library_item.ts | 21 +++++++++++++++++++ packages/api/src/entity/subscription.ts | 6 ++++++ packages/api/src/generated/graphql.ts | 4 ++++ packages/api/src/generated/schema.graphql | 4 ++++ .../api/src/resolvers/subscriptions/index.ts | 5 ++++- packages/api/src/routers/svc/rss_feed.ts | 3 ++- packages/api/src/schema.ts | 4 ++++ packages/api/src/utils/createTask.ts | 2 ++ packages/rss-handler/src/index.ts | 4 +++- 9 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 5689c83d6..07cc2b879 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -196,4 +196,25 @@ export class LibraryItem { @Column('text', { nullable: true }) recommenderNames?: string[] | null + + @Column('timestamptz') + hiddenAt?: Date | null + + @Column('timestamptz') + sharedAt?: Date | null + + @Column('text') + sharedBy?: string | null + + @Column('jsonb') + links?: Record[] | null + + @Column('text') + previewContent?: string | null + + @Column('timestamptz') + seenAt?: Date | null + + @Column('boolean') + isInLibrary!: boolean } diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index fea63d890..af5c54fa4 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -70,4 +70,10 @@ export class Subscription { @Column('timestamp', { nullable: true }) scheduledAt?: Date | null + + @Column('boolean') + isPublic?: boolean | null + + @Column('boolean') + isFetchingContent?: boolean | null } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 849afba50..d3f3147c8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2678,6 +2678,8 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { + isFetchingContent?: InputMaybe; + isPublic?: InputMaybe; subscriptionType?: InputMaybe; url: Scalars['String']; }; @@ -3038,6 +3040,8 @@ export enum UpdateSubscriptionErrorCode { export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; + isFetchingContent?: InputMaybe; + isPublic?: InputMaybe; lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; name?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 8b704c249..5e1ae27af 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2112,6 +2112,8 @@ enum SubscribeErrorCode { } input SubscribeInput { + isFetchingContent: Boolean + isPublic: Boolean subscriptionType: SubscriptionType url: String! } @@ -2443,6 +2445,8 @@ enum UpdateSubscriptionErrorCode { input UpdateSubscriptionInput { description: String id: ID! + isFetchingContent: Boolean + isPublic: Boolean lastFetchedAt: Date lastFetchedChecksum: String name: String diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 8d5b74a83..20d9825c1 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -213,6 +213,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [updatedSubscription.lastFetchedAt || null], checksums: [updatedSubscription.lastFetchedChecksum || null], + isFetchingContents: [!!updatedSubscription.isFetchingContent], }) return { @@ -228,7 +229,7 @@ export const subscribeResolver = authorized< // limit number of rss subscriptions to 150 const results = (await getRepository(Subscription).query( `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + select $1, $2, $3, $4, $5, $6, $8 from omnivore.subscriptions where user_id = $5 and type = 'RSS' and status = 'ACTIVE' having count(*) < $7 returning *;`, @@ -240,6 +241,7 @@ export const subscribeResolver = authorized< uid, feed.image?.url || null, MAX_RSS_SUBSCRIPTIONS, + !!input.isFetchingContent, ] )) as Subscription[] @@ -259,6 +261,7 @@ export const subscribeResolver = authorized< scheduledDates: [new Date()], // fetch immediately fetchedDates: [null], checksums: [null], + isFetchingContents: [!!newSubscription.isFetchingContent], }) return { diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 3b6a21247..793fa8843 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -34,7 +34,8 @@ export function rssFeedRouter() { ARRAY_AGG(user_id) AS "userIds", ARRAY_AGG(last_fetched_at) AS "fetchedDates", ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS checksums + ARRAY_AGG(last_fetched_checksum) AS checksums, + ARRAY_AGG(coalesce(is_fetching_content, false)) AS "isFetchingContents", FROM omnivore.subscriptions WHERE diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4cd073b8f..e272c9d76 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2554,6 +2554,8 @@ const schema = gql` input SubscribeInput { url: String! subscriptionType: SubscriptionType + isPublic: Boolean + isFetchingContent: Boolean } input UpdateSubscriptionInput { @@ -2564,6 +2566,8 @@ const schema = gql` lastFetchedChecksum: String status: SubscriptionStatus scheduledAt: Date + isPublic: Boolean + isFetchingContent: Boolean } union UpdateSubscriptionResult = diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 643772116..8764f4495 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -615,6 +615,7 @@ export interface RssSubscriptionGroup { fetchedDates: (Date | null)[] scheduledDates: Date[] checksums: (string | null)[] + isFetchingContents: boolean[] } export const enqueueRssFeedFetch = async ( @@ -632,6 +633,7 @@ export const enqueueRssFeedFetch = async ( timestamp.getTime() ), // unix timestamp in milliseconds userIds: subscriptionGroup.userIds, + isFetchingContents: subscriptionGroup.isFetchingContents, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 04abddecb..e5b9f68f2 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -14,6 +14,7 @@ interface RssFeedRequest { scheduledTimestamps: number[] // unix timestamp in milliseconds lastFetchedChecksums: string[] userIds: string[] + isFetchingContents: boolean[] } // link can be a string or an object @@ -26,7 +27,8 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { 'lastFetchedTimestamps' in body && 'scheduledTimestamps' in body && 'userIds' in body && - 'lastFetchedChecksums' in body + 'lastFetchedChecksums' in body && + 'isFetchingContents' in body ) }