Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3247 from omnivore-app/feature/feeds

Browse Source 
feat: fetch content and folder settings for newsletters and subscriptions
This commit is contained in:
Hongbo Wu
2023-12-15 16:46:02 +08:00
committed by GitHub
parent 29a13345ff b86ee139e0
commit 58f16ec62c
17 changed files with 237 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/api/src/entity/newsletter_email.ts
View File

@ -34,4 +34,13 @@ export class NewsletterEmail {
@OneToMany(() => Subscription, (subscription) => subscription.newsletterEmail)
subscriptions!: Subscription[]
@Column('text')
folder!: string
@Column('text')
name?: string | null
@Column('text')
description?: string | null
}

6
packages/api/src/entity/subscription.ts
View File

@ -76,4 +76,10 @@ export class Subscription {
@Column('boolean')
autoAddToLibrary?: boolean | null
@Column('boolean')
fetchContent!: boolean
@Column('text')
folder!: string
}

29
packages/api/src/generated/graphql.ts
View File

@ -438,6 +438,12 @@ export enum CreateNewsletterEmailErrorCode {
Unauthorized = 'UNAUTHORIZED'
}
export type CreateNewsletterEmailInput = {
description?: InputMaybe<Scalars['String']>;
folder?: InputMaybe<Scalars['String']>;
name?: InputMaybe<Scalars['String']>;
};
export type CreateNewsletterEmailResult = CreateNewsletterEmailError | CreateNewsletterEmailSuccess;
export type CreateNewsletterEmailSuccess = {
@ -1415,6 +1421,11 @@ export type MutationCreateLabelArgs = {
};
export type MutationCreateNewsletterEmailArgs = {
input?: InputMaybe<CreateNewsletterEmailInput>;
};
export type MutationDeleteAccountArgs = {
userID: Scalars['ID'];
};
@ -1677,7 +1688,10 @@ export type NewsletterEmail = {
address: Scalars['String'];
confirmationCode?: Maybe<Scalars['String']>;
createdAt: Scalars['Date'];
description?: Maybe<Scalars['String']>;
folder: Scalars['String'];
id: Scalars['ID'];
name?: Maybe<Scalars['String']>;
subscriptionCount: Scalars['Int'];
};
@ -2752,6 +2766,8 @@ export enum SubscribeErrorCode {
export type SubscribeInput = {
autoAddToLibrary?: InputMaybe<Scalars['Boolean']>;
fetchContent?: InputMaybe<Scalars['Boolean']>;
folder?: InputMaybe<Scalars['String']>;
isPrivate?: InputMaybe<Scalars['Boolean']>;
subscriptionType?: InputMaybe<SubscriptionType>;
url: Scalars['String'];
@ -2770,6 +2786,8 @@ export type Subscription = {
count: Scalars['Int'];
createdAt: Scalars['Date'];
description?: Maybe<Scalars['String']>;
fetchContent: Scalars['Boolean'];
folder: Scalars['String'];
icon?: Maybe<Scalars['String']>;
id: Scalars['ID'];
isPrivate?: Maybe<Scalars['Boolean']>;
@ -3116,6 +3134,8 @@ export enum UpdateSubscriptionErrorCode {
export type UpdateSubscriptionInput = {
autoAddToLibrary?: InputMaybe<Scalars['Boolean']>;
description?: InputMaybe<Scalars['String']>;
fetchContent?: InputMaybe<Scalars['Boolean']>;
folder?: InputMaybe<Scalars['String']>;
id: Scalars['ID'];
isPrivate?: InputMaybe<Scalars['Boolean']>;
lastFetchedAt?: InputMaybe<Scalars['Date']>;
@ -3526,6 +3546,7 @@ export type ResolversTypes = {
CreateLabelSuccess: ResolverTypeWrapper<CreateLabelSuccess>;
CreateNewsletterEmailError: ResolverTypeWrapper<CreateNewsletterEmailError>;
CreateNewsletterEmailErrorCode: CreateNewsletterEmailErrorCode;
CreateNewsletterEmailInput: CreateNewsletterEmailInput;
CreateNewsletterEmailResult: ResolversTypes['CreateNewsletterEmailError'] | ResolversTypes['CreateNewsletterEmailSuccess'];
CreateNewsletterEmailSuccess: ResolverTypeWrapper<CreateNewsletterEmailSuccess>;
CreateReactionError: ResolverTypeWrapper<CreateReactionError>;
@ -4036,6 +4057,7 @@ export type ResolversParentTypes = {
CreateLabelResult: ResolversParentTypes['CreateLabelError'] | ResolversParentTypes['CreateLabelSuccess'];
CreateLabelSuccess: CreateLabelSuccess;
CreateNewsletterEmailError: CreateNewsletterEmailError;
CreateNewsletterEmailInput: CreateNewsletterEmailInput;
CreateNewsletterEmailResult: ResolversParentTypes['CreateNewsletterEmailError'] | ResolversParentTypes['CreateNewsletterEmailSuccess'];
CreateNewsletterEmailSuccess: CreateNewsletterEmailSuccess;
CreateReactionError: CreateReactionError;
@ -5345,7 +5367,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
createGroup?: Resolver<ResolversTypes['CreateGroupResult'], ParentType, ContextType, RequireFields<MutationCreateGroupArgs, 'input'>>;
createHighlight?: Resolver<ResolversTypes['CreateHighlightResult'], ParentType, ContextType, RequireFields<MutationCreateHighlightArgs, 'input'>>;
createLabel?: Resolver<ResolversTypes['CreateLabelResult'], ParentType, ContextType, RequireFields<MutationCreateLabelArgs, 'input'>>;
createNewsletterEmail?: Resolver<ResolversTypes['CreateNewsletterEmailResult'], ParentType, ContextType>;
createNewsletterEmail?: Resolver<ResolversTypes['CreateNewsletterEmailResult'], ParentType, ContextType, Partial<MutationCreateNewsletterEmailArgs>>;
deleteAccount?: Resolver<ResolversTypes['DeleteAccountResult'], ParentType, ContextType, RequireFields<MutationDeleteAccountArgs, 'userID'>>;
deleteFilter?: Resolver<ResolversTypes['DeleteFilterResult'], ParentType, ContextType, RequireFields<MutationDeleteFilterArgs, 'id'>>;
deleteHighlight?: Resolver<ResolversTypes['DeleteHighlightResult'], ParentType, ContextType, RequireFields<MutationDeleteHighlightArgs, 'highlightId'>>;
@ -5404,7 +5426,10 @@ export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType e
address?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
confirmationCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
description?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
folder?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
name?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
subscriptionCount?: Resolver<ResolversTypes['Int'], ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
@ -6036,6 +6061,8 @@ export type SubscriptionResolvers<ContextType = ResolverContext, ParentType exte
count?: SubscriptionResolver<ResolversTypes['Int'], "count", ParentType, ContextType>;
createdAt?: SubscriptionResolver<ResolversTypes['Date'], "createdAt", ParentType, ContextType>;
description?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "description", ParentType, ContextType>;
fetchContent?: SubscriptionResolver<ResolversTypes['Boolean'], "fetchContent", ParentType, ContextType>;
folder?: SubscriptionResolver<ResolversTypes['String'], "folder", ParentType, ContextType>;
icon?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "icon", ParentType, ContextType>;
id?: SubscriptionResolver<ResolversTypes['ID'], "id", ParentType, ContextType>;
isPrivate?: SubscriptionResolver<Maybe<ResolversTypes['Boolean']>, "isPrivate", ParentType, ContextType>;

17
packages/api/src/generated/schema.graphql
View File

@ -383,6 +383,12 @@ enum CreateNewsletterEmailErrorCode {
UNAUTHORIZED
}
input CreateNewsletterEmailInput {
description: String
folder: String
name: String
}
union CreateNewsletterEmailResult = CreateNewsletterEmailError | CreateNewsletterEmailSuccess
type CreateNewsletterEmailSuccess {
@ -1182,7 +1188,7 @@ type Mutation {
createGroup(input: CreateGroupInput!): CreateGroupResult!
createHighlight(input: CreateHighlightInput!): CreateHighlightResult!
createLabel(input: CreateLabelInput!): CreateLabelResult!
createNewsletterEmail: CreateNewsletterEmailResult!
createNewsletterEmail(input: CreateNewsletterEmailInput): CreateNewsletterEmailResult!
deleteAccount(userID: ID!): DeleteAccountResult!
deleteFilter(id: ID!): DeleteFilterResult!
deleteHighlight(highlightId: ID!): DeleteHighlightResult!
@ -1241,7 +1247,10 @@ type NewsletterEmail {
address: String!
confirmationCode: String
createdAt: Date!
description: String
folder: String!
id: ID!
name: String
subscriptionCount: Int!
}
@ -2169,6 +2178,8 @@ enum SubscribeErrorCode {
input SubscribeInput {
autoAddToLibrary: Boolean
fetchContent: Boolean
folder: String
isPrivate: Boolean
subscriptionType: SubscriptionType
url: String!
@ -2185,6 +2196,8 @@ type Subscription {
count: Int!
createdAt: Date!
description: String
fetchContent: Boolean!
folder: String!
icon: String
id: ID!
isPrivate: Boolean
@ -2504,6 +2517,8 @@ enum UpdateSubscriptionErrorCode {
input UpdateSubscriptionInput {
autoAddToLibrary: Boolean
description: String
fetchContent: Boolean
folder: String
id: ID!
isPrivate: Boolean
lastFetchedAt: Date

44
packages/api/src/resolvers/article/index.ts
View File

@ -415,7 +415,6 @@ export const getArticleResolver = authorized<
deletedAt: IsNull(),
},
relations: {
labels: true,
highlights: {
user: true,
labels: true,
@ -912,7 +911,7 @@ export const moveToFolderResolver = authorized<
MoveToFolderSuccess,
MoveToFolderError,
MutationMoveToFolderArgs
>(async (_, { id, folder }, { authTrx, pubsub, uid }) => {
>(async (_, { id, folder }, { authTrx, log, pubsub, uid }) => {
analytics.track({
userId: uid,
event: 'move_to_folder',
@ -945,24 +944,6 @@ export const moveToFolderResolver = authorized<
const savedAt = new Date()
// // if the content is not fetched yet, create a page save request
// if (!item.readableContent) {
// const articleSavingRequest = await createPageSaveRequest({
// userId: uid,
// url: item.originalUrl,
// articleSavingRequestId: id,
// priority: 'high',
// publishedAt: item.publishedAt || undefined,
// savedAt,
// pubsub,
// })
// return {
// __typename: 'MoveToFolderSuccess',
// articleSavingRequest,
// }
// }
await updateLibraryItem(
item.id,
{
@ -973,8 +954,29 @@ export const moveToFolderResolver = authorized<
pubsub
)
// if the content is not fetched yet, create a page save request
if (!item.readableContent) {
try {
await createPageSaveRequest({
userId: uid,
url: item.originalUrl,
articleSavingRequestId: id,
priority: 'high',
publishedAt: item.publishedAt || undefined,
savedAt,
folder,
pubsub,
})
} catch (error) {
log.error('moveToFolderResolver error', error)
return {
errorCodes: [MoveToFolderErrorCode.BadRequest],
}
}
}
return {
__typename: 'MoveToFolderSuccess',
success: true,
}
})

14
packages/api/src/resolvers/function_resolvers.ts
View File

@ -319,6 +319,19 @@ export const functionResolvers = {
if (article.wordCount) return article.wordCount
return article.content ? wordsCount(article.content) : undefined
},
async labels(
article: { id: string; labels?: Label[]; labelNames?: string[] | null },
_: unknown,
ctx: WithDataSourcesContext
) {
if (article.labels) return article.labels
if (article.labelNames && article.labelNames.length > 0) {
return findLabelsByLibraryItemId(article.id, ctx.uid)
}
return []
},
},
Highlight: {
// async reactions(
@ -518,4 +531,5 @@ export const functionResolvers = {
...resultResolveTypeResolver('UpdateSubscription'),
...resultResolveTypeResolver('UpdateEmail'),
...resultResolveTypeResolver('ScanFeeds'),
...resultResolveTypeResolver('MoveToFolder'),
}

18
packages/api/src/resolvers/newsletters/index.ts
View File

@ -7,6 +7,7 @@ import {
DeleteNewsletterEmailError,
DeleteNewsletterEmailErrorCode,
DeleteNewsletterEmailSuccess,
MutationCreateNewsletterEmailArgs,
MutationDeleteNewsletterEmailArgs,
NewsletterEmailsError,
NewsletterEmailsErrorCode,
@ -24,8 +25,9 @@ import { authorized } from '../../utils/helpers'
export const createNewsletterEmailResolver = authorized<
CreateNewsletterEmailSuccess,
CreateNewsletterEmailError
>(async (_parent, _args, { claims, log }) => {
CreateNewsletterEmailError,
MutationCreateNewsletterEmailArgs
>(async (_parent, { input }, { claims, log }) => {
log.info('createNewsletterEmailResolver')
analytics.track({
userId: claims.uid,
@ -36,7 +38,13 @@ export const createNewsletterEmailResolver = authorized<
})
try {
const newsletterEmail = await createNewsletterEmail(claims.uid)
const newsletterEmail = await createNewsletterEmail(
claims.uid,
undefined,
input?.folder || 'following',
input?.name || undefined,
input?.description || undefined
)
return {
newsletterEmail: {
@ -45,7 +53,7 @@ export const createNewsletterEmailResolver = authorized<
},
}
} catch (e) {
log.info(e)
log.error('createNewsletterEmailResolver', e)
return {
errorCodes: [CreateNewsletterEmailErrorCode.BadRequest],
@ -67,7 +75,7 @@ export const newsletterEmailsResolver = authorized<
})),
}
} catch (e) {
log.info(e)
log.error('newsletterEmailsResolver', e)
return {
errorCodes: [NewsletterEmailsErrorCode.BadRequest],

26
packages/api/src/resolvers/subscriptions/index.ts
View File

@ -206,6 +206,9 @@ export const subscribeResolver = authorized<
// re-subscribe
const updatedSubscription = await getRepository(Subscription).save({
...existingSubscription,
fetchContent: input.fetchContent ?? undefined,
folder: input.folder ?? undefined,
isPrivate: input.isPrivate,
status: SubscriptionStatus.Active,
})
@ -217,7 +220,8 @@ export const subscribeResolver = authorized<
scheduledDates: [new Date()], // fetch immediately
fetchedDates: [updatedSubscription.lastFetchedAt || null],
checksums: [updatedSubscription.lastFetchedChecksum || null],
addToLibraryFlags: [!!updatedSubscription.autoAddToLibrary],
fetchContents: [updatedSubscription.fetchContent],
folders: [updatedSubscription.folder],
})
return {
@ -237,20 +241,21 @@ export const subscribeResolver = authorized<
// limit number of rss subscriptions to max
const results = (await getRepository(Subscription).query(
`insert into omnivore.subscriptions (name, url, description, type, user_id, icon, auto_add_to_library, is_private)
select $1, $2, $3, $4, $5, $6, $7, $8 from omnivore.subscriptions
`insert into omnivore.subscriptions (name, url, description, type, user_id, icon, is_private, fetch_content, folder)
select $1, $2, $3, $4, $5, $6, $7, $8, $9 from omnivore.subscriptions
where user_id = $5 and type = 'RSS' and status = 'ACTIVE'
having count(*) < $9
having count(*) < $10
returning *;`,
[
feed.title,
feed.url,
feed.description || null,
feed.description,
SubscriptionType.Rss,
uid,
feed.thumbnail || null,
input.autoAddToLibrary ?? null,
input.isPrivate ?? null,
feed.thumbnail,
input.isPrivate,
input.fetchContent ?? true,
input.folder ?? 'following',
MAX_RSS_SUBSCRIPTIONS,
]
)) as any[]
@ -272,7 +277,8 @@ export const subscribeResolver = authorized<
scheduledDates: [new Date()], // fetch immediately
fetchedDates: [null],
checksums: [null],
addToLibraryFlags: [!!newSubscription.autoAddToLibrary],
fetchContents: [newSubscription.fetchContent],
folders: [newSubscription.folder],
})
return {
@ -328,6 +334,8 @@ export const updateSubscriptionResolver = authorized<
: undefined,
autoAddToLibrary: input.autoAddToLibrary ?? undefined,
isPrivate: input.isPrivate ?? undefined,
fetchContent: input.fetchContent ?? undefined,
folder: input.folder ?? undefined,
})
return repo.findOneByOrFail({

3
packages/api/src/routers/svc/rss_feed.ts
View File

@ -35,7 +35,8 @@ export function rssFeedRouter() {
ARRAY_AGG(last_fetched_at) AS "fetchedDates",
ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates",
ARRAY_AGG(last_fetched_checksum) AS checksums,
ARRAY_AGG(coalesce(auto_add_to_library, false)) AS "addToLibraryFlags"
ARRAY_AGG(fetch_content) AS "fetchContents",
ARRAY_AGG(folder) AS folders
FROM
omnivore.subscriptions
WHERE

19
packages/api/src/schema.ts
View File

@ -1254,6 +1254,9 @@ const schema = gql`
confirmationCode: String
createdAt: Date!
subscriptionCount: Int!
folder: String!
name: String
description: String
}
type NewsletterEmailsSuccess {
@ -1266,6 +1269,12 @@ const schema = gql`
union NewsletterEmailsResult = NewsletterEmailsSuccess | NewsletterEmailsError
input CreateNewsletterEmailInput {
name: String
description: String
folder: String
}
# Mutation: CreateNewsletterEmail
enum CreateNewsletterEmailErrorCode {
UNAUTHORIZED
@ -1674,6 +1683,8 @@ const schema = gql`
updatedAt: Date
isPrivate: Boolean
autoAddToLibrary: Boolean
fetchContent: Boolean!
folder: String!
}
enum SubscriptionStatus {
@ -2577,6 +2588,8 @@ const schema = gql`
subscriptionType: SubscriptionType
isPrivate: Boolean
autoAddToLibrary: Boolean
fetchContent: Boolean
folder: String
}
input UpdateSubscriptionInput {
@ -2589,6 +2602,8 @@ const schema = gql`
scheduledAt: Date
isPrivate: Boolean
autoAddToLibrary: Boolean
fetchContent: Boolean
folder: String
}
union UpdateSubscriptionResult =
@ -2753,7 +2768,9 @@ const schema = gql`
# input: UpdateLinkShareInfoInput!
# ): UpdateLinkShareInfoResult!
setLinkArchived(input: ArchiveLinkInput!): ArchiveLinkResult!
createNewsletterEmail: CreateNewsletterEmailResult!
createNewsletterEmail(
input: CreateNewsletterEmailInput
): CreateNewsletterEmailResult!
deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult!
saveUrl(input: SaveUrlInput!): SaveResult!
savePage(input: SavePageInput!): SaveResult!

8
packages/api/src/services/newsletters.ts
View File

@ -19,7 +19,10 @@ const parsedAddress = (emailAddress: string) => {
export const createNewsletterEmail = async (
userId: string,
confirmationCode?: string
confirmationCode?: string,
folder?: string,
name?: string,
description?: string
): Promise<NewsletterEmail> => {
const user = await userRepository.findById(userId)
if (!user) {
@ -34,6 +37,9 @@ export const createNewsletterEmail = async (
address: emailAddress,
user,
confirmationCode,
name,
description,
folder,
})
}

2
packages/api/src/services/save_email.ts
View File

@ -34,6 +34,7 @@ export type SaveEmailInput = {
unsubHttpUrl?: string
newsletterEmailId?: string
receivedEmailId: string
folder?: string
}
const isStubUrl = (url: string): boolean => {
@ -105,6 +106,7 @@ export const saveEmail = async (
siteName: parseResult.parsedContent?.siteName ?? undefined,
wordCount: wordsCount(content),
subscription: input.author,
folder: input.folder,
},
input.userId
)

16
packages/api/src/services/save_newsletter_email.ts
View File

@ -1,8 +1,10 @@
import { NewsletterEmail } from '../entity/newsletter_email'
import { Subscription } from '../entity/subscription'
import { env } from '../env'
import { analytics } from '../utils/analytics'
import { logger } from '../utils/logger'
import { saveEmail, SaveEmailInput } from './save_email'
import { getSubscriptionByName } from './subscriptions'
export interface NewsletterMessage {
email: string
@ -20,7 +22,8 @@ export interface NewsletterMessage {
// send the push but that is ok and we wont retry in that case.
export const saveNewsletter = async (
data: NewsletterMessage,
newsletterEmail: NewsletterEmail
newsletterEmail: NewsletterEmail,
existingSubscription?: Subscription
): Promise<boolean> => {
analytics.track({
userId: newsletterEmail.user.id,
@ -38,6 +41,16 @@ export const saveNewsletter = async (
return false
}
// find existing subscription if not provided
if (!existingSubscription) {
existingSubscription =
(await getSubscriptionByName(data.author, newsletterEmail.user.id)) ||
undefined
}
// subscription's folder takes precedence over newsletter email's folder
const folder = existingSubscription?.folder || newsletterEmail.folder
const input: SaveEmailInput = {
userId: newsletterEmail.user.id,
url: data.url,
@ -48,6 +61,7 @@ export const saveNewsletter = async (
unsubHttpUrl: data.unsubHttpUrl,
newsletterEmailId: newsletterEmail.id,
receivedEmailId: data.receivedEmailId,
folder,
}
const savedLibraryItem = await saveEmail(input)
if (!savedLibraryItem) {

6
packages/api/src/utils/createTask.ts
View File

@ -625,7 +625,8 @@ export interface RssSubscriptionGroup {
fetchedDates: (Date | null)[]
scheduledDates: Date[]
checksums: (string | null)[]
addToLibraryFlags: boolean[]
fetchContents: boolean[]
folders: string[]
}
export const enqueueRssFeedFetch = async (
@ -643,7 +644,8 @@ export const enqueueRssFeedFetch = async (
timestamp.getTime()
), // unix timestamp in milliseconds
userIds: subscriptionGroup.userIds,
addToLibraryFlags: subscriptionGroup.addToLibraryFlags,
fetchContents: subscriptionGroup.fetchContents,
folders: subscriptionGroup.folders,
}
// If there is no Google Cloud Project Id exposed, it means that we are in local environment

16
packages/db/migrations/0149.do.add_fetch_content_to_subscriptions.sql Executable file
View File

@ -0,0 +1,16 @@
-- Type: DO
-- Name: add_fetch_content_to_subscriptions
-- Description: Add fetch_content column to subscriptions tables
BEGIN;
ALTER TABLE omnivore.subscriptions
ADD COLUMN fetch_content BOOLEAN NOT NULL DEFAULT TRUE,
ADD COLUMN folder TEXT NOT NULL DEFAULT 'following';
ALTER TABLE omnivore.newsletter_emails
ADD COLUMN name TEXT,
ADD COLUMN description TEXT,
ADD COLUMN folder TEXT NOT NULL DEFAULT 'inbox';
COMMIT;

16
packages/db/migrations/0149.undo.add_fetch_content_to_subscriptions.sql Executable file
View File

@ -0,0 +1,16 @@
-- Type: UNDO
-- Name: add_fetch_content_to_subscriptions
-- Description: Add fetch_content column to subscriptions tables
BEGIN;
ALTER TABLE omnivore.subscriptions
DROP COLUMN fetch_content,
DROP COLUMN folder;
ALTER TABLE omnivore.newsletter_emails
DROP COLUMN name,
DROP COLUMN description,
DROP COLUMN folder;
COMMIT;

44
packages/rss-handler/src/index.ts
View File

@ -8,6 +8,8 @@ import Parser, { Item } from 'rss-parser'
import { promisify } from 'util'
import { CONTENT_FETCH_URL, createCloudTask } from './task'
type FolderType = 'following' | 'inbox'
interface RssFeedRequest {
subscriptionIds: string[]
feedUrl: string
@ -15,7 +17,8 @@ interface RssFeedRequest {
scheduledTimestamps: number[] // unix timestamp in milliseconds
lastFetchedChecksums: string[]
userIds: string[]
addToLibraryFlags: boolean[]
fetchContents: boolean[]
folders: FolderType[]
}
// link can be a string or an object
@ -58,7 +61,8 @@ function isRssFeedRequest(body: any): body is RssFeedRequest {
'scheduledTimestamps' in body &&
'userIds' in body &&
'lastFetchedChecksums' in body &&
'addToLibraryFlags' in body
'fetchContents' in body &&
'folders' in body
)
}
@ -198,13 +202,17 @@ const createTask = async (
userId: string,
feedUrl: string,
item: RssFeedItem,
autoAddToLibrary: boolean
fetchContent: boolean,
folder: FolderType
) => {
const folder = autoAddToLibrary ? 'inbox' : 'following'
return createSavingItemTask(userId, feedUrl, item, folder)
if (folder === 'following' && !fetchContent) {
return createItemWithPreviewContent(userId, feedUrl, item)
}
return fetchContentAndCreateItem(userId, feedUrl, item, folder)
}
const createSavingItemTask = async (
const fetchContentAndCreateItem = async (
userId: string,
feedUrl: string,
item: RssFeedItem,
@ -235,7 +243,7 @@ const createSavingItemTask = async (
}
}
const createFollowingTask = async (
const createItemWithPreviewContent = async (
userId: string,
feedUrl: string,
item: RssFeedItem
@ -247,7 +255,7 @@ const createFollowingTask = async (
author: item.creator,
description: item.summary,
addedToFollowingFrom: 'feed',
previewContent: item.content || item.contentSnippet,
previewContent: item.content || item.contentSnippet || item.summary,
addedToFollowingBy: feedUrl,
savedAt: item.isoDate,
publishedAt: item.isoDate,
@ -372,7 +380,8 @@ const processSubscription = async (
lastFetchedAt: number,
scheduledAt: number,
lastFetchedChecksum: string,
autoAddToLibrary: boolean,
fetchContent: boolean,
folder: FolderType,
feed: RssFeed
) => {
let lastItemFetchedAt: Date | null = null
@ -440,7 +449,13 @@ const processSubscription = async (
continue
}
const created = await createTask(userId, feedUrl, item, autoAddToLibrary)
const created = await createTask(
userId,
feedUrl,
item,
fetchContent,
folder
)
if (!created) {
console.error('Failed to create task for feed item', item.link)
continue
@ -467,7 +482,8 @@ const processSubscription = async (
userId,
feedUrl,
lastValidItem,
autoAddToLibrary
fetchContent,
folder
)
if (!created) {
console.error('Failed to create task for feed item', lastValidItem.link)
@ -514,7 +530,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
scheduledTimestamps,
userIds,
lastFetchedChecksums,
addToLibraryFlags,
fetchContents,
folders,
} = req.body
console.log('Processing feed', feedUrl)
@ -537,7 +554,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
lastFetchedTimestamps[i],
scheduledTimestamps[i],
lastFetchedChecksums[i],
addToLibraryFlags[i],
fetchContents[i],
folders[i],
feed
)
)