From 8d8c7e8a2635e783ffabac4e448fb5c67d395bc1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 28 Mar 2024 12:13:28 +0800 Subject: [PATCH 1/6] fix: allow max 1000 users to use notion feature --- packages/api/src/services/features.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index f6de25593..a479a146c 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -8,6 +8,7 @@ import { logger } from '../utils/logger' const MAX_ULTRA_REALISTIC_USERS = 1500 const MAX_YOUTUBE_TRANSCRIPT_USERS = 100 +const MAX_NOTION_USERS = 1000 export enum FeatureName { AISummaries = 'ai-summaries', @@ -38,7 +39,7 @@ export const optInFeature = async ( MAX_YOUTUBE_TRANSCRIPT_USERS ) case FeatureName.Notion: - return optInLimitedFeature(FeatureName.Notion, uid, 1) + return optInLimitedFeature(FeatureName.Notion, uid, MAX_NOTION_USERS) default: return undefined } From 45417efd0c28123365b774703d3fee5ca0afda09 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 28 Mar 2024 12:15:36 +0800 Subject: [PATCH 2/6] bump youtube users to 500 --- packages/api/src/services/features.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index a479a146c..e82a21bbc 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -7,7 +7,7 @@ import { getRepository } from '../repository' import { logger } from '../utils/logger' const MAX_ULTRA_REALISTIC_USERS = 1500 -const MAX_YOUTUBE_TRANSCRIPT_USERS = 100 +const MAX_YOUTUBE_TRANSCRIPT_USERS = 500 const MAX_NOTION_USERS = 1000 export enum FeatureName { From 1afec1ddf393dbe592a0056f8b77aa404cd3fdd6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 28 Mar 2024 14:41:30 +0800 Subject: [PATCH 3/6] check if feature is granted when process youtube video --- .../api/src/jobs/process-youtube-video.ts | 31 ++++++++++--------- packages/api/src/pubsub.ts | 13 ++++---- packages/api/src/services/features.ts | 5 +-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts index 2802e89a6..d8a067897 100644 --- a/packages/api/src/jobs/process-youtube-video.ts +++ b/packages/api/src/jobs/process-youtube-video.ts @@ -1,21 +1,19 @@ -import { logger } from '../utils/logger' +import { Storage } from '@google-cloud/storage' +import { PromptTemplate } from '@langchain/core/prompts' +import { OpenAI } from '@langchain/openai' +import { parseHTML } from 'linkedom' +import showdown from 'showdown' +import * as stream from 'stream' +import { Chapter, Client as YouTubeClient } from 'youtubei' +import { LibraryItem, LibraryItemState } from '../entity/library_item' +import { env } from '../env' import { authTrx } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { LibraryItem, LibraryItemState } from '../entity/library_item' - -import { Chapter, Client as YouTubeClient } from 'youtubei' -import showdown from 'showdown' -import { parseHTML } from 'linkedom' -import { parsePreparedContent } from '../utils/parser' -import { OpenAI } from '@langchain/openai' -import { PromptTemplate } from '@langchain/core/prompts' +import { FeatureName, findGrantedFeatureByName } from '../services/features' import { enqueueProcessYouTubeTranscript } from '../utils/createTask' -import { env } from '../env' -import * as stream from 'stream' - -import { Storage } from '@google-cloud/storage' import { stringToHash } from '../utils/helpers' -import { FeatureName, findFeatureByName } from '../services/features' +import { logger } from '../utils/logger' +import { parsePreparedContent } from '../utils/parser' export interface ProcessYouTubeVideoJobData { userId: string @@ -394,7 +392,10 @@ export const processYouTubeVideo = async ( } if ( - await findFeatureByName(FeatureName.YouTubeTranscripts, jobData.userId) + await findGrantedFeatureByName( + FeatureName.YouTubeTranscripts, + jobData.userId + ) ) { if ('getTranscript' in video && duration > 0 && duration < 1801) { // If the video has a transcript available, put a placehold in and diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 237a5801f..2214d787b 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -3,7 +3,6 @@ import express from 'express' import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' -import { FeatureName, findFeatureByName } from './services/features' import { enqueueExportItem, enqueueProcessYouTubeVideo, @@ -87,12 +86,12 @@ export const createPubSubClient = (): PubsubClient => { }) if (type === EntityType.PAGE) { - if (await findFeatureByName(FeatureName.AISummaries, userId)) { - // await enqueueAISummarizeJob({ - // userId, - // libraryItemId, - // }) - } + // if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) { + // await enqueueAISummarizeJob({ + // userId, + // libraryItemId, + // }) + // } const isYoutubeVideo = (data: any): data is { originalUrl: string } => { return 'originalUrl' in data diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index e82a21bbc..cb5ca67b2 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -132,13 +132,14 @@ export const findUserFeatures = async (userId: string): Promise => { ).map((feature) => feature.name) } -export const findFeatureByName = async ( +export const findGrantedFeatureByName = async ( name: FeatureName, userId: string ): Promise => { - return await getRepository(Feature).findOneBy({ + return getRepository(Feature).findOneBy({ name, user: { id: userId }, + grantedAt: Not(IsNull()), }) } From cc2fe045686ff6b67549bc3a615c11e3e704b147 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 28 Mar 2024 14:42:08 +0800 Subject: [PATCH 4/6] add featureList to the User type which contains a list of feature type --- packages/api/src/generated/graphql.ts | 2 ++ packages/api/src/generated/schema.graphql | 1 + packages/api/src/resolvers/function_resolvers.ts | 13 ++++++++++++- packages/api/src/schema.ts | 1 + packages/api/src/services/features.ts | 14 ++++++-------- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index c1758f65f..82b4cef2c 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -3696,6 +3696,7 @@ export enum UploadImportFileType { export type User = { __typename?: 'User'; email?: Maybe; + featureList?: Maybe>>; features?: Maybe>>; followersCount?: Maybe; friendsCount?: Maybe; @@ -7139,6 +7140,7 @@ export type UploadImportFileSuccessResolvers = { email?: Resolver, ParentType, ContextType>; + featureList?: Resolver>>, ParentType, ContextType>; features?: Resolver>>, ParentType, ContextType>; followersCount?: Resolver, ParentType, ContextType>; friendsCount?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 104a34987..11bca2d29 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2984,6 +2984,7 @@ enum UploadImportFileType { type User { email: String + featureList: [Feature] features: [String] followersCount: Int friendsCount: Int diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 8757c10a1..cf391b6db 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -376,7 +376,18 @@ export const functionResolvers = { if (!ctx.claims?.uid) { return undefined } - return findUserFeatures(ctx.claims.uid) + const userFeatures = await findUserFeatures(ctx.claims.uid) + return userFeatures.map((feature) => feature.name) + }, + async featureList( + _: User, + __: Record, + ctx: WithDataSourcesContext + ) { + if (!ctx.uid) { + return undefined + } + return findUserFeatures(ctx.uid) }, }, Article: { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 42d234a7d..2e75d0ee6 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -89,6 +89,7 @@ const schema = gql` source: String intercomHash: String features: [String] + featureList: [Feature] } type Profile { diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index cb5ca67b2..5c1172d27 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -122,14 +122,12 @@ export const signFeatureToken = ( ) } -export const findUserFeatures = async (userId: string): Promise => { - return ( - await getRepository(Feature).find({ - where: { - user: { id: userId }, - }, - }) - ).map((feature) => feature.name) +export const findUserFeatures = async (userId: string) => { + return getRepository(Feature).find({ + where: { + user: { id: userId }, + }, + }) } export const findGrantedFeatureByName = async ( From f677f1167f628aaedae7abd4374b47ccd771cee3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 28 Mar 2024 15:27:37 +0800 Subject: [PATCH 5/6] get feature list on web --- packages/api/src/generated/graphql.ts | 4 +-- packages/api/src/generated/schema.graphql | 2 +- .../api/src/resolvers/function_resolvers.ts | 25 ++++++++++--------- packages/api/src/schema.ts | 2 +- packages/api/src/services/features.ts | 6 ++--- packages/web/lib/featureFlag.ts | 7 +++++- .../networking/fragments/featureFragment.ts | 21 ++++++++++++++++ .../mutations/optIntoFeatureMutation.ts | 16 ++++++------ .../networking/queries/useGetViewerQuery.tsx | 6 +++++ packages/web/pages/settings/features/beta.tsx | 7 +++--- 10 files changed, 62 insertions(+), 34 deletions(-) create mode 100644 packages/web/lib/networking/fragments/featureFragment.ts diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 82b4cef2c..ae5bcdbd8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -3696,7 +3696,7 @@ export enum UploadImportFileType { export type User = { __typename?: 'User'; email?: Maybe; - featureList?: Maybe>>; + featureList?: Maybe>; features?: Maybe>>; followersCount?: Maybe; friendsCount?: Maybe; @@ -7140,7 +7140,7 @@ export type UploadImportFileSuccessResolvers = { email?: Resolver, ParentType, ContextType>; - featureList?: Resolver>>, ParentType, ContextType>; + featureList?: Resolver>, ParentType, ContextType>; features?: Resolver>>, ParentType, ContextType>; followersCount?: Resolver, ParentType, ContextType>; friendsCount?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 11bca2d29..94695e350 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2984,7 +2984,7 @@ enum UploadImportFileType { type User { email: String - featureList: [Feature] + featureList: [Feature!] features: [String] followersCount: Int friendsCount: Int diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index cf391b6db..9a8fcfa58 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -368,6 +368,17 @@ export const functionResolvers = { } return undefined }, + async featureList( + _: User, + __: Record, + ctx: WithDataSourcesContext + ) { + if (!ctx.claims?.uid) { + return undefined + } + + return findUserFeatures(ctx.claims.uid) + }, async features( user: User, __: Record, @@ -376,18 +387,8 @@ export const functionResolvers = { if (!ctx.claims?.uid) { return undefined } - const userFeatures = await findUserFeatures(ctx.claims.uid) - return userFeatures.map((feature) => feature.name) - }, - async featureList( - _: User, - __: Record, - ctx: WithDataSourcesContext - ) { - if (!ctx.uid) { - return undefined - } - return findUserFeatures(ctx.uid) + + return (await findUserFeatures(ctx.claims.uid)).map((f) => f.name) }, }, Article: { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 2e75d0ee6..e106514e4 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -89,7 +89,7 @@ const schema = gql` source: String intercomHash: String features: [String] - featureList: [Feature] + featureList: [Feature!] } type Profile { diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index 5c1172d27..664461a50 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -123,10 +123,8 @@ export const signFeatureToken = ( } export const findUserFeatures = async (userId: string) => { - return getRepository(Feature).find({ - where: { - user: { id: userId }, - }, + return getRepository(Feature).findBy({ + user: { id: userId }, }) } diff --git a/packages/web/lib/featureFlag.ts b/packages/web/lib/featureFlag.ts index f79369b78..4cae69bf6 100644 --- a/packages/web/lib/featureFlag.ts +++ b/packages/web/lib/featureFlag.ts @@ -7,5 +7,10 @@ export const userHasFeature = ( if (!user) { return false } - return user.features.includes(feature) + return user.featureList.some( + (f) => + f.name === feature && + f.grantedAt && + (!f.expiresAt || new Date(f.expiresAt) > new Date()) + ) } diff --git a/packages/web/lib/networking/fragments/featureFragment.ts b/packages/web/lib/networking/fragments/featureFragment.ts new file mode 100644 index 000000000..78dea92f4 --- /dev/null +++ b/packages/web/lib/networking/fragments/featureFragment.ts @@ -0,0 +1,21 @@ +import { gql } from 'graphql-request' + +export const featureFragment = gql` + fragment FeatureFields on Feature { + id + name + createdAt + updatedAt + grantedAt + expiresAt + } +` + +export interface Feature { + id: string + name: string + createdAt: Date + updatedAt?: Date + grantedAt?: Date + expiresAt?: Date +} diff --git a/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts b/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts index 26ac49d47..92b5f3b96 100644 --- a/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts +++ b/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts @@ -1,16 +1,17 @@ import { gql } from 'graphql-request' +import { Feature, featureFragment } from '../fragments/featureFragment' import { gqlFetcher } from '../networkHelpers' export interface OptInFeatureInput { name: string } -export interface OptInFeatureSuccess { - feature: { id: string } +export interface OptInFeatureResponse { + feature?: Feature } interface Response { - optInFeature: OptInFeatureSuccess + optInFeature: OptInFeatureResponse } export async function optInFeature( @@ -21,7 +22,7 @@ export async function optInFeature( optInFeature(input: $input) { ... on OptInFeatureSuccess { feature { - id + ...FeatureFields } } ... on OptInFeatureError { @@ -29,17 +30,14 @@ export async function optInFeature( } } } + ${featureFragment} ` try { const data = await gqlFetcher(mutation, { input, }) const output = data as Response | undefined - if ( - !output || - !output.optInFeature || - 'errorCodes' in output?.optInFeature - ) { + if (!output || !output.optInFeature.feature) { return false } return true diff --git a/packages/web/lib/networking/queries/useGetViewerQuery.tsx b/packages/web/lib/networking/queries/useGetViewerQuery.tsx index abf88c1ff..60649a8d5 100644 --- a/packages/web/lib/networking/queries/useGetViewerQuery.tsx +++ b/packages/web/lib/networking/queries/useGetViewerQuery.tsx @@ -1,5 +1,6 @@ import { gql } from 'graphql-request' import useSWR from 'swr' +import { Feature, featureFragment } from '../fragments/featureFragment' import { publicGqlFetcher } from '../networkHelpers' type ViewerQueryResponse = { @@ -22,6 +23,7 @@ export type UserBasicData = { source: string intercomHash: string features: string[] + featureList: Feature[] } export type UserProfile = { @@ -48,8 +50,12 @@ export function useGetViewerQuery(): ViewerQueryResponse { source intercomHash features + featureList { + ...FeatureFields + } } } + ${featureFragment} ` const { data, error, mutate } = useSWR(query, publicGqlFetcher) diff --git a/packages/web/pages/settings/features/beta.tsx b/packages/web/pages/settings/features/beta.tsx index 21ab7f17a..b24e17df9 100644 --- a/packages/web/pages/settings/features/beta.tsx +++ b/packages/web/pages/settings/features/beta.tsx @@ -6,6 +6,7 @@ import { HStack, VStack } from '../../../components/elements/LayoutPrimitives' import { StyledText } from '../../../components/elements/StyledText' import { SettingsLayout } from '../../../components/templates/SettingsLayout' import { styled } from '../../../components/tokens/stitches.config' +import { userHasFeature } from '../../../lib/featureFlag' import { optInFeature } from '../../../lib/networking/mutations/optIntoFeatureMutation' import { useGetViewerQuery } from '../../../lib/networking/queries/useGetViewerQuery' import { applyStoredTheme } from '../../../lib/themeUpdater' @@ -43,13 +44,11 @@ export default function Account(): JSX.Element { ) const hasYouTube = useMemo(() => { - return ( - (viewerData?.me?.features.indexOf('youtube-transcripts') ?? -1) !== -1 - ) + return userHasFeature(viewerData?.me, 'youtube-transcripts') }, [viewerData]) const hasNotion = useMemo(() => { - return (viewerData?.me?.features.indexOf('notion') ?? -1) !== -1 + return userHasFeature(viewerData?.me, 'notion') }, [viewerData]) applyStoredTheme() From ce180723246f9941c9dec3c4f81547a01b333555 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 28 Mar 2024 15:53:13 +0800 Subject: [PATCH 6/6] remove button if feature is requested --- .../mutations/optIntoFeatureMutation.ts | 6 +++++- packages/web/pages/settings/features/beta.tsx | 16 +++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts b/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts index 92b5f3b96..27633c322 100644 --- a/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts +++ b/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts @@ -37,7 +37,11 @@ export async function optInFeature( input, }) const output = data as Response | undefined - if (!output || !output.optInFeature.feature) { + if ( + !output || + !output.optInFeature.feature || + !output.optInFeature.feature.grantedAt + ) { return false } return true diff --git a/packages/web/pages/settings/features/beta.tsx b/packages/web/pages/settings/features/beta.tsx index b24e17df9..733333a86 100644 --- a/packages/web/pages/settings/features/beta.tsx +++ b/packages/web/pages/settings/features/beta.tsx @@ -44,11 +44,13 @@ export default function Account(): JSX.Element { ) const hasYouTube = useMemo(() => { - return userHasFeature(viewerData?.me, 'youtube-transcripts') + return viewerData?.me?.featureList?.some( + (f) => f.name === 'youtube-transcripts' + ) }, [viewerData]) const hasNotion = useMemo(() => { - return userHasFeature(viewerData?.me, 'notion') + return viewerData?.me?.featureList?.some((f) => f.name === 'notion') }, [viewerData]) applyStoredTheme() @@ -88,7 +90,7 @@ export default function Account(): JSX.Element { Enabled beta features {!showSpinner ? ( <> - {viewerData?.me?.features.map((feature) => { + {viewerData?.me?.featureList.map((feature) => { return ( - {feature} + {`${feature.name}${ + userHasFeature(viewerData?.me, feature.name) + ? '' + : ' - Requested' + }`} ) })}