From 8eac9375a2883688465d43f0b1b3c09fffdc7d93 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 31 May 2024 16:34:32 +0800 Subject: [PATCH 1/2] feat: add a hidden section in home feed and load on demand --- packages/api/src/generated/graphql.ts | 44 ++++++ packages/api/src/generated/schema.graphql | 17 +++ packages/api/src/jobs/update_home.ts | 42 +++--- .../api/src/resolvers/function_resolvers.ts | 10 +- packages/api/src/resolvers/home/index.ts | 44 +++++- packages/api/src/schema.ts | 19 +++ .../queries/useGetHiddenHomeSection.tsx | 103 +++++++++++++ packages/web/pages/justread/index.tsx | 135 +++++++++++++----- 8 files changed, 354 insertions(+), 60 deletions(-) create mode 100644 packages/web/lib/networking/queries/useGetHiddenHomeSection.tsx diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 23b96e9fa..3ed794529 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1232,6 +1232,24 @@ export type GroupsSuccess = { groups: Array; }; +export type HiddenHomeSectionError = { + __typename?: 'HiddenHomeSectionError'; + errorCodes: Array; +}; + +export enum HiddenHomeSectionErrorCode { + BadRequest = 'BAD_REQUEST', + Pending = 'PENDING', + Unauthorized = 'UNAUTHORIZED' +} + +export type HiddenHomeSectionResult = HiddenHomeSectionError | HiddenHomeSectionSuccess; + +export type HiddenHomeSectionSuccess = { + __typename?: 'HiddenHomeSectionSuccess'; + section?: Maybe; +}; + export type Highlight = { __typename?: 'Highlight'; annotation?: Maybe; @@ -2242,6 +2260,7 @@ export type Query = { getUserPersonalization: GetUserPersonalizationResult; groups: GroupsResult; hello?: Maybe; + hiddenHomeSection: HiddenHomeSectionResult; home: HomeResult; integration: IntegrationResult; integrations: IntegrationsResult; @@ -4307,6 +4326,10 @@ export type ResolversTypes = { GroupsErrorCode: GroupsErrorCode; GroupsResult: ResolversTypes['GroupsError'] | ResolversTypes['GroupsSuccess']; GroupsSuccess: ResolverTypeWrapper; + HiddenHomeSectionError: ResolverTypeWrapper; + HiddenHomeSectionErrorCode: HiddenHomeSectionErrorCode; + HiddenHomeSectionResult: ResolversTypes['HiddenHomeSectionError'] | ResolversTypes['HiddenHomeSectionSuccess']; + HiddenHomeSectionSuccess: ResolverTypeWrapper; Highlight: ResolverTypeWrapper; HighlightReply: ResolverTypeWrapper; HighlightStats: ResolverTypeWrapper; @@ -4873,6 +4896,9 @@ export type ResolversParentTypes = { GroupsError: GroupsError; GroupsResult: ResolversParentTypes['GroupsError'] | ResolversParentTypes['GroupsSuccess']; GroupsSuccess: GroupsSuccess; + HiddenHomeSectionError: HiddenHomeSectionError; + HiddenHomeSectionResult: ResolversParentTypes['HiddenHomeSectionError'] | ResolversParentTypes['HiddenHomeSectionSuccess']; + HiddenHomeSectionSuccess: HiddenHomeSectionSuccess; Highlight: Highlight; HighlightReply: HighlightReply; HighlightStats: HighlightStats; @@ -6038,6 +6064,20 @@ export type GroupsSuccessResolvers; }; +export type HiddenHomeSectionErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type HiddenHomeSectionResultResolvers = { + __resolveType: TypeResolveFn<'HiddenHomeSectionError' | 'HiddenHomeSectionSuccess', ParentType, ContextType>; +}; + +export type HiddenHomeSectionSuccessResolvers = { + section?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type HighlightResolvers = { annotation?: Resolver, ParentType, ContextType>; color?: Resolver, ParentType, ContextType>; @@ -6540,6 +6580,7 @@ export type QueryResolvers; groups?: Resolver; hello?: Resolver, ParentType, ContextType>; + hiddenHomeSection?: Resolver; home?: Resolver>; integration?: Resolver>; integrations?: Resolver; @@ -7752,6 +7793,9 @@ export type Resolvers = { GroupsError?: GroupsErrorResolvers; GroupsResult?: GroupsResultResolvers; GroupsSuccess?: GroupsSuccessResolvers; + HiddenHomeSectionError?: HiddenHomeSectionErrorResolvers; + HiddenHomeSectionResult?: HiddenHomeSectionResultResolvers; + HiddenHomeSectionSuccess?: HiddenHomeSectionSuccessResolvers; Highlight?: HighlightResolvers; HighlightReply?: HighlightReplyResolvers; HighlightStats?: HighlightStatsResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index ab28f495b..bb2185cbf 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1107,6 +1107,22 @@ type GroupsSuccess { groups: [RecommendationGroup!]! } +type HiddenHomeSectionError { + errorCodes: [HiddenHomeSectionErrorCode!]! +} + +enum HiddenHomeSectionErrorCode { + BAD_REQUEST + PENDING + UNAUTHORIZED +} + +union HiddenHomeSectionResult = HiddenHomeSectionError | HiddenHomeSectionSuccess + +type HiddenHomeSectionSuccess { + section: HomeSection +} + type Highlight { annotation: String color: String @@ -1721,6 +1737,7 @@ type Query { getUserPersonalization: GetUserPersonalizationResult! groups: GroupsResult! hello: String + hiddenHomeSection: HiddenHomeSectionResult! home(after: String, first: Int): HomeResult! integration(name: String!): IntegrationResult! integrations: IntegrationsResult! diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index 4241e1633..02f994770 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -166,10 +166,10 @@ const selectCandidates = async ( subscriptionNames ) - // map library items to candidates and limit to 70 - const privateCandidates: Array = libraryItems - .map((item) => libraryItemToCandidate(item, subscriptions)) - .slice(0, 70) + // map library items to candidates + const privateCandidates: Array = libraryItems.map((item) => + libraryItemToCandidate(item, subscriptions) + ) const privateCandidatesSize = privateCandidates.length logger.info(`Found ${privateCandidatesSize} private candidates`) @@ -241,7 +241,7 @@ const redisKey = (userId: string) => `home:${userId}` export const getHomeSections = async ( userId: string, - limit: number, + limit = 100, maxScore?: number ): Promise> => { const redisClient = redisDataSource.redisClient @@ -311,13 +311,12 @@ const appendSectionsToHome = async ( JSON.stringify(section), ]) + // sections expire in 24 hours + pipeline.expire(key, 24 * 60 * 60) + // add section to the sorted set pipeline.zadd(key, ...scoreMembers) - // remove expired sections and sections expire in 24 hours - const ttl = 86_400_000 - pipeline.zremrangebyscore(key, '-inf', Date.now() - ttl) - // keep only the new sections and remove the oldest ones pipeline.zremrangebyrank(key, 0, -(sections.length + 1)) @@ -379,15 +378,15 @@ const mixHomeItems = ( } } + const topCandidates = rankedHomeItems.slice(0, 50) + // find the median word count - const wordCounts = rankedHomeItems.map((item) => item.wordCount) - wordCounts.sort((a, b) => a - b) - const medianWordCount = wordCounts[Math.floor(wordCounts.length / 2)] + const wordCountThreshold = 500 // separate items into two groups based on word count const shortItems: Array = [] const longItems: Array = [] - for (const item of rankedHomeItems) { - if (item.wordCount < medianWordCount) { + for (const item of topCandidates) { + if (item.wordCount < wordCountThreshold) { shortItems.push(item) } else { longItems.push(item) @@ -410,6 +409,13 @@ const mixHomeItems = ( // convert batches to sections const sections = [] + const hiddenCandidates = rankedHomeItems.slice(50) + + sections.push({ + items: hiddenCandidates.map(candidateToItem), + layout: 'hidden', + }) + sections.push({ items: batches.short.flat().map(candidateToItem), layout: 'quick_links', @@ -472,17 +478,15 @@ export const updateHome = async (data: UpdateHomeJobData) => { message: `Ranked ${rankedCandidates.length} candidates`, }) - // TODO: filter candidates - logger.profile('mixing') - const rankedSections = mixHomeItems(justAddedCandidates, rankedCandidates) + const sections = mixHomeItems(justAddedCandidates, rankedCandidates) logger.profile('mixing', { level: 'info', - message: `Created ${rankedSections.length} sections`, + message: `Created ${sections.length} sections`, }) logger.profile('saving') - await appendSectionsToHome(userId, rankedSections, cursor) + await appendSectionsToHome(userId, sections, cursor) logger.profile('saving', { level: 'info', message: 'Sections appended to home', diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index b9e7a4976..ba5079831 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -60,7 +60,11 @@ import { saveDiscoverArticleResolver, } from './discover_feeds' import { optInFeatureResolver } from './features' -import { homeResolver, refreshHomeResolver } from './home' +import { + hiddenHomeSectionResolver, + homeResolver, + refreshHomeResolver, +} from './home' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { addPopularReadResolver, @@ -371,6 +375,7 @@ export const functionResolvers = { integration: integrationResolver, home: homeResolver, subscription: subscriptionResolver, + hiddenHomeSection: hiddenHomeSectionResolver, }, User: { async intercomHash( @@ -645,6 +650,8 @@ export const functionResolvers = { return 'Top Picks' case 'quick_links': return 'Quick Links' + case 'hidden': + return 'Hidden Gems' default: return '' } @@ -878,4 +885,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('Home'), ...resultResolveTypeResolver('Subscription'), ...resultResolveTypeResolver('RefreshHome'), + ...resultResolveTypeResolver('HiddenHomeSection'), } diff --git a/packages/api/src/resolvers/home/index.ts b/packages/api/src/resolvers/home/index.ts index 1c1cb7249..9e22c2b9b 100644 --- a/packages/api/src/resolvers/home/index.ts +++ b/packages/api/src/resolvers/home/index.ts @@ -1,4 +1,6 @@ import { + HiddenHomeSectionError, + HiddenHomeSectionSuccess, HomeError, HomeErrorCode, HomeItem, @@ -63,10 +65,16 @@ export const homeResolver = authorized< const endCursor = sections[sections.length - 1].score.toString() - const edges = sections.map((section) => ({ - cursor: section.score.toString(), - node: section.member, - })) + const edges = sections.map((section) => { + if (section.member.layout === 'hidden') { + section.member.items = [] + } + + return { + cursor: section.score.toString(), + node: section.member, + } + }) return { edges, @@ -105,3 +113,31 @@ export const refreshHomeResolver = authorized< success: true, } }) + +type PartialHiddenHomeSectionSuccess = Merge< + HiddenHomeSectionSuccess, + { + section?: PartialHomeSection + } +> +export const hiddenHomeSectionResolver = authorized< + PartialHiddenHomeSectionSuccess, + HiddenHomeSectionError +>(async (_, __, { uid, log }) => { + const sections = await getHomeSections(uid) + log.info('Home sections fetched') + + if (sections.length === 0) { + return { + errorCodes: [HomeErrorCode.Pending], + } + } + + const hiddenSection = sections.find( + (section) => section.member.layout === 'hidden' + ) + + return { + section: hiddenSection?.member, + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 1821ab37a..14cdeddcd 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -3208,6 +3208,24 @@ const schema = gql` union RefreshHomeResult = RefreshHomeSuccess | RefreshHomeError + union HiddenHomeSectionResult = + HiddenHomeSectionSuccess + | HiddenHomeSectionError + + type HiddenHomeSectionSuccess { + section: HomeSection + } + + type HiddenHomeSectionError { + errorCodes: [HiddenHomeSectionErrorCode!]! + } + + enum HiddenHomeSectionErrorCode { + UNAUTHORIZED + BAD_REQUEST + PENDING + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -3406,6 +3424,7 @@ const schema = gql` scanFeeds(input: ScanFeedsInput!): ScanFeedsResult! home(first: Int, after: String): HomeResult! subscription(id: ID!): SubscriptionResult! + hiddenHomeSection: HiddenHomeSectionResult! } schema { diff --git a/packages/web/lib/networking/queries/useGetHiddenHomeSection.tsx b/packages/web/lib/networking/queries/useGetHiddenHomeSection.tsx new file mode 100644 index 000000000..f3fcc9ddf --- /dev/null +++ b/packages/web/lib/networking/queries/useGetHiddenHomeSection.tsx @@ -0,0 +1,103 @@ +import { gql } from 'graphql-request' +import useSWR from 'swr' +import { publicGqlFetcher } from '../networkHelpers' +import { HomeSection } from './useGetHome' + +type HiddenHomeSectionResult = { + hiddenHomeSection: { + section?: HomeSection + errorCodes?: string[] + } +} + +export type HiddenHomeSectionResponse = { + error: boolean + isValidating: boolean + errorMessage?: string + section?: HomeSection + mutate?: () => void +} + +export function useGetHiddenHomeSection(): HiddenHomeSectionResponse { + const query = gql` + query HiddenHomeSection { + hiddenHomeSection { + ... on HiddenHomeSectionSuccess { + section { + title + layout + thumbnail + items { + id + title + url + slug + score + thumbnail + previewContent + saveCount + likeCount + broadcastCount + date + author + dir + seen_at + wordCount + source { + id + name + url + icon + type + } + canSave + canComment + canShare + canArchive + canDelete + } + } + } + ... on HiddenHomeSectionError { + errorCodes + } + } + } + ` + + const { data, error, isValidating, mutate } = useSWR(query, publicGqlFetcher) + console.log('HiddenHomeSection data', data) + + if (error) { + return { + error: true, + isValidating, + errorMessage: error.toString(), + } + } + + const result = data as HiddenHomeSectionResult + + if (result && result.hiddenHomeSection.errorCodes) { + const errorCodes = result.hiddenHomeSection.errorCodes + return { + error: true, + isValidating, + errorMessage: errorCodes.length > 0 ? errorCodes[0] : undefined, + } + } + + if (result && result.hiddenHomeSection && result.hiddenHomeSection.section) { + return { + mutate, + error: false, + isValidating, + section: result.hiddenHomeSection.section, + } + } + + return { + isValidating, + error: !!error, + } +} diff --git a/packages/web/pages/justread/index.tsx b/packages/web/pages/justread/index.tsx index baf33dedb..7037619bf 100644 --- a/packages/web/pages/justread/index.tsx +++ b/packages/web/pages/justread/index.tsx @@ -1,7 +1,7 @@ import * as HoverCard from '@radix-ui/react-hover-card' import { styled } from '@stitches/react' import { useRouter } from 'next/router' -import { useMemo } from 'react' +import { useMemo, useState } from 'react' import { Button } from '../../components/elements/Button' import { AddToLibraryActionIcon } from '../../components/elements/icons/home/AddToLibraryActionIcon' import { ArchiveActionIcon } from '../../components/elements/icons/home/ArchiveActionIcon' @@ -12,6 +12,7 @@ import Pagination from '../../components/elements/Pagination' import { timeAgo } from '../../components/patterns/LibraryCards/LibraryCardStyles' import { theme } from '../../components/tokens/stitches.config' import { useApplyLocalTheme } from '../../lib/hooks/useApplyLocalTheme' +import { useGetHiddenHomeSection } from '../../lib/networking/queries/useGetHiddenHomeSection' import { HomeItem, HomeItemSource, @@ -79,6 +80,15 @@ export default function Home(): JSX.Element { homeSection={homeSection} /> ) + case 'hidden': + return ( + + ) + default: + return <> } })} @@ -139,9 +149,9 @@ const TopPicksHomeSection = (props: HomeSectionProps): JSX.Element => { ( - + )} /> @@ -170,7 +180,7 @@ const QuickLinksHomeSection = (props: HomeSectionProps): JSX.Element => { ( )} @@ -179,6 +189,81 @@ const QuickLinksHomeSection = (props: HomeSectionProps): JSX.Element => { ) } +const HiddenHomeSection = (props: HomeSectionProps): JSX.Element => { + const [isHidden, setIsHidden] = useState(true) + return ( + + setIsHidden(!isHidden)} + > + + {props.homeSection.title} + + + {isHidden ? 'Show' : 'Hide'} + + + + {isHidden ? <> : } + + ) +} + +const HiddenHomeSectionView = (): JSX.Element => { + const hiddenSectionData = useGetHiddenHomeSection() + + if (hiddenSectionData.error) { + return Error loading hidden section + } + + if (hiddenSectionData.isValidating) { + return Loading... + } + + if (!hiddenSectionData.section) { + return No hidden section data + } + + return ( + + {hiddenSectionData.section.items.map((homeItem) => { + return + })} + + ) +} + const CoverImage = styled('img', { objectFit: 'cover', }) @@ -229,7 +314,7 @@ const JustReadItemView = (props: HomeItemViewProps): JSX.Element => { { } }} > - - - - - - - - </VStack> - <SpanBox css={{ ml: 'auto' }}> - {props.homeItem.thumbnail && ( - <CoverImage - css={{ - mt: '6px', - width: '120px', - height: '70px', - borderRadius: '4px', - }} - src={props.homeItem.thumbnail} - ></CoverImage> - )} - </SpanBox> + <HStack + distribution="start" + alignment="center" + css={{ gap: '5px', lineHeight: '1' }} + > + <SourceInfo homeItem={props.homeItem} /> + <TimeAgo homeItem={props.homeItem} /> </HStack> + <Title homeItem={props.homeItem} /> </VStack> ) } -const LongHomeItemView = (props: HomeItemViewProps): JSX.Element => { +const TopicPickHomeItemView = (props: HomeItemViewProps): JSX.Element => { const router = useRouter() return ( @@ -372,11 +440,6 @@ const QuickLinkHomeItemView = (props: HomeItemViewProps): JSX.Element => { > <TimeAgo homeItem={props.homeItem} /> <Title homeItem={props.homeItem} /> - <SpanBox - css={{ fontFamily: '$inter', fontSize: '13px', lineHeight: '23px' }} - > - {props.homeItem.previewContent} - </SpanBox> </VStack> ) } From ad2863e54ac42f20787121f7edfe37f7a33cc097 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Mon, 3 Jun 2024 10:56:09 +0800 Subject: [PATCH 2/2] reduce just added item padding --- packages/web/pages/justread/index.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/web/pages/justread/index.tsx b/packages/web/pages/justread/index.tsx index 7037619bf..936484476 100644 --- a/packages/web/pages/justread/index.tsx +++ b/packages/web/pages/justread/index.tsx @@ -61,7 +61,7 @@ export default function Home(): JSX.Element { switch (homeSection.layout) { case 'just_added': return ( - <JustReadHomeSection + <JustAddedHomeSection key={`section-${idx}`} homeSection={homeSection} /> @@ -100,7 +100,7 @@ type HomeSectionProps = { homeSection: HomeSection } -const JustReadHomeSection = (props: HomeSectionProps): JSX.Element => { +const JustAddedHomeSection = (props: HomeSectionProps): JSX.Element => { return ( <VStack distribution="start" @@ -121,7 +121,7 @@ const JustReadHomeSection = (props: HomeSectionProps): JSX.Element => { </SpanBox> {props.homeSection.items.map((homeItem) => { - return <JustReadItemView key={homeItem.id} homeItem={homeItem} /> + return <JustAddedItemView key={homeItem.id} homeItem={homeItem} /> })} </VStack> ) @@ -307,14 +307,14 @@ const Title = (props: HomeItemViewProps): JSX.Element => { ) } -const JustReadItemView = (props: HomeItemViewProps): JSX.Element => { +const JustAddedItemView = (props: HomeItemViewProps): JSX.Element => { const router = useRouter() return ( <VStack css={{ width: '100%', - padding: '10px', + padding: '5px', borderRadius: '5px', '&:hover': { bg: '$thBackground',