Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1502 from omnivore-app/recommend-api

Browse Source 
recommend api
This commit is contained in:
Hongbo Wu
2022-12-05 10:07:25 +08:00
committed by GitHub
parent 1c152bcccd 6b65c386a0
commit fc9cfb4413
12 changed files with 383 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
packages/api/src/elastic/recommendation.ts Normal file
View File

@ -0,0 +1,53 @@
import { Page, PageContext, Recommendation } from './types'
import { createPage, getPageByParam, updatePage } from './pages'
export const addRecommendation = async (
ctx: PageContext,
page: Page,
recommendation: Recommendation
): Promise<string | undefined> => {
try {
// check if the page is already recommended to the group
const existingPage = await getPageByParam({
userId: ctx.uid,
url: page.url,
})
if (existingPage) {
if (existingPage.recommendedBy?.includes(recommendation)) {
return existingPage._id
}
// update recommendedBy in the existing page
const recommendedBy = (existingPage.recommendedBy || []).concat(
recommendation
)
await updatePage(
existingPage.id,
{
recommendedBy,
},
ctx
)
return existingPage.id
}
// create a new page
const newPage: Page = {
...page,
id: '',
recommendedBy: [recommendation],
userId: ctx.uid,
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
sharedAt: new Date(),
highlights: [],
readAt: undefined,
labels: [],
}
return createPage(newPage, ctx)
} catch (err) {
console.error(err)
}
}

8
packages/api/src/elastic/types.ts
View File

@ -188,6 +188,12 @@ export interface Highlight {
highlightPositionAnchorIndex?: number | null
}
export interface Recommendation {
id: string
name: string
recommendedAt: Date
}
export interface Page {
id: string
userId: string
@ -224,6 +230,7 @@ export interface Page {
readAt?: Date
listenedAt?: Date
wordsCount?: number
recommendedBy?: Recommendation[]
}
export interface SearchItem {
@ -256,6 +263,7 @@ export interface SearchItem {
wordsCount?: number
siteName?: string
siteIcon?: string
recommendedBy?: Recommendation[]
}
const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const

2
packages/api/src/entity/groups/group_membership.ts
View File

@ -16,7 +16,7 @@ import { Invite } from './invite'
@Entity()
export class GroupMembership {
@PrimaryGeneratedColumn('uuid')
id?: string
id!: string
@OneToOne(() => User)
@JoinColumn()

56
packages/api/src/generated/graphql.ts
View File

@ -1132,6 +1132,7 @@ export type Mutation = {
moveFilter: MoveFilterResult;
moveLabel: MoveLabelResult;
optInFeature: OptInFeatureResult;
recommend: RecommendResult;
reportItem: ReportItemResult;
revokeApiKey: RevokeApiKeyResult;
saveArticleReadingProgress: SaveArticleReadingProgressResult;
@ -1301,6 +1302,11 @@ export type MutationOptInFeatureArgs = {
};
export type MutationRecommendArgs = {
input: RecommendInput;
};
export type MutationReportItemArgs = {
input: ReportItemInput;
};
@ -1743,6 +1749,29 @@ export type RecentSearchesSuccess = {
searches: Array<RecentSearch>;
};
export type RecommendError = {
__typename?: 'RecommendError';
errorCodes: Array<RecommendErrorCode>;
};
export enum RecommendErrorCode {
BadRequest = 'BAD_REQUEST',
NotFound = 'NOT_FOUND',
Unauthorized = 'UNAUTHORIZED'
}
export type RecommendInput = {
groupIds: Array<Scalars['ID']>;
pageId: Scalars['ID'];
};
export type RecommendResult = RecommendError | RecommendSuccess;
export type RecommendSuccess = {
__typename?: 'RecommendSuccess';
taskNames: Array<Scalars['String']>;
};
export type RecommendationGroup = {
__typename?: 'RecommendationGroup';
admins: Array<User>;
@ -3157,6 +3186,11 @@ export type ResolversTypes = {
RecentSearchesErrorCode: RecentSearchesErrorCode;
RecentSearchesResult: ResolversTypes['RecentSearchesError'] | ResolversTypes['RecentSearchesSuccess'];
RecentSearchesSuccess: ResolverTypeWrapper<RecentSearchesSuccess>;
RecommendError: ResolverTypeWrapper<RecommendError>;
RecommendErrorCode: RecommendErrorCode;
RecommendInput: RecommendInput;
RecommendResult: ResolversTypes['RecommendError'] | ResolversTypes['RecommendSuccess'];
RecommendSuccess: ResolverTypeWrapper<RecommendSuccess>;
RecommendationGroup: ResolverTypeWrapper<RecommendationGroup>;
Reminder: ResolverTypeWrapper<Reminder>;
ReminderError: ResolverTypeWrapper<ReminderError>;
@ -3550,6 +3584,10 @@ export type ResolversParentTypes = {
RecentSearchesError: RecentSearchesError;
RecentSearchesResult: ResolversParentTypes['RecentSearchesError'] | ResolversParentTypes['RecentSearchesSuccess'];
RecentSearchesSuccess: RecentSearchesSuccess;
RecommendError: RecommendError;
RecommendInput: RecommendInput;
RecommendResult: ResolversParentTypes['RecommendError'] | ResolversParentTypes['RecommendSuccess'];
RecommendSuccess: RecommendSuccess;
RecommendationGroup: RecommendationGroup;
Reminder: Reminder;
ReminderError: ReminderError;
@ -4547,6 +4585,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
moveFilter?: Resolver<ResolversTypes['MoveFilterResult'], ParentType, ContextType, RequireFields<MutationMoveFilterArgs, 'input'>>;
moveLabel?: Resolver<ResolversTypes['MoveLabelResult'], ParentType, ContextType, RequireFields<MutationMoveLabelArgs, 'input'>>;
optInFeature?: Resolver<ResolversTypes['OptInFeatureResult'], ParentType, ContextType, RequireFields<MutationOptInFeatureArgs, 'input'>>;
recommend?: Resolver<ResolversTypes['RecommendResult'], ParentType, ContextType, RequireFields<MutationRecommendArgs, 'input'>>;
reportItem?: Resolver<ResolversTypes['ReportItemResult'], ParentType, ContextType, RequireFields<MutationReportItemArgs, 'input'>>;
revokeApiKey?: Resolver<ResolversTypes['RevokeApiKeyResult'], ParentType, ContextType, RequireFields<MutationRevokeApiKeyArgs, 'id'>>;
saveArticleReadingProgress?: Resolver<ResolversTypes['SaveArticleReadingProgressResult'], ParentType, ContextType, RequireFields<MutationSaveArticleReadingProgressArgs, 'input'>>;
@ -4722,6 +4761,20 @@ export type RecentSearchesSuccessResolvers<ContextType = ResolverContext, Parent
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecommendErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecommendError'] = ResolversParentTypes['RecommendError']> = {
errorCodes?: Resolver<Array<ResolversTypes['RecommendErrorCode']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecommendResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecommendResult'] = ResolversParentTypes['RecommendResult']> = {
__resolveType: TypeResolveFn<'RecommendError' | 'RecommendSuccess', ParentType, ContextType>;
};
export type RecommendSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecommendSuccess'] = ResolversParentTypes['RecommendSuccess']> = {
taskNames?: Resolver<Array<ResolversTypes['String']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecommendationGroupResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecommendationGroup'] = ResolversParentTypes['RecommendationGroup']> = {
admins?: Resolver<Array<ResolversTypes['User']>, ParentType, ContextType>;
createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
@ -5591,6 +5644,9 @@ export type Resolvers<ContextType = ResolverContext> = {
RecentSearchesError?: RecentSearchesErrorResolvers<ContextType>;
RecentSearchesResult?: RecentSearchesResultResolvers<ContextType>;
RecentSearchesSuccess?: RecentSearchesSuccessResolvers<ContextType>;
RecommendError?: RecommendErrorResolvers<ContextType>;
RecommendResult?: RecommendResultResolvers<ContextType>;
RecommendSuccess?: RecommendSuccessResolvers<ContextType>;
RecommendationGroup?: RecommendationGroupResolvers<ContextType>;
Reminder?: ReminderResolvers<ContextType>;
ReminderError?: ReminderErrorResolvers<ContextType>;

22
packages/api/src/generated/schema.graphql
View File

@ -1010,6 +1010,7 @@ type Mutation {
moveFilter(input: MoveFilterInput!): MoveFilterResult!
moveLabel(input: MoveLabelInput!): MoveLabelResult!
optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
recommend(input: RecommendInput!): RecommendResult!
reportItem(input: ReportItemInput!): ReportItemResult!
revokeApiKey(id: ID!): RevokeApiKeyResult!
saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult!
@ -1219,6 +1220,27 @@ type RecentSearchesSuccess {
searches: [RecentSearch!]!
}
type RecommendError {
errorCodes: [RecommendErrorCode!]!
}
enum RecommendErrorCode {
BAD_REQUEST
NOT_FOUND
UNAUTHORIZED
}
input RecommendInput {
groupIds: [ID!]!
pageId: ID!
}
union RecommendResult = RecommendError | RecommendSuccess
type RecommendSuccess {
taskNames: [String!]!
}
type RecommendationGroup {
admins: [User!]!
createdAt: Date!

3
packages/api/src/resolvers/function_resolvers.ts
View File

@ -63,6 +63,7 @@ import {
moveFilterResolver,
moveLabelResolver,
newsletterEmailsResolver,
recommendResolver,
reminderResolver,
reportItemResolver,
revokeApiKeyResolver,
@ -189,6 +190,7 @@ export const functionResolvers = {
deleteFilter: deleteFilterResolver,
moveFilter: moveFilterResolver,
createGroup: createGroupResolver,
recommend: recommendResolver,
},
Query: {
me: getMeUserResolver,
@ -640,4 +642,5 @@ export const functionResolvers = {
...resultResolveTypeResolver('MoveFilter'),
...resultResolveTypeResolver('CreateGroup'),
...resultResolveTypeResolver('Groups'),
...resultResolveTypeResolver('Recommend'),
}

91
packages/api/src/resolvers/recommendations/index.ts
View File

@ -6,6 +6,10 @@ import {
GroupsErrorCode,
GroupsSuccess,
MutationCreateGroupArgs,
MutationRecommendArgs,
RecommendError,
RecommendErrorCode,
RecommendSuccess,
} from '../../generated/graphql'
import {
createGroup,
@ -15,6 +19,11 @@ import {
import { authorized, userDataToUser } from '../../utils/helpers'
import { getRepository } from '../../entity/utils'
import { User } from '../../entity/user'
import { Group } from '../../entity/groups/group'
import { In } from 'typeorm'
import { getPageByParam } from '../../elastic/pages'
import { enqueueRecommendation } from '../../utils/createTask'
import { env } from '../../env'
export const createGroupResolver = authorized<
CreateGroupSuccess,
@ -116,3 +125,85 @@ export const groupsResolver = authorized<GroupsSuccess, GroupsError>(
}
}
)
export const recommendResolver = authorized<
RecommendSuccess,
RecommendError,
MutationRecommendArgs
>(async (_, { input }, { claims: { uid }, log, signToken }) => {
log.info('Recommend', {
input,
labels: {
source: 'resolver',
resolver: 'recommendResolver',
uid,
},
})
try {
const user = await getRepository(User).findOneBy({
id: uid,
})
if (!user) {
return {
errorCodes: [RecommendErrorCode.Unauthorized],
}
}
const groups = await getRepository(Group).find({
where: { id: In(input.groupIds) },
relations: ['members', 'members.user'],
})
if (groups.length === 0) {
return {
errorCodes: [RecommendErrorCode.NotFound],
}
}
const page = await getPageByParam({ _id: input.pageId, userId: uid })
if (!page) {
return {
errorCodes: [RecommendErrorCode.NotFound],
}
}
const exp = Math.floor(Date.now() / 1000) + 60 * 60 * 24 // 1 day
const auth = (await signToken({ uid, exp }, env.server.jwtSecret)) as string
const taskNames = await Promise.all(
groups
.map((group) =>
group.members
.filter((member) => member.user.id !== uid)
.map((member) =>
enqueueRecommendation(
member.user.id,
page.id,
{
...group,
recommendedAt: new Date(),
},
auth
)
)
)
.flat()
)
return {
taskNames,
}
} catch (error) {
log.error('Error recommending', {
error,
labels: {
source: 'resolver',
resolver: 'recommendResolver',
uid,
},
})
return {
errorCodes: [RecommendErrorCode.BadRequest],
}
}
})

56
packages/api/src/routers/page_router.ts
View File

@ -5,17 +5,14 @@
import express from 'express'
import {
ArticleSavingRequestStatus,
CreateArticleErrorCode,
PageType,
UploadFileStatus,
} from '../generated/graphql'
import { isSiteBlockedForParse } from '../utils/blocked'
import cors from 'cors'
import { env } from '../env'
import { buildLogger } from '../utils/logger'
import * as jwt from 'jsonwebtoken'
import { corsConfig } from '../utils/corsConfig'
import { createPageSaveRequest } from '../services/create_page_save_request'
import { initModels } from '../server'
import { kx } from '../datalayer/knex_config'
import {
@ -32,6 +29,8 @@ import {
import { Claims } from '../resolvers/types'
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
import { createPubSubClient } from '../datalayer/pubsub'
import { Recommendation } from '../elastic/types'
import { addRecommendation } from '../elastic/recommendation'
const logger = buildLogger('app.dispatch')
@ -149,5 +148,56 @@ export function pageRouter() {
return res.redirect(signedUrl)
})
// Add recommended pages to a user's library
router.options(
'/recommend',
cors<express.Request>({ ...corsConfig, maxAge: 600 })
)
router.post(
'/recommend',
cors<express.Request>(corsConfig),
async (req, res) => {
const token = req?.cookies?.auth || req?.headers?.authorization
if (!token || !jwt.verify(token, env.server.jwtSecret)) {
return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
}
const claims = jwt.decode(token) as Claims
const { userId, pageId, recommendation } = req.body as {
userId: string
pageId: string
recommendation: Recommendation
}
if (!userId || !pageId || !recommendation) {
return res.status(400).send({ errorCode: 'BAD_DATA' })
}
const ctx = {
uid: userId,
pubsub: createPubSubClient(),
}
const page = await getPageByParam({
userId: claims.uid,
_id: pageId,
})
if (!page) {
return res.status(404).send({ errorCode: 'NOT_FOUND' })
}
const recommendedPageId = await addRecommendation(
ctx,
page,
recommendation
)
if (!recommendedPageId) {
logger.error('Failed to add recommendation to page')
return res.sendStatus(500)
}
return res.send({ recommendedPageId })
}
)
return router
}

22
packages/api/src/schema.ts
View File

@ -2183,6 +2183,27 @@ const schema = gql`
BAD_REQUEST
}
input RecommendInput {
pageId: ID!
groupIds: [ID!]!
}
union RecommendResult = RecommendSuccess | RecommendError
type RecommendSuccess {
taskNames: [String!]!
}
type RecommendError {
errorCodes: [RecommendErrorCode!]!
}
enum RecommendErrorCode {
UNAUTHORIZED
BAD_REQUEST
NOT_FOUND
}
# Mutations
type Mutation {
googleLogin(input: GoogleLoginInput!): LoginResult!
@ -2261,6 +2282,7 @@ const schema = gql`
deleteFilter(id: ID!): DeleteFilterResult!
moveFilter(input: MoveFilterInput!): MoveFilterResult!
createGroup(input: CreateGroupInput!): CreateGroupResult!
recommend(input: RecommendInput!): RecommendResult!
}
# FIXME: remove sort from feedArticles after all cached tabs are closed

7
packages/api/src/util.ts
View File

@ -62,9 +62,10 @@ interface BackendEnv {
name: string
contentFetchUrl: string
contentFetchGCFUrl: string
reminderTaskHanderUrl: string
reminderTaskHandlerUrl: string
integrationTaskHandlerUrl: string
textToSpeechTaskHandlerUrl: string
recommendationTaskHandlerUrl: string
}
fileUpload: {
gcsUploadBucket: string
@ -152,6 +153,7 @@ const nullableEnvVars = [
'AZURE_SPEECH_KEY',
'AZURE_SPEECH_REGION',
'GCP_LOCATION',
'RECOMMENDATION_TASK_HANDLER_URL',
] // Allow some vars to be null/empty
/* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
@ -234,9 +236,10 @@ export function getEnv(): BackendEnv {
name: parse('PUPPETEER_QUEUE_NAME'),
contentFetchUrl: parse('CONTENT_FETCH_URL'),
contentFetchGCFUrl: parse('CONTENT_FETCH_GCF_URL'),
reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'),
reminderTaskHandlerUrl: parse('REMINDER_TASK_HANDLER_URL'),
integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'),
textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'),
recommendationTaskHandlerUrl: parse('RECOMMENDATION_TASK_HANDLER_URL'),
}
const imageProxy = {
url: parse('IMAGE_PROXY_URL'),

56
packages/api/src/utils/createTask.ts
View File

@ -10,6 +10,7 @@ import { nanoid } from 'nanoid'
import { google } from '@google-cloud/tasks/build/protos/protos'
import { IntegrationType } from '../entity/integration'
import { signFeatureToken } from '../services/features'
import { Recommendation } from '../elastic/types'
import View = google.cloud.tasks.v2.Task.View
const logger = buildLogger('app.dispatch')
@ -26,6 +27,7 @@ const createHttpTaskWithToken = async ({
payload,
priority = 'high',
scheduleTime,
requestHeaders,
}: {
project: string
queue?: string
@ -35,6 +37,7 @@ const createHttpTaskWithToken = async ({
payload: unknown
priority?: 'low' | 'high'
scheduleTime?: number
requestHeaders?: Record<string, string>
}): Promise<
[
protos.google.cloud.tasks.v2.ITask,
@ -70,6 +73,7 @@ const createHttpTaskWithToken = async ({
url: taskHandlerUrl,
headers: {
'Content-Type': 'application/json',
...requestHeaders,
},
body,
...(serviceAccountEmail
@ -95,7 +99,7 @@ export const createAppEngineTask = async ({
project,
queue = env.queue.name,
location = env.queue.location,
taskHandlerUrl = env.queue.reminderTaskHanderUrl,
taskHandlerUrl = env.queue.reminderTaskHandlerUrl,
payload,
priority = 'high',
scheduleTime,
@ -278,7 +282,7 @@ export const enqueueReminder = async (
project: GOOGLE_CLOUD_PROJECT,
payload,
scheduleTime,
taskHandlerUrl: env.queue.reminderTaskHanderUrl,
taskHandlerUrl: env.queue.reminderTaskHandlerUrl,
})
if (!createdTasks || !createdTasks[0].name) {
@ -405,4 +409,52 @@ export const enqueueTextToSpeech = async ({
return createdTasks[0].name
}
export const enqueueRecommendation = async (
userId: string,
pageId: string,
recommendation: Recommendation,
authToken: string
): Promise<string> => {
const { GOOGLE_CLOUD_PROJECT } = process.env
const payload = {
userId,
pageId,
recommendation,
}
const headers = {
Authorization: authToken,
}
// If there is no Google Cloud Project Id exposed, it means that we are in local environment
if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) {
// Calling the handler function directly.
setTimeout(() => {
axios
.post(env.queue.recommendationTaskHandlerUrl, payload, {
headers,
})
.catch((error) => {
logger.error(error)
})
}, 0)
return ''
}
const createdTasks = await createHttpTaskWithToken({
project: GOOGLE_CLOUD_PROJECT,
payload,
taskHandlerUrl: env.queue.recommendationTaskHandlerUrl,
requestHeaders: headers,
})
if (!createdTasks || !createdTasks[0].name) {
logger.error(`Unable to get the name of the task`, {
payload,
createdTasks,
})
throw new CreateTaskError(`Unable to get the name of the task`)
}
return createdTasks[0].name
}
export default createHttpTaskWithToken

15
packages/db/elastic_migrations/index_settings.json
View File

@ -160,6 +160,21 @@
},
"wordsCount": {
"type": "integer"
},
"recommendedBy": {
"type": "nested",
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"recommendedAt": {
"type": "date"
}
}
}
}
}