Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace library item

Browse Source
This commit is contained in:
Hongbo Wu
2023-09-03 22:53:34 +08:00
parent c655f78f8c
commit b1899e340d
24 changed files with 772 additions and 1026 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/api/src/entity/library_item.ts
View File

@ -178,7 +178,8 @@ export class LibraryItem {
@OneToMany(
() => Recommendation,
(recommendation) => recommendation.libraryItem
(recommendation) => recommendation.libraryItem,
{ cascade: true }
)
@JoinTable({
name: 'recommendation',
@ -190,7 +191,9 @@ export class LibraryItem {
@Column('enum', { enum: DirectionalityType, default: DirectionalityType.LTR })
directionality!: DirectionalityType
@OneToMany(() => Highlight, (highlight) => highlight.libraryItem)
@OneToMany(() => Highlight, (highlight) => highlight.libraryItem, {
cascade: true,
})
@JoinTable({
name: 'highlight',
joinColumn: { name: 'library_item_id' },

44
packages/api/src/repository/index.ts
View File

@ -1,23 +1,5 @@
import { EntityManager, EntityTarget, Repository } from 'typeorm'
import { EntityManager } from 'typeorm'
import { appDataSource } from '../data_source'
import { Feature } from '../entity/feature'
import { Filter } from '../entity/filter'
import { Follower } from '../entity/follower'
import { Group } from '../entity/groups/group'
import { GroupMembership } from '../entity/groups/group_membership'
import { Invite } from '../entity/groups/invite'
import { Integration } from '../entity/integration'
import { NewsletterEmail } from '../entity/newsletter_email'
import { Profile } from '../entity/profile'
import { ReceivedEmail } from '../entity/received_email'
import { Recommendation } from '../entity/recommendation'
import { AbuseReport } from '../entity/reports/abuse_report'
import { ContentDisplayReport } from '../entity/reports/content_display_report'
import { Rule } from '../entity/rule'
import { Subscription } from '../entity/subscription'
import { UserDeviceToken } from '../entity/user_device_tokens'
import { UserPersonalization } from '../entity/user_personalization'
import { Webhook } from '../entity/webhook'
export const setClaims = async (
manager: EntityManager,
@ -30,10 +12,6 @@ export const setClaims = async (
])
}
export const getRepository = <T>(entity: EntityTarget<T>): Repository<T> => {
return entityManager.getRepository(entity)
}
export const authTrx = async <T>(
fn: (manager: EntityManager) => Promise<T>,
uid = '00000000-0000-0000-0000-000000000000',
@ -46,23 +24,3 @@ export const authTrx = async <T>(
}
export const entityManager = appDataSource.manager
export const groupMembershipRepository = getRepository(GroupMembership)
export const groupRepository = getRepository(Group)
export const inviteRepository = getRepository(Invite)
export const abuseReportRepository = getRepository(AbuseReport)
export const contentDisplayReportRepository =
getRepository(ContentDisplayReport)
export const featureRepository = getRepository(Feature)
export const filterRepository = getRepository(Filter)
export const followerRepository = getRepository(Follower)
export const integrationRepository = getRepository(Integration)
export const newsletterEmailRepository = getRepository(NewsletterEmail)
export const profileRepository = getRepository(Profile)
export const receivedEmailRepository = getRepository(ReceivedEmail)
export const recommendationRepository = getRepository(Recommendation)
export const ruleRepository = getRepository(Rule)
export const subscriptionRepository = getRepository(Subscription)
export const userDeviceTokenRepository = getRepository(UserDeviceToken)
export const userPersonalizationRepository = getRepository(UserPersonalization)
export const webhookRepository = getRepository(Webhook)

10
packages/api/src/repository/library_item.ts
View File

@ -1,16 +1,6 @@
import { entityManager } from '.'
import { LibraryItem } from '../entity/library_item'
export const getLibraryItemById = async (id: string) => {
return libraryItemRepository.findOneBy({ id })
}
export const getLibraryItemByUrl = async (url: string) => {
return libraryItemRepository.findOneBy({
originalUrl: url,
})
}
export const libraryItemRepository = entityManager
.getRepository(LibraryItem)
.extend({

4
packages/api/src/repository/profile.ts Normal file
View File

@ -0,0 +1,4 @@
import { Profile } from '../entity/profile'
import { entityManager } from '.'
export const profileRepository = entityManager.getRepository(Profile)

8
packages/api/src/repository/upload_file.ts
View File

@ -1,4 +1,10 @@
import { entityManager } from '.'
import { UploadFile } from '../entity/upload_file'
export const uploadFileRepository = entityManager.getRepository(UploadFile)
export const uploadFileRepository = entityManager
.getRepository(UploadFile)
.extend({
findById(id: string) {
return this.findOneBy({ id })
},
})

17
packages/api/src/resolvers/api_key/index.ts
View File

@ -1,4 +1,3 @@
import { ApiKey } from '../../entity/api_key'
import { env } from '../../env'
import {
ApiKeysError,
@ -13,6 +12,7 @@ import {
RevokeApiKeyErrorCode,
RevokeApiKeySuccess,
} from '../../generated/graphql'
import { apiKeyRepository } from '../../repository/api_key'
import { analytics } from '../../utils/analytics'
import { generateApiKey, hashApiKey } from '../../utils/auth'
import { authorized } from '../../utils/helpers'
@ -20,8 +20,8 @@ import { authorized } from '../../utils/helpers'
export const apiKeysResolver = authorized<ApiKeysSuccess, ApiKeysError>(
async (_, __, { log, authTrx }) => {
try {
const apiKeys = await authTrx<Promise<ApiKey[]>>(async (tx) => {
return tx.find(ApiKey, {
const apiKeys = await authTrx(async (tx) => {
return tx.withRepository(apiKeyRepository).find({
select: ['id', 'name', 'scopes', 'expiresAt', 'createdAt', 'usedAt'],
order: {
usedAt: { direction: 'DESC', nulls: 'last' },
@ -51,8 +51,8 @@ export const generateApiKeyResolver = authorized<
try {
const exp = new Date(expiresAt)
const originalKey = generateApiKey()
const apiKeyCreated = await authTrx<Promise<ApiKey>>(async (tx) => {
return tx.save(ApiKey, {
const apiKeyCreated = await authTrx(async (tx) => {
return tx.withRepository(apiKeyRepository).save({
user: { id: uid },
name,
key: hashApiKey(originalKey),
@ -89,13 +89,14 @@ export const revokeApiKeyResolver = authorized<
MutationRevokeApiKeyArgs
>(async (_, { id }, { claims: { uid }, log, authTrx }) => {
try {
const deletedApiKey = await authTrx<Promise<ApiKey | null>>(async (tx) => {
const apiKey = await tx.findOneBy(ApiKey, { id })
const deletedApiKey = await authTrx(async (tx) => {
const apiRepo = tx.withRepository(apiKeyRepository)
const apiKey = await apiRepo.findOneBy({ id })
if (!apiKey) {
return null
}
return tx.remove(ApiKey, apiKey)
return apiRepo.remove(apiKey)
})
if (!deletedApiKey) {

718
packages/api/src/resolvers/article/index.ts
View File

File diff suppressed because it is too large Load Diff

54
packages/api/src/resolvers/article_saving_request/index.ts
View File

@ -1,11 +1,9 @@
/* eslint-disable prefer-const */
import { getPageByParam } from '../../elastic/pages'
import { User } from '../../entity/user'
import { LibraryItem, LibraryItemState } from '../../entity/library_item'
import { env } from '../../env'
import {
ArticleSavingRequestError,
ArticleSavingRequestErrorCode,
ArticleSavingRequestStatus,
ArticleSavingRequestSuccess,
CreateArticleSavingRequestError,
CreateArticleSavingRequestErrorCode,
@ -13,14 +11,18 @@ import {
MutationCreateArticleSavingRequestArgs,
QueryArticleSavingRequestArgs,
} from '../../generated/graphql'
import { getRepository } from '../../repository'
import { userRepository } from '../../repository/user'
import { createPageSaveRequest } from '../../services/create_page_save_request'
import {
findLibraryItemById,
findLibraryItemByUrl,
} from '../../services/library_item'
import { analytics } from '../../utils/analytics'
import {
authorized,
cleanUrl,
isParsingTimeout,
pageToArticleSavingRequest,
libraryItemToArticleSavingRequest,
} from '../../utils/helpers'
import { isErrorWithCode } from '../user'
@ -63,35 +65,37 @@ export const articleSavingRequestResolver = authorized<
ArticleSavingRequestSuccess,
ArticleSavingRequestError,
QueryArticleSavingRequestArgs
>(async (_, { id, url }, { claims }) => {
>(async (_, { id, url }, { uid, log }) => {
try {
if (!id && !url) {
return { errorCodes: [ArticleSavingRequestErrorCode.BadData] }
}
const user = await getRepository(User).findOne({
where: { id: claims.uid },
relations: ['profile'],
})
const user = await userRepository.findById(uid)
if (!user) {
return { errorCodes: [ArticleSavingRequestErrorCode.Unauthorized] }
}
const normalizedUrl = url ? cleanUrl(url) : undefined
const params = {
_id: id || undefined,
url: normalizedUrl,
userId: claims.uid,
state: [
ArticleSavingRequestStatus.Succeeded,
ArticleSavingRequestStatus.Processing,
],
let libraryItem: LibraryItem | null = null
if (id) {
libraryItem = await findLibraryItemById(id, uid)
} else if (url) {
libraryItem = await findLibraryItemByUrl(cleanUrl(url), uid)
}
const page = await getPageByParam(params)
if (!page) {
if (!libraryItem) {
return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] }
}
if (isParsingTimeout(page)) {
page.state = ArticleSavingRequestStatus.Succeeded
if (isParsingTimeout(libraryItem)) {
libraryItem.state = LibraryItemState.Succeeded
}
return {
articleSavingRequest: libraryItemToArticleSavingRequest(
user,
libraryItem
),
}
} catch (error) {
log.error('articleSavingRequestResolver error', error)
return { errorCodes: [ArticleSavingRequestErrorCode.Unauthorized] }
}
return { articleSavingRequest: pageToArticleSavingRequest(user, page) }
})

2
packages/api/src/resolvers/filters/index.ts
View File

@ -33,7 +33,7 @@ export const saveFilterResolver = authorized<
SaveFilterSuccess,
SaveFilterError,
MutationSaveFilterArgs
>(async (_, { input }, { claims: { uid }, log }) => {
>(async (_, { input }, { authTrx, uid, log }) => {
log.info('Saving filters', {
input,
labels: {

111
packages/api/src/resolvers/function_resolvers.ts
View File

@ -4,22 +4,12 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
import { getPageByParam } from '../elastic/pages'
import { getRepository } from '../repository'
import { Subscription } from '../entity/subscription'
import { UploadFile } from '../entity/upload_file'
import { User } from '../entity/user'
import {
Article,
ArticleHighlightsInput,
Highlight,
HighlightType,
PageType,
SearchItem,
} from '../generated/graphql'
import { userDataToUser, validatedDate, wordsCount } from '../utils/helpers'
import { Article, PageType, SearchItem } from '../generated/graphql'
import { findUploadFileById } from '../services/upload_file'
import { validatedDate, wordsCount } from '../utils/helpers'
import { createImageProxyUrl } from '../utils/imageproxy'
import {
contentReaderForPage,
generateDownloadSignedUrl,
generateUploadFilePathName,
} from '../utils/uploads'
@ -51,7 +41,6 @@ import {
generateApiKeyResolver,
getAllUsersResolver,
getArticleResolver,
getArticlesResolver,
// getFollowersResolver,
// getFollowingResolver,
getMeUserResolver,
@ -219,7 +208,6 @@ export const functionResolvers = {
validateUsername: validateUsernameResolver,
article: getArticleResolver,
// sharedArticle: getSharedArticleResolver,
articles: getArticlesResolver,
// feedArticles: getUserFeedArticlesResolver,
// getFollowers: getFollowersResolver,
// getFollowing: getFollowingResolver,
@ -387,9 +375,10 @@ export const functionResolvers = {
ctx.claims &&
article.uploadFileId
) {
const upload = await getRepository(UploadFile).findOneBy({
id: article.uploadFileId,
})
const upload = await findUploadFileById(
article.uploadFileId,
ctx.claims.uid
)
if (!upload || !upload.fileName) {
return undefined
}
@ -431,24 +420,6 @@ export const functionResolvers = {
})
return !!page?.sharedAt
},
async savedAt(
article: { id: string; savedAt?: Date; createdAt?: Date },
__: unknown,
ctx: WithDataSourcesContext & { claims: Claims }
) {
if (!ctx.claims?.uid) return new Date()
if (article.savedAt) return article.savedAt
return (
(
await getPageByParam({
userId: ctx.claims.uid,
_id: article.id,
})
)?.savedAt ||
article.createdAt ||
new Date()
)
},
hasContent(article: {
content: string | null
originalHtml: string | null
@ -458,37 +429,6 @@ export const functionResolvers = {
publishedAt(article: { publishedAt: Date }) {
return validatedDate(article.publishedAt)
},
async isArchived(
article: {
id: string
isArchived?: boolean | null
archivedAt?: Date | undefined
},
__: unknown,
ctx: WithDataSourcesContext & { claims: Claims }
) {
if ('isArchived' in article) return article.isArchived
if ('archivedAt' in article) return !!article.archivedAt
if (!ctx.claims?.uid) return false
const page = await getPageByParam({
userId: ctx.claims.uid,
_id: article.id,
})
return !!page?.archivedAt || false
},
contentReader(article: {
pageType: PageType
uploadFileId: string | undefined
}) {
return contentReaderForPage(article.pageType, article.uploadFileId)
},
highlights(
article: { id: string; userId?: string; highlights?: Highlight[] },
_: { input: ArticleHighlightsInput },
ctx: WithDataSourcesContext
) {
return article.highlights || []
},
// async shareInfo(
// article: { id: string; sharedBy?: User; shareInfo?: LinkShareInfo },
// __: unknown,
@ -511,29 +451,7 @@ export const functionResolvers = {
return article.content ? wordsCount(article.content) : undefined
},
},
ArticleSavingRequest: {
async article(request: { userId: string; articleId: string }, __: unknown) {
if (!request.userId || !request.articleId) return undefined
return getPageByParam({
userId: request.userId,
_id: request.articleId,
})
},
},
Highlight: {
async user(
highlight: { userId: string },
__: unknown,
ctx: WithDataSourcesContext
) {
const userData = await getRepository(User).findOneBy({
id: highlight.userId,
})
if (!userData) return null
return userDataToUser(userData)
},
// async reactions(
// highlight: { id: string; reactions?: Reaction[] },
// _: unknown,
@ -549,10 +467,7 @@ export const functionResolvers = {
__: unknown,
ctx: WithDataSourcesContext
) {
return highlight.createdByMe ?? highlight.userId === ctx.claims?.uid
},
type(highlight: { type: HighlightType }) {
return highlight.type || HighlightType.Highlight
return highlight.createdByMe ?? highlight.userId === ctx.uid
},
},
// Reaction: {
@ -568,12 +483,10 @@ export const functionResolvers = {
async url(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) {
if (
(item.pageType == PageType.File || item.pageType == PageType.Book) &&
ctx.claims &&
ctx.uid &&
item.uploadFileId
) {
const upload = await getRepository(UploadFile).findOneBy({
id: item.uploadFileId,
})
const upload = await findUploadFileById(item.uploadFileId, ctx.uid)
if (!upload || !upload.fileName) {
return undefined
}
@ -582,8 +495,8 @@ export const functionResolvers = {
}
return item.url
},
pageType(item: SearchItem) {
return item.pageType || PageType.Unknown
image(item: SearchItem) {
return item.image && createImageProxyUrl(item.image, 320, 320)
},
},
Subscription: {

8
packages/api/src/schema.ts
View File

@ -2701,14 +2701,6 @@ const schema = gql`
hello: String
me: User
user(userId: ID, username: String): UserResult!
articles(
sharedOnly: Boolean
sort: SortParams
after: String
first: Int
query: String
includePending: Boolean
): ArticlesResult!
article(username: String!, slug: String!, format: String): ArticleResult!
# sharedArticle(
# username: String!

19
packages/api/src/services/archive_link.ts
View File

@ -1,19 +0,0 @@
import { appDataSource } from '../data_source'
import { Link } from '../entity/link'
import { setClaims } from '../repository'
export const setLinkArchived = async (
userId: string,
linkId: string,
archived: boolean
): Promise<void> => {
await appDataSource.transaction(async (t) => {
await setClaims(t, userId)
await t.getRepository(Link).update(
{
id: linkId,
},
{ archivedAt: archived ? new Date() : null }
)
})
}

85
packages/api/src/services/create_page_save_request.ts
View File

@ -1,34 +1,36 @@
import * as privateIpLib from 'private-ip'
import { v4 as uuidv4 } from 'uuid'
import { countByCreatedAt, createPage, updatePage } from '../elastic/pages'
import { countByCreatedAt } from '../elastic/pages'
import { ArticleSavingRequestStatus, PageType } from '../elastic/types'
import { LibraryItemState } from '../entity/library_item'
import { User } from '../entity/user'
import { LibraryItemState, LibraryItemType } from '../entity/library_item'
import {
ArticleSavingRequest,
CreateArticleSavingRequestErrorCode,
CreateLabelInput
CreateLabelInput,
} from '../generated/graphql'
import { createPubSubClient, PubsubClient } from '../pubsub'
import { libraryItemRepository } from '../repository/library_item'
import { userRepository } from '../repository/user'
import { enqueueParseRequest } from '../utils/createTask'
import {
cleanUrl,
generateSlug,
pageToArticleSavingRequest
libraryItemToArticleSavingRequest,
} from '../utils/helpers'
import { logger } from '../utils/logger'
import {
createLibraryItem,
findLibraryItemByUrl,
updateLibraryItem,
} from './library_item'
interface PageSaveRequest {
userId: string
url: string
pubsub?: PubsubClient
articleSavingRequestId?: string
archivedAt?: Date | null
state?: ArticleSavingRequestStatus
labels?: CreateLabelInput[]
priority?: 'low' | 'high'
user?: User | null
locale?: string
timezone?: string
savedAt?: Date
@ -80,10 +82,9 @@ export const createPageSaveRequest = async ({
url,
pubsub = createPubSubClient(),
articleSavingRequestId = uuidv4(),
archivedAt,
state,
priority,
labels,
user,
locale,
timezone,
savedAt,
@ -92,62 +93,52 @@ export const createPageSaveRequest = async ({
try {
validateUrl(url)
} catch (error) {
logger.error('invalid url', { url, error })
logger.info('invalid url', { url, error })
return Promise.reject({
errorCode: CreateArticleSavingRequestErrorCode.BadData,
})
}
// if user is not specified, get it from the database
if (!user) {
user = await userRepository.findById(userId)
const user = await userRepository.findById(userId)
if (!user) {
logger.info('User not found', userId)
return Promise.reject({
errorCode: CreateArticleSavingRequestErrorCode.BadData,
})
}
}
url = cleanUrl(url)
// look for existing library item
const existingLibraryItem = await libraryItemRepository.findByUrl(url)
if (!existingLibraryItem) {
let libraryItem = await findLibraryItemByUrl(url, userId)
if (!libraryItem) {
logger.info('libraryItem does not exist', { url })
libraryItem = {
id: articleSavingRequestId,
user: { id: userId },
content: SAVING_CONTENT,
hash: '',
pageType: PageType.Unknown,
readingProgressAnchorIndex: 0,
readingProgressPercent: 0,
slug: generateSlug(url),
title: url,
url,
state: LibraryItemState.Processing,
createdAt: new Date(),
savedAt: savedAt || new Date(),
publishedAt,
archivedAt,
}
// create processing page
const pageId = await createPage(page, ctx)
if (!pageId) {
logger.info('Failed to create page', url)
return Promise.reject({
errorCode: CreateArticleSavingRequestErrorCode.BadData,
})
}
libraryItem = await createLibraryItem(
{
id: articleSavingRequestId,
user: { id: userId },
readableContent: SAVING_CONTENT,
itemType: LibraryItemType.Unknown,
slug: generateSlug(url),
title: url,
originalUrl: url,
state: LibraryItemState.Processing,
publishedAt,
},
userId,
pubsub
)
}
// reset state to processing
if (existingLibraryItem.state !== ArticleSavingRequestStatus.Processing) {
await updatePage(
page.id,
if (libraryItem.state !== LibraryItemState.Processing) {
libraryItem = await updateLibraryItem(
libraryItem.id,
{
state: ArticleSavingRequestStatus.Processing,
state: LibraryItemState.Processing,
},
ctx
userId,
pubsub
)
}
@ -160,7 +151,7 @@ export const createPageSaveRequest = async ({
userId,
saveRequestId: articleSavingRequestId,
priority,
state: archivedAt ? ArticleSavingRequestStatus.Archived : undefined,
state,
labels,
locale,
timezone,
@ -168,5 +159,5 @@ export const createPageSaveRequest = async ({
publishedAt,
})
return pageToArticleSavingRequest(user, page)
return libraryItemToArticleSavingRequest(user, libraryItem)
}

4
packages/api/src/services/create_user.ts
View File

@ -6,7 +6,7 @@ import { Profile } from '../entity/profile'
import { StatusType, User } from '../entity/user'
import { SignupErrorCode } from '../generated/graphql'
import { getRepository } from '../repository'
import { getUserByEmail } from '../repository/user'
import { userRepository } from '../repository/user'
import { AuthProvider } from '../routers/auth/auth_types'
import { logger } from '../utils/logger'
import { validateUsername } from '../utils/usernamePolicy'
@ -41,7 +41,7 @@ export const createUser = async (input: {
pendingConfirmation?: boolean
}): Promise<[User, Profile]> => {
const trimmedEmail = input.email.trim()
const existingUser = await getUserByEmail(trimmedEmail)
const existingUser = await userRepository.findByEmail(trimmedEmail)
if (existingUser) {
if (existingUser.profile) {
return Promise.reject({ errorCode: SignupErrorCode.UserExists })

83
packages/api/src/services/labels.ts
View File

@ -1,35 +1,35 @@
import DataLoader from 'dataloader'
import { In } from 'typeorm'
import { Highlight } from '../entity/highlight'
import { Label } from '../entity/label'
import { LibraryItem } from '../entity/library_item'
import { Link } from '../entity/link'
import { EntityType, PubsubClient } from '../pubsub'
import { authTrx, getRepository } from '../repository'
import { createPubSubClient, EntityType } from '../pubsub'
import { entityManager, setClaims } from '../repository'
import { highlightRepository } from '../repository/highlight'
import { CreateLabelInput, labelRepository } from '../repository/label'
import { libraryItemRepository } from '../repository/library_item'
const batchGetLabelsFromLinkIds = async (
linkIds: readonly string[]
): Promise<Label[][]> => {
const links = await getRepository(Link).find({
where: { id: In(linkIds as string[]) },
relations: ['labels'],
})
// const batchGetLabelsFromLinkIds = async (
// linkIds: readonly string[]
// ): Promise<Label[][]> => {
// const links = await getRepository(Link).find({
// where: { id: In(linkIds as string[]) },
// relations: ['labels'],
// })
return linkIds.map(
(linkId) => links.find((link) => link.id === linkId)?.labels || []
)
}
// return linkIds.map(
// (linkId) => links.find((link) => link.id === linkId)?.labels || []
// )
// }
export const labelsLoader = new DataLoader(batchGetLabelsFromLinkIds)
// export const labelsLoader = new DataLoader(batchGetLabelsFromLinkIds)
export const getLabelsAndCreateIfNotExist = async (
labels: CreateLabelInput[],
userId: string
userId: string,
em = entityManager
): Promise<Label[]> => {
return authTrx(async (tx) => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
const labelRepo = tx.withRepository(labelRepository)
// find existing labels
const labelEntities = await labelRepo.findByNames(labels.map((l) => l.name))
@ -55,15 +55,14 @@ export const saveLabelsInLibraryItem = async (
labels: Label[],
libraryItemId: string,
userId: string,
pubsub: PubsubClient
pubsub = createPubSubClient(),
em = entityManager
) => {
await authTrx(async (tx) => {
await em.transaction(async (tx) => {
await setClaims(tx, userId)
await tx
.withRepository(libraryItemRepository)
.createQueryBuilder()
.relation(LibraryItem, 'labels')
.of(libraryItemId)
.set(labels)
.update(libraryItemId, { labels })
})
// create pubsub event
@ -78,9 +77,11 @@ export const addLabelsToLibraryItem = async (
labels: Label[],
libraryItemId: string,
userId: string,
pubsub: PubsubClient
pubsub = createPubSubClient(),
em = entityManager
) => {
await authTrx(async (tx) => {
await em.transaction(async (tx) => {
await setClaims(tx, userId)
await tx
.withRepository(libraryItemRepository)
.createQueryBuilder()
@ -101,15 +102,13 @@ export const saveLabelsInHighlight = async (
labels: Label[],
highlightId: string,
userId: string,
pubsub: PubsubClient
pubsub = createPubSubClient(),
em = entityManager
) => {
await authTrx(async (tx) => {
await tx
.withRepository(highlightRepository)
.createQueryBuilder()
.relation(Highlight, 'labels')
.of(highlightId)
.set(labels)
await em.transaction(async (tx) => {
await setClaims(tx, userId)
await tx.withRepository(highlightRepository).update(highlightId, { labels })
})
// create pubsub event
@ -119,3 +118,17 @@ export const saveLabelsInHighlight = async (
userId
)
}
export const findLabelsByIds = async (
ids: string[],
userId: string,
em = entityManager
): Promise<Label[]> => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx.withRepository(labelRepository).findBy({
id: In(ids),
})
})
}

157
packages/api/src/services/library_item.ts
View File

@ -7,9 +7,8 @@ import {
LibraryItemType,
} from '../entity/library_item'
import { createPubSubClient, EntityType } from '../pubsub'
import { entityManager } from '../repository'
import { entityManager, setClaims } from '../repository'
import { libraryItemRepository } from '../repository/library_item'
import { logger } from '../utils/logger'
import {
DateFilter,
FieldFilter,
@ -21,16 +20,13 @@ import {
ReadFilter,
SortBy,
SortOrder,
SortParams,
Sort,
} from '../utils/search'
const MAX_CONTENT_LENGTH = 10 * 1024 * 1024 // 10MB for readable content
const CONTENT_LENGTH_ERROR = 'Your page content is too large to be saved.'
export interface PageSearchArgs {
export interface SearchArgs {
from?: number
size?: number
sort?: SortParams
sort?: Sort
query?: string
inFilter?: InFilter
readFilter?: ReadFilter
@ -49,7 +45,7 @@ export interface PageSearchArgs {
siteName?: string
}
export interface SearchItem {
export interface SearchResultItem {
annotation?: string | null
author?: string | null
createdAt: Date
@ -83,36 +79,9 @@ export interface SearchItem {
content?: string
}
export const createLibraryItem = async (
libraryItem: DeepPartial<LibraryItem>,
pubsub = createPubSubClient()
): Promise<LibraryItem> => {
if (
libraryItem.readableContent &&
libraryItem.readableContent.length > MAX_CONTENT_LENGTH
) {
logger.warn('page content is too large', {
url: libraryItem.originalUrl,
contentLength: libraryItem.readableContent.length,
})
libraryItem.readableContent = CONTENT_LENGTH_ERROR
}
const newItem = await libraryItemRepository.save(libraryItem)
await pubsub.entityCreated<LibraryItem>(
EntityType.PAGE,
newItem,
newItem.user.id
)
return newItem
}
const buildWhereClause = (
queryBuilder: SelectQueryBuilder<LibraryItem>,
args: PageSearchArgs
args: SearchArgs
) => {
if (args.query) {
queryBuilder
@ -255,9 +224,10 @@ const buildWhereClause = (
}
export const searchLibraryItems = async (
args: PageSearchArgs,
userId: string
): Promise<[LibraryItem[], number] | null> => {
args: SearchArgs,
userId: string,
em = entityManager
): Promise<{ libraryItems: LibraryItem[]; count: number }> => {
const { from = 0, size = 10, sort } = args
// default order is descending
@ -265,7 +235,11 @@ export const searchLibraryItems = async (
// default sort by saved_at
const sortField = sort?.by || SortBy.SAVED
const queryBuilder = entityManager
// add pagination and sorting
return em.transaction(async (tx) => {
await setClaims(tx, userId)
const queryBuilder = tx
.createQueryBuilder(LibraryItem, 'library_item')
.leftJoinAndSelect('library_item.labels', 'labels')
.leftJoinAndSelect('library_item.highlights', 'highlights')
@ -274,7 +248,6 @@ export const searchLibraryItems = async (
// build the where clause
buildWhereClause(queryBuilder, args)
// add pagination and sorting
const libraryItems = await queryBuilder
.orderBy(`library_item.${sortField}`, sortOrder)
.offset(from)
@ -283,5 +256,103 @@ export const searchLibraryItems = async (
const count = await queryBuilder.getCount()
return [libraryItems, count]
return { libraryItems, count }
})
}
export const findLibraryItemById = async (
id: string,
userId: string,
em = entityManager
): Promise<LibraryItem | null> => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx
.createQueryBuilder(LibraryItem, 'library_item')
.leftJoinAndSelect('library_item.labels', 'labels')
.leftJoinAndSelect('library_item.highlights', 'highlights')
.where('library_item.user_id = :userId', { userId })
.andWhere('library_item.id = :id', { id })
.getOne()
})
}
export const findLibraryItemByUrl = async (
url: string,
userId: string,
em = entityManager
): Promise<LibraryItem | null> => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx
.createQueryBuilder(LibraryItem, 'library_item')
.leftJoinAndSelect('library_item.labels', 'labels')
.leftJoinAndSelect('library_item.highlights', 'highlights')
.where('library_item.user_id = :userId', { userId })
.andWhere('library_item.url = :url', { url })
.getOne()
})
}
export const updateLibraryItem = async (
id: string,
libraryItem: DeepPartial<LibraryItem>,
userId: string,
pubsub = createPubSubClient(),
em = entityManager
): Promise<LibraryItem> => {
const updatedLibraryItem = await em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx.withRepository(libraryItemRepository).save({ id, ...libraryItem })
})
await pubsub.entityUpdated<DeepPartial<LibraryItem>>(
EntityType.PAGE,
libraryItem,
userId
)
return updatedLibraryItem
}
export const createLibraryItem = async (
libraryItem: DeepPartial<LibraryItem>,
userId: string,
pubsub = createPubSubClient(),
em = entityManager
): Promise<LibraryItem> => {
const newLibraryItem = await em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx.withRepository(libraryItemRepository).save(libraryItem)
})
await pubsub.entityCreated<LibraryItem>(
EntityType.PAGE,
newLibraryItem,
userId
)
return newLibraryItem
}
export const findLibraryItemsByPrefix = async (
prefix: string,
userId: string,
limit = 5,
em = entityManager
): Promise<LibraryItem[]> => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx
.createQueryBuilder(LibraryItem, 'library_item')
.where('library_item.title ILIKE :prefix', { prefix: `${prefix}%` })
.orWhere('library_item.site_name ILIKE :prefix', { prefix: `${prefix}%` })
.limit(limit)
.getMany()
})
}

10
packages/api/src/services/save_file.ts
View File

@ -1,4 +1,3 @@
import { updatePage } from '../elastic/pages'
import { UploadFile } from '../entity/upload_file'
import { User } from '../entity/user'
import { homePageURL } from '../env'
@ -14,13 +13,6 @@ import { logger } from '../utils/logger'
import { getStorageFileDetails } from '../utils/uploads'
import { getLabelsAndCreateIfNotExist } from './labels'
export const setFileUploadComplete = async (
id: string,
em = entityManager
): Promise<UploadFile | null> => {
return em.getRepository(UploadFile).save({ id, status: 'COMPLETED' })
}
export const saveFile = async (
ctx: WithDataSourcesContext,
user: User,
@ -55,7 +47,7 @@ export const saveFile = async (
input.state === ArticleSavingRequestStatus.Archived ? new Date() : null
// add labels to page
const labels = input.labels
? await getLabelsAndCreateIfNotExist(ctx, input.labels)
? await getLabelsAndCreateIfNotExist(input.labels, user.id)
: undefined
if (input.state || input.labels) {
const updated = await updatePage(

4
packages/api/src/services/save_page.ts
View File

@ -109,7 +109,7 @@ export const savePage = async (
try {
await createPageSaveRequest({
userId: user.id,
url: itemToSave.originalUrl!,
url: itemToSave.originalUrl,
pubsub: ctx.pubsub,
articleSavingRequestId: input.clientRequestId,
archivedAt,
@ -230,7 +230,7 @@ export const parsedContentToLibraryItem = ({
saveTime?: Date
rssFeedUrl?: string | null
publishedAt?: Date | null
}): DeepPartial<LibraryItem> => {
}): DeepPartial<LibraryItem> & { originalUrl: string } => {
return {
id: pageId ?? undefined,
slug,

30
packages/api/src/services/upload_file.ts Normal file
View File

@ -0,0 +1,30 @@
import { entityManager, setClaims } from '../repository'
import { uploadFileRepository } from '../repository/upload_file'
export const findUploadFileById = async (
id: string,
userId: string,
em = entityManager
) => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
const uploadFile = await tx
.withRepository(uploadFileRepository)
.findById(id)
return uploadFile
})
}
export const setFileUploadComplete = async (
id: string,
userId: string,
em = entityManager
) => {
return em.transaction(async (tx) => {
await setClaims(tx, userId)
return tx
.withRepository(uploadFileRepository)
.save({ id, status: 'COMPLETED' })
})
}

12
packages/api/src/utils/auth.ts
View File

@ -4,9 +4,9 @@ import express from 'express'
import * as jwt from 'jsonwebtoken'
import { promisify } from 'util'
import { v4 as uuidv4 } from 'uuid'
import { getRepository } from '../repository'
import { ApiKey } from '../entity/api_key'
import { env } from '../env'
import { authTrx } from '../repository'
import { apiKeyRepository } from '../repository/api_key'
import { Claims, ClaimsToSet } from '../resolvers/types'
import { logger } from './logger'
@ -33,7 +33,10 @@ export const hashApiKey = (apiKey: string) => {
export const claimsFromApiKey = async (key: string): Promise<Claims> => {
const hashedKey = hashApiKey(key)
const apiKey = await getRepository(ApiKey).findOne({
return authTrx(async (tx) => {
const apiKeyRepo = tx.withRepository(apiKeyRepository)
const apiKey = await apiKeyRepo.findOne({
where: {
key: hashedKey,
},
@ -50,13 +53,14 @@ export const claimsFromApiKey = async (key: string): Promise<Claims> => {
}
// update last used
await getRepository(ApiKey).update(apiKey.id, { usedAt: new Date() })
await apiKeyRepo.update(apiKey.id, { usedAt: new Date() })
return {
uid: apiKey.user.id,
iat,
exp,
}
})
}
// verify jwt token first

2
packages/api/src/utils/createTask.ts
View File

@ -5,7 +5,7 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks'
import { google } from '@google-cloud/tasks/build/protos/protos'
import axios from 'axios'
import { nanoid } from 'nanoid'
import { Recommendation } from '../elastic/types'
import { Recommendation } from '../entity/recommendation'
import { Subscription } from '../entity/subscription'
import { env } from '../env'
import {

88
packages/api/src/utils/helpers.ts
View File

@ -5,20 +5,29 @@ import path from 'path'
import _ from 'underscore'
import slugify from 'voca/slugify'
import wordsCounter from 'word-counting'
import { updatePage } from '../elastic/pages'
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
import { LibraryItem } from '../entity/library_item'
import { Highlight as HighlightData } from '../entity/highlight'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { Recommendation as RecommendationData } from '../entity/recommendation'
import { RegistrationType, User } from '../entity/user'
import {
ArticleSavingRequest,
ArticleSavingRequestStatus,
ContentReader,
CreateArticleError,
FeedArticle,
Highlight,
HighlightType,
PageType,
Profile,
Recommendation,
ResolverFn,
SearchItem,
} from '../generated/graphql'
import { CreateArticlesSuccessPartial } from '../resolvers'
import { createPubSubClient } from '../pubsub'
import { CreateArticlesSuccessPartial, PartialArticle } from '../resolvers'
import { Claims, WithDataSourcesContext } from '../resolvers/types'
import { validateUrl } from '../services/create_page_save_request'
import { updateLibraryItem } from '../services/library_item'
import { Merge } from '../util'
import { logger } from './logger'
interface InputObject {
@ -170,23 +179,42 @@ export const MAX_CONTENT_LENGTH = 5e7 //50MB
export const pageError = async (
result: CreateArticleError,
ctx: WithDataSourcesContext,
pageId?: string | null
userId: string,
pageId?: string | null,
pubsub = createPubSubClient()
): Promise<CreateArticleError | CreateArticlesSuccessPartial> => {
if (!pageId) return result
await updatePage(
await updateLibraryItem(
pageId,
{
state: ArticleSavingRequestStatus.Failed,
state: LibraryItemState.Failed,
},
ctx
userId,
pubsub
)
return result
}
export const pageToArticleSavingRequest = (
const highlightDataToHighlight = (highlight: HighlightData): Highlight => ({
...highlight,
createdByMe: true,
reactions: [],
replies: [],
type: highlight.highlightType as unknown as HighlightType,
user: userDataToUser(highlight.user),
})
const recommandationDataToRecommendation = (
recommendation: RecommendationData
): Recommendation => ({
...recommendation,
name: recommendation.recommender.name,
recommendedAt: recommendation.createdAt,
})
export const libraryItemToArticleSavingRequest = (
user: User,
item: LibraryItem
): ArticleSavingRequest => ({
@ -197,11 +225,45 @@ export const pageToArticleSavingRequest = (
userId: user.id,
})
export const isParsingTimeout = (page: Page): boolean => {
export const libraryItemToPartialArticle = (
item: LibraryItem
): PartialArticle => ({
...item,
url: item.originalUrl,
state: item.state as unknown as ArticleSavingRequestStatus,
content: item.readableContent,
hash: item.textContentHash || '',
isArchived: item.state === LibraryItemState.Archived,
recommendations: item.recommendations?.map(
recommandationDataToRecommendation
),
subscription: item.subscription?.name,
image: item.thumbnail,
})
export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({
...item,
url: item.originalUrl,
state: item.state as unknown as ArticleSavingRequestStatus,
content: item.readableContent,
isArchived: item.state === LibraryItemState.Archived,
pageType: item.itemType as unknown as PageType,
readingProgressPercent: item.readingProgressTopPercent,
contentReader: item.contentReader as unknown as ContentReader,
readingProgressAnchorIndex: item.readingProgressHighestReadAnchor,
subscription: item.subscription?.name,
recommendations: item.recommendations?.map(
recommandationDataToRecommendation
),
image: item.thumbnail,
highlights: item.highlights?.map(highlightDataToHighlight),
})
export const isParsingTimeout = (libraryItem: LibraryItem): boolean => {
return (
// page processed more than 30 seconds ago
page.state === ArticleSavingRequestStatus.Processing &&
new Date(page.savedAt).getTime() < new Date().getTime() - 1000 * 30
libraryItem.state === LibraryItemState.Processing &&
libraryItem.savedAt.getTime() < new Date().getTime() - 1000 * 30
)
}

9
packages/api/src/utils/parser.ts
View File

@ -15,11 +15,10 @@ import { ElementNode } from 'node-html-markdown/dist/nodes'
import { ILike } from 'typeorm'
import { promisify } from 'util'
import { v4 as uuid } from 'uuid'
import { Highlight } from '../elastic/types'
import { getRepository } from '../repository'
import { User } from '../entity/user'
import { Highlight } from '../entity/highlight'
import { env } from '../env'
import { PageType, PreparedDocumentInput } from '../generated/graphql'
import { userRepository } from '../repository/user'
import { ArticleFormat } from '../resolvers/article'
import {
EmbeddedHighlightData,
@ -469,7 +468,7 @@ export const isProbablyArticle = async (
email: string,
subject: string
): Promise<boolean> => {
const user = await getRepository(User).findOneBy({
const user = await userRepository.findOneBy({
email: ILike(email),
})
return !!user || subject.includes(ARTICLE_PREFIX)
@ -655,7 +654,7 @@ export const htmlToHighlightedMarkdown = (
// wrap highlights in special tags
highlights
.filter((h) => h.type == 'HIGHLIGHT' && h.patch)
.filter((h) => h.highlightType == 'HIGHLIGHT' && h.patch)
.forEach((highlight) => {
try {
makeHighlightNodeAttributes(

38
packages/api/src/utils/search.ts
View File

@ -10,6 +10,7 @@ import {
SearchParserTextOffset,
} from 'search-query-parser'
import { LibraryItemType } from '../entity/library_item'
import { InputMaybe, SortParams } from '../generated/graphql'
export enum ReadFilter {
ALL,
@ -32,7 +33,7 @@ export interface SearchFilter {
readFilter: ReadFilter
typeFilter?: LibraryItemType
labelFilters: LabelFilter[]
sortParams?: SortParams
sort?: Sort
hasFilters: HasFilter[]
dateFilters: DateFilter[]
termFilters: FieldFilter[]
@ -67,7 +68,6 @@ export interface DateFilter {
export enum SortBy {
SAVED = 'saved_at',
UPDATED = 'updated_at',
SCORE = '_score',
PUBLISHED = 'published_at',
READ = 'read_at',
LISTENED = 'listened_at',
@ -79,7 +79,7 @@ export enum SortOrder {
DESCENDING = 'DESC',
}
export interface SortParams {
export interface Sort {
by: SortBy
order?: SortOrder
}
@ -179,7 +179,7 @@ const parseLabelFilter = (
}
}
const parseSortParams = (str?: string): SortParams | undefined => {
const parseSort = (str?: string): Sort | undefined => {
if (str === undefined) {
return undefined
}
@ -199,11 +199,6 @@ const parseSortParams = (str?: string): SortParams | undefined => {
by: SortBy.SAVED,
order: sortOrder,
}
case 'SCORE':
// sort by score does not need an order
return {
by: SortBy.SCORE,
}
case 'PUBLISHED':
return {
by: SortBy.PUBLISHED,
@ -417,7 +412,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
break
}
case 'sort':
result.sortParams = parseSortParams(keyword.value)
result.sort = parseSort(keyword.value)
break
case 'has': {
const hasFilter = parseHasFilter(keyword.value)
@ -476,3 +471,26 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
return result
}
export const sortParamsToSort = (
sortParams: InputMaybe<SortParams> | undefined
) => {
const sort = { by: SortBy.UPDATED, order: SortOrder.DESCENDING }
if (sortParams) {
sortParams.order === 'ASCENDING' && (sort.order = SortOrder.ASCENDING)
switch (sortParams.by) {
case 'UPDATED_TIME':
sort.by = SortBy.UPDATED
break
case 'PUBLISHED_AT':
sort.by = SortBy.PUBLISHED
break
case 'SAVED_AT':
sort.by = SortBy.SAVED
break
}
}
return sort
}