diff --git a/Makefile b/Makefile index 1688ff4af..aec0605b2 100644 --- a/Makefile +++ b/Makefile @@ -20,3 +20,15 @@ webview_gen: yarn workspace @omnivore/appreader build cp packages/appreader/build/bundle.js apple/OmnivoreKit/Sources/Views/Resources/bundle.js cp packages/appreader/build/bundle.js android/Omnivore/app/src/main/assets/bundle.js + +api: + yarn workspace @omnivore/api dev + +web: + yarn workspace @omnivore/web dev + +qp: + yarn workspace @omnivore/api dev_qp + +content: + yarn workspace @omnivore/content_fetch start diff --git a/packages/api/package.json b/packages/api/package.json index cea023808..554e99f45 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -56,7 +56,7 @@ "cookie-parser": "^1.4.5", "cors": "^2.8.5", "csv-stringify": "^6.4.0", - "dataloader": "^2.0.0", + "dataloader": "^2.2.2", "diff-match-patch": "^1.0.5", "dompurify": "^2.0.17", "dot-case": "^3.0.4", diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index 68e1c3645..11008c18b 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -15,6 +15,7 @@ import { import { ApolloServer } from 'apollo-server-express' import { ExpressContext } from 'apollo-server-express/dist/ApolloServer' import { ApolloServerPlugin } from 'apollo-server-plugin-base' +import DataLoader from 'dataloader' import { Express } from 'express' import * as httpContext from 'express-http-context2' import type http from 'http' @@ -30,10 +31,14 @@ import { functionResolvers } from './resolvers/function_resolvers' import { ClaimsToSet, RequestContext, ResolverContext } from './resolvers/types' import ScalarResolvers from './scalars' import typeDefs from './schema' +import { batchGetHighlightsFromLibraryItemIds } from './services/highlights' +import { batchGetLabelsFromLibraryItemIds } from './services/labels' +import { batchGetRecommendationsFromLibraryItemIds } from './services/recommendation' import { countDailyServiceUsage, createServiceUsage, } from './services/service_usage' +import { batchGetUploadFilesByIds } from './services/upload_file' import { tracer } from './tracing' import { getClaimsByToken, setAuthInCookie } from './utils/auth' import { SetClaimsRole } from './utils/dictionary' @@ -100,6 +105,14 @@ const contextFunc: ContextFunction = async ({ dataSources: { readingProgress: new ReadingProgressDataSource(), }, + dataLoaders: { + labels: new DataLoader(batchGetLabelsFromLibraryItemIds), + highlights: new DataLoader(batchGetHighlightsFromLibraryItemIds), + recommendations: new DataLoader( + batchGetRecommendationsFromLibraryItemIds + ), + uploadFiles: new DataLoader(batchGetUploadFilesByIds), + }, } return ctx diff --git a/packages/api/src/entity/profile.ts b/packages/api/src/entity/profile.ts index 9fdc96b76..a1a0372ea 100644 --- a/packages/api/src/entity/profile.ts +++ b/packages/api/src/entity/profile.ts @@ -7,7 +7,6 @@ import { PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' - import { User } from './user' @Entity({ name: 'user_profile' }) diff --git a/packages/api/src/entity/recommendation.ts b/packages/api/src/entity/recommendation.ts index e4007a515..0b0d2ae13 100644 --- a/packages/api/src/entity/recommendation.ts +++ b/packages/api/src/entity/recommendation.ts @@ -23,6 +23,9 @@ export class Recommendation { @JoinColumn({ name: 'library_item_id' }) libraryItem!: LibraryItem + @Column('uuid') + libraryItemId!: string + @ManyToOne(() => Group, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'group_id' }) group!: Group diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index cd4b67b31..a632e26a8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2748,6 +2748,7 @@ export type SearchItem = { directionality?: Maybe; feedContent?: Maybe; folder: Scalars['String']; + format?: Maybe; highlights?: Maybe>; id: Scalars['ID']; image?: Maybe; @@ -6617,6 +6618,7 @@ export type SearchItemResolvers, ParentType, ContextType>; feedContent?: Resolver, ParentType, ContextType>; folder?: Resolver; + format?: Resolver, ParentType, ContextType>; highlights?: Resolver>, ParentType, ContextType>; id?: Resolver; image?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index e48fe3cff..815fc9502 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2102,6 +2102,7 @@ type SearchItem { directionality: DirectionalityType feedContent: String folder: String! + format: String highlights: [Highlight!] id: ID! image: String diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 4cb0116d4..fea9139b7 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -7,13 +7,11 @@ import { enqueueProcessYouTubeVideo, enqueueTriggerRuleJob, } from './utils/createTask' -import { buildLogger } from './utils/logger' +import { logger } from './utils/logger' import { isYouTubeVideoURL } from './utils/youtube' export type EntityEvent = { id: string } -const logger = buildLogger('pubsub') - const client = new PubSub() export const createPubSubClient = (): PubsubClient => { diff --git a/packages/api/src/repository/user.ts b/packages/api/src/repository/user.ts index a8f327b0d..d8592222a 100644 --- a/packages/api/src/repository/user.ts +++ b/packages/api/src/repository/user.ts @@ -16,7 +16,13 @@ export const MAX_RECORDS_LIMIT = 1000 export const userRepository = appDataSource.getRepository(User).extend({ findById(id: string) { - return this.findOneBy({ id, status: StatusType.Active }) + return this.createQueryBuilder('user') + .leftJoinAndSelect('user.profile', 'profile') + .where('user.id = :id AND user.status = :status', { + id, + status: StatusType.Active, + }) + .getOne() }, findByEmail(email: string) { diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index f2fbe9c3a..874d235da 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -5,7 +5,6 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ import { Readability } from '@omnivore/readability' import graphqlFields from 'graphql-fields' -import { IsNull } from 'typeorm' import { LibraryItem, LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { @@ -64,7 +63,6 @@ import { libraryItemRepository } from '../../repository/library_item' import { userRepository } from '../../repository/user' import { clearCachedReadingPosition } from '../../services/cached_reading_position' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, createAndSaveLabelsInLibraryItem, @@ -104,7 +102,6 @@ import { userDataToUser, } from '../../utils/helpers' import { - contentConverter, getDistillerResult, htmlToMarkdown, ParsedContentPuppeteer, @@ -385,31 +382,33 @@ export const getArticleResolver = authorized< if (!includeOriginalHtml) { selectColumns.splice(selectColumns.indexOf('originalContent'), 1) } - // We allow the backend to use the ID instead of a slug to fetch the article - // query against id if slug is a uuid - const where = slug.match(/^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/i) - ? { id: slug } - : { slug } - const libraryItem = await authTrx((tx) => - tx.withRepository(libraryItemRepository).findOne({ - select: selectColumns, - where: { - ...where, - deletedAt: IsNull(), - }, - relations: { - highlights: { - user: true, - labels: true, - }, - uploadFile: true, - recommendations: { - recommender: true, - group: true, - }, - }, - }) - ) + + const libraryItem = await authTrx((tx) => { + const qb = tx + .createQueryBuilder(LibraryItem, 'libraryItem') + .select(selectColumns.map((column) => `libraryItem.${column}`)) + .leftJoinAndSelect('libraryItem.labels', 'labels') + .leftJoinAndSelect('libraryItem.highlights', 'highlights') + .leftJoinAndSelect('highlights.labels', 'highlights_labels') + .leftJoinAndSelect('highlights.user', 'highlights_user') + .leftJoinAndSelect('highlights_user.profile', 'highlights_user_profile') + .leftJoinAndSelect('libraryItem.uploadFile', 'uploadFile') + .leftJoinAndSelect('libraryItem.recommendations', 'recommendations') + .leftJoinAndSelect('recommendations.group', 'recommendations_group') + .leftJoinAndSelect( + 'recommendations.recommender', + 'recommendations_recommender' + ) + .where('libraryItem.user_id = :uid', { uid }) + + // We allow the backend to use the ID instead of a slug to fetch the article + // query against id if slug is a uuid + slug.match(/^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/i) + ? qb.andWhere('libraryItem.id = :id', { id: slug }) + : qb.andWhere('libraryItem.slug = :slug', { slug }) + + return qb.andWhere('libraryItem.deleted_at IS NULL').getOne() + }) if (!libraryItem) { return { errorCodes: [ArticleErrorCode.NotFound] } @@ -667,7 +666,7 @@ export const searchResolver = authorized< SearchSuccess, SearchError, QuerySearchArgs ->(async (_obj, params, { log, uid }) => { +>(async (_obj, params, { uid }) => { const startCursor = params.after || '' const first = Math.min(params.first || 10, 100) // limit to 100 items @@ -699,38 +698,11 @@ export const searchResolver = authorized< libraryItems.pop() } - const edges = await Promise.all( - libraryItems.map(async (libraryItem) => { - libraryItem.highlights = await findHighlightsByLibraryItemId( - libraryItem.id, - uid - ) - - if (params.includeContent && libraryItem.readableContent) { - // convert html to the requested format - const format = params.format || ArticleFormat.Html - try { - const converter = contentConverter(format) - if (converter) { - libraryItem.readableContent = converter( - libraryItem.readableContent, - libraryItem.highlights - ) - } - } catch (error) { - log.error('Error converting content', error) - } - } - - return { - node: libraryItemToSearchItem(libraryItem), - cursor: endCursor, - } - }) - ) - return { - edges, + edges: libraryItems.map((item) => ({ + node: libraryItemToSearchItem(item, params.format as ArticleFormat), + cursor: endCursor, + })), pageInfo: { hasPreviousPage: false, startCursor, diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index d65d054c8..b4ace7f7b 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -4,6 +4,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { createHmac } from 'crypto' +import { Highlight as HighlightEntity } from '../entity/highlight' import { EXISTING_NEWSLETTER_FOLDER, NewsletterEmail, @@ -24,10 +25,6 @@ import { } from '../generated/graphql' import { getAISummary } from '../services/ai-summaries' import { findUserFeatures } from '../services/features' -import { findHighlightsByLibraryItemId } from '../services/highlights' -import { findLabelsByLibraryItemId } from '../services/labels' -import { findRecommendationsByLibraryItemId } from '../services/recommendation' -import { findUploadFileById } from '../services/upload_file' import { highlightDataToHighlight, isBase64Image, @@ -36,11 +33,16 @@ import { wordsCount, } from '../utils/helpers' import { createImageProxyUrl } from '../utils/imageproxy' +import { contentConverter } from '../utils/parser' import { generateDownloadSignedUrl, generateUploadFilePathName, } from '../utils/uploads' -import { emptyTrashResolver, fetchContentResolver } from './article' +import { + ArticleFormat, + emptyTrashResolver, + fetchContentResolver, +} from './article' import { addDiscoverFeedResolver, deleteDiscoverArticleResolver, @@ -404,7 +406,9 @@ export const functionResolvers = { ctx.claims && article.uploadFileId ) { - const upload = await findUploadFileById(article.uploadFileId) + const upload = await ctx.dataLoaders.uploadFiles.load( + article.uploadFileId + ) if (!upload || !upload.fileName) { return undefined } @@ -439,7 +443,7 @@ export const functionResolvers = { ) { if (article.labels) return article.labels - return findLabelsByLibraryItemId(article.id, ctx.uid) + return ctx.dataLoaders.labels.load(article.id) }, ...readingProgressHandlers, }, @@ -478,7 +482,7 @@ export const functionResolvers = { ctx.claims && item.uploadFileId ) { - const upload = await findUploadFileById(item.uploadFileId) + const upload = await ctx.dataLoaders.uploadFiles.load(item.uploadFileId) if (!upload || !upload.fileName) { return undefined } @@ -511,28 +515,22 @@ export const functionResolvers = { ) { if (item.labels) return item.labels - return findLabelsByLibraryItemId(item.id, ctx.uid) + return ctx.dataLoaders.labels.load(item.id) }, async recommendations( item: { id: string recommendations?: Recommendation[] - recommenderNames?: string[] | null }, _: unknown, ctx: WithDataSourcesContext ) { if (item.recommendations) return item.recommendations - if (item.recommenderNames && item.recommenderNames.length > 0) { - const recommendations = await findRecommendationsByLibraryItemId( - item.id, - ctx.uid - ) - return recommendations.map(recommandationDataToRecommendation) - } - - return [] + const recommendations = await ctx.dataLoaders.recommendations.load( + item.id + ) + return recommendations.map(recommandationDataToRecommendation) }, async aiSummary(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { return ( @@ -553,10 +551,46 @@ export const functionResolvers = { ) { if (item.highlights) return item.highlights - const highlights = await findHighlightsByLibraryItemId(item.id, ctx.uid) + const highlights = await ctx.dataLoaders.highlights.load(item.id) return highlights.map(highlightDataToHighlight) }, ...readingProgressHandlers, + async content( + item: { + id: string + content?: string + highlightAnnotations?: string[] + format?: ArticleFormat + }, + _: unknown, + ctx: WithDataSourcesContext + ) { + // convert html to the requested format if requested + if (item.format && item.format !== ArticleFormat.Html && item.content) { + let highlights: HighlightEntity[] = [] + // load highlights if needed + if ( + item.format === ArticleFormat.HighlightedMarkdown && + item.highlightAnnotations?.length + ) { + highlights = await ctx.dataLoaders.highlights.load(item.id) + } + + try { + ctx.log.info(`Converting content to: ${item.format}`) + + // convert html to the requested format + const converter = contentConverter(item.format) + if (converter) { + return converter(item.content, highlights) + } + } catch (error) { + ctx.log.error('Error converting content', error) + } + } + + return item.content + }, }, Subscription: { newsletterEmail(subscription: Subscription) { diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 6fc24405f..f880a0145 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -1,11 +1,16 @@ /* eslint-disable @typescript-eslint/ban-types */ import { Span } from '@opentelemetry/api' import { Context as ApolloContext } from 'apollo-server-core' +import DataLoader from 'dataloader' import * as jwt from 'jsonwebtoken' import { EntityManager } from 'typeorm' import winston from 'winston' -import { PubsubClient } from '../pubsub' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' +import { Highlight } from '../entity/highlight' +import { Label } from '../entity/label' +import { Recommendation } from '../entity/recommendation' +import { UploadFile } from '../entity/upload_file' +import { PubsubClient } from '../pubsub' export interface Claims { uid: string @@ -41,6 +46,12 @@ export interface RequestContext { dataSources: { readingProgress: ReadingProgressDataSource } + dataLoaders: { + labels: DataLoader + highlights: DataLoader + recommendations: DataLoader + uploadFiles: DataLoader + } } export type ResolverContext = ApolloContext diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index acae6f520..8d112d509 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1657,6 +1657,7 @@ const schema = gql` folder: String! aiSummary: String directionality: DirectionalityType + format: String } type SearchItemEdge { diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 3893bdbc5..dd6685362 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -1,5 +1,5 @@ import { diff_match_patch } from 'diff-match-patch' -import { DeepPartial } from 'typeorm' +import { DeepPartial, In } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' import { EntityLabel } from '../entity/entity_label' import { Highlight } from '../entity/highlight' @@ -20,6 +20,21 @@ export type HighlightEvent = Merge< EntityEvent > +export const batchGetHighlightsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const highlights = await authTrx(async (tx) => + tx.getRepository(Highlight).find({ + where: { libraryItem: { id: In(libraryItemIds as string[]) } }, + relations: ['user'], + }) + ) + + return libraryItemIds.map((libraryItemId) => + highlights.filter((highlight) => highlight.libraryItemId === libraryItemId) + ) +} + export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() const patches = dmp.patch_fromText(patch) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 21b66fa4c..e3400520e 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -22,20 +22,22 @@ export type LabelEvent = Merge< EntityEvent > -// const batchGetLabelsFromLinkIds = async ( -// linkIds: readonly string[] -// ): Promise => { -// const links = await getRepository(Link).find({ -// where: { id: In(linkIds as string[]) }, -// relations: ['labels'], -// }) +export const batchGetLabelsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const labels = await authTrx(async (tx) => + tx.getRepository(EntityLabel).find({ + where: { libraryItemId: In(libraryItemIds as string[]) }, + relations: ['label'], + }) + ) -// return linkIds.map( -// (linkId) => links.find((link) => link.id === linkId)?.labels || [] -// ) -// } - -// export const labelsLoader = new DataLoader(batchGetLabelsFromLinkIds) + return libraryItemIds.map((libraryItemId) => + labels + .filter((label) => label.libraryItemId === libraryItemId) + .map((label) => label.label) + ) +} export const findOrCreateLabels = async ( labels: CreateLabelInput[], diff --git a/packages/api/src/services/recommendation.ts b/packages/api/src/services/recommendation.ts index 72b6ff0bd..9e58224f4 100644 --- a/packages/api/src/services/recommendation.ts +++ b/packages/api/src/services/recommendation.ts @@ -1,5 +1,5 @@ import { nanoid } from 'nanoid' -import { DeepPartial } from 'typeorm' +import { DeepPartial, In } from 'typeorm' import { LibraryItem } from '../entity/library_item' import { Recommendation } from '../entity/recommendation' import { authTrx } from '../repository' @@ -12,6 +12,23 @@ import { updateLibraryItem, } from './library_item' +export const batchGetRecommendationsFromLibraryItemIds = async ( + libraryItemIds: readonly string[] +): Promise => { + const recommendations = await authTrx(async (tx) => + tx.getRepository(Recommendation).find({ + where: { libraryItem: { id: In(libraryItemIds as string[]) } }, + relations: ['group', 'recommender'], + }) + ) + + return libraryItemIds.map((libraryItemId) => + recommendations.filter( + (recommendation) => recommendation.libraryItemId === libraryItemId + ) + ) +} + export const addRecommendation = async ( item: LibraryItem, recommendation: Recommendation, diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index 148614352..fd96f313c 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -1,5 +1,7 @@ import normalizeUrl from 'normalize-url' import path from 'path' +import { In } from 'typeorm' +import { v4 as uuid } from 'uuid' import { LibraryItemState } from '../entity/library_item' import { UploadFile } from '../entity/upload_file' import { @@ -18,7 +20,16 @@ import { } from '../utils/uploads' import { validateUrl } from './create_page_save_request' import { createOrUpdateLibraryItem } from './library_item' -import { v4 as uuid } from 'uuid' + +export const batchGetUploadFilesByIds = async ( + ids: readonly string[] +): Promise<(UploadFile | undefined)[]> => { + const uploadFiles = await getRepository(UploadFile).findBy({ + id: In(ids as string[]), + }) + + return ids.map((id) => uploadFiles.find((uploadFile) => uploadFile.id === id)) +} const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) diff --git a/packages/api/src/utils/auth.ts b/packages/api/src/utils/auth.ts index c3269aec8..2dbc56d1f 100644 --- a/packages/api/src/utils/auth.ts +++ b/packages/api/src/utils/auth.ts @@ -36,12 +36,13 @@ export const claimsFromApiKey = async (key: string): Promise => { const apiKeyRepo = getRepository(ApiKey) - const apiKey = await apiKeyRepo.findOne({ - where: { + const apiKey = await apiKeyRepo + .createQueryBuilder('apiKey') + .innerJoinAndSelect('apiKey.user', 'user') + .where({ key: hashedKey, - }, - relations: ['user'], - }) + }) + .getOne() if (!apiKey) { throw new Error('api key not found') } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 785e5d5ef..85abc5467 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -26,6 +26,7 @@ import { SearchItem, } from '../generated/graphql' import { createPubSubClient } from '../pubsub' +import { ArticleFormat } from '../resolvers' import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' @@ -230,7 +231,10 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ directionality: item.directionality as unknown as DirectionalityType, }) -export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ +export const libraryItemToSearchItem = ( + item: LibraryItem, + format?: ArticleFormat +): SearchItem => ({ ...item, url: item.originalUrl, state: item.state as unknown as ArticleSavingRequestStatus, @@ -247,6 +251,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ highlights: item.highlights?.map(highlightDataToHighlight), wordsCount: item.wordCount, directionality: item.directionality as unknown as DirectionalityType, + format, }) export const isParsingTimeout = (libraryItem: LibraryItem): boolean => { diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 364cfffa3..50606a069 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -31,7 +31,7 @@ export class CustomTypeOrmLogger constructor(options?: TypeOrmLoggerOptions) { super(options) - this.logger = buildLogger('typeorm') + this.logger = logger } logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { @@ -132,8 +132,8 @@ const truncateObjectDeep = (object: any, length: number): any => { class GcpLoggingTransport extends LoggingWinston { log(info: any, callback: (err: Error | null, apiResponse?: any) => void) { - // reduce the size of the log entry by truncating any string values to 500 characters - info = truncateObjectDeep(info, 500) as never + // reduce the size of the log entry by truncating any string values to 10000 characters + info = truncateObjectDeep(info, 10000) as never super.log(info, callback) } } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 8107d978a..a9ddd4bb4 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -33,7 +33,7 @@ import { makeHighlightNodeAttributes, } from './highlightGenerator' import { createImageProxyUrl } from './imageproxy' -import { buildLogger, LogRecord } from './logger' +import { logger, LogRecord } from './logger' interface Feed { title: string @@ -43,7 +43,6 @@ interface Feed { description?: string } -const logger = buildLogger('utils.parse') const signToken = promisify(jwt.sign) const axiosInstance = axios.create({ @@ -647,7 +646,6 @@ export const contentConverter = ( return htmlToMarkdown case ArticleFormat.HighlightedMarkdown: return htmlToHighlightedMarkdown - case ArticleFormat.Html: default: return undefined } @@ -671,7 +669,7 @@ export const htmlToHighlightedMarkdown = ( throw new Error('Invalid html content') } } catch (err) { - logger.info(err) + logger.error(err) return nhm.translate(/* html */ html) } @@ -691,7 +689,7 @@ export const htmlToHighlightedMarkdown = ( articleTextNodes ) } catch (err) { - logger.info(err) + logger.error(err) } }) html = document.documentElement.outerHTML diff --git a/yarn.lock b/yarn.lock index 9c651f55f..08b76b456 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13348,11 +13348,16 @@ data-urls@^2.0.0: whatwg-mimetype "^2.3.0" whatwg-url "^8.0.0" -dataloader@2.1.0, dataloader@^2.0.0: +dataloader@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/dataloader/-/dataloader-2.1.0.tgz#c69c538235e85e7ac6c6c444bae8ecabf5de9df7" integrity sha512-qTcEYLen3r7ojZNgVUaRggOI+KM7jrKxXeSHhogh/TWxYMeONEMqY+hmkobiYQozsGIyg9OYVzO4ZIfoB4I0pQ== +dataloader@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/dataloader/-/dataloader-2.2.2.tgz#216dc509b5abe39d43a9b9d97e6e5e473dfbe3e0" + integrity sha512-8YnDaaf7N3k/q5HnTJVuzSyLETjoZjVmHc4AeKAzOvKHEFQKcn64OKBfzHYtE9zGjctNM7V9I0MfnUVLpi7M5g== + date-fns@2.x: version "2.29.3" resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.3.tgz#27402d2fc67eb442b511b70bbdf98e6411cd68a8"