From a4e707075fbd44713093a38bf12d558f954fb535 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sat, 20 Jan 2024 21:07:02 +0800 Subject: [PATCH] Refactor savePage a bit to use it as a service from the jobs --- packages/api/src/jobs/save_page.ts | 76 +++---------------- packages/api/src/resolvers/article/index.ts | 2 +- .../resolvers/article_saving_request/index.ts | 3 +- packages/api/src/resolvers/highlight/index.ts | 3 +- .../src/resolvers/recommendations/index.ts | 3 +- .../api/src/resolvers/subscriptions/index.ts | 7 +- packages/api/src/resolvers/update/index.ts | 3 +- .../api/src/resolvers/upload_files/index.ts | 4 +- packages/api/src/resolvers/user/index.ts | 3 +- packages/api/src/services/save_page.ts | 37 ++++++--- 10 files changed, 53 insertions(+), 88 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 2fa2d0daf..cdeddc969 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -3,10 +3,10 @@ import jwt from 'jsonwebtoken' import { promisify } from 'util' import { env } from '../env' import { redisDataSource } from '../redis_data_source' -import { savePage } from '../services/save_page' +import { savePage, stringToRequestStatus } from '../services/save_page' import { userRepository } from '../repository/user' -import { ArticleSavingRequestStatus, ParseResult } from '../generated/graphql' import { logger } from '../utils/logger' +import { Readability } from '@omnivore/readability' const signToken = promisify(jwt.sign) @@ -69,7 +69,7 @@ interface FetchResult { title: string content?: string contentType?: string - readabilityResult?: unknown + readabilityResult?: Readability.ParseResult } const isFetchResult = (obj: unknown): obj is FetchResult => { @@ -238,60 +238,6 @@ const sendCreateArticleMutation = async (userId: string, input: unknown) => { } } -const sendSavePageMutation = async (userId: string, input: unknown) => { - const data = JSON.stringify({ - query: `mutation SavePage ($input: SavePageInput!){ - savePage(input:$input){ - ... on SaveSuccess{ - url - clientRequestId - } - ... on SaveError{ - errorCodes - } - } - }`, - variables: { - input, - }, - }) - - const auth = await signToken({ uid: userId }, JWT_SECRET) - try { - const response = await axios.post( - `${REST_BACKEND_ENDPOINT}/graphql`, - data, - { - headers: { - Cookie: `auth=${auth as string};`, - 'Content-Type': 'application/json', - }, - timeout: REQUEST_TIMEOUT, - } - ) - - if ( - response.data.data.savePage.errorCodes && - response.data.data.savePage.errorCodes.length > 0 - ) { - console.error( - 'error while saving page', - response.data.data.savePage.errorCodes[0] - ) - if (response.data.data.savePage.errorCodes[0] === 'UNAUTHORIZED') { - return { error: 'UNAUTHORIZED' } - } - - return null - } - - return response.data.data.savePage - } catch (error) { - console.error('error saving page', error) - return null - } -} - const sendImportStatusUpdate = async ( userId: string, taskId: string, @@ -409,26 +355,26 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { clientRequestId: articleSavingRequestId, title, originalContent: content, - parseResult: readabilityResult as ParseResult, - state: state as ArticleSavingRequestStatus, + parseResult: readabilityResult, + state: stringToRequestStatus(state), labels: labels?.map((name) => { return { name, } }), rssFeedUrl, - savedAt, - publishedAt, + savedAt: savedAt ? new Date(savedAt) : new Date(), + publishedAt: publishedAt ? new Date(publishedAt) : null, source, folder, }, user ) - if (result.__typename == 'SaveError') { - logger.error('Error saving page', { userId, url, result }) - throw new Error('Error saving page') - } + // if (result.__typename == 'SaveError') { + // logger.error('Error saving page', { userId, url, result }) + // throw new Error('Error saving page') + // } // if the readability result is not parsed, the import is failed isImported = !!readabilityResult diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 08f0401a1..03cce990e 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -93,7 +93,6 @@ import { traceAs } from '../../tracing' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' import { - authorized, cleanUrl, errorHandler, generateSlug, @@ -103,6 +102,7 @@ import { titleForFilePath, userDataToUser, } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' import { contentConverter, getDistillerResult, diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 731de1c72..9f4cf7c4a 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -19,11 +19,12 @@ import { } from '../../services/library_item' import { analytics } from '../../utils/analytics' import { - authorized, cleanUrl, isParsingTimeout, libraryItemToArticleSavingRequest, } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' + import { isErrorWithCode } from '../user' export const createArticleSavingRequestResolver = authorized< diff --git a/packages/api/src/resolvers/highlight/index.ts b/packages/api/src/resolvers/highlight/index.ts index 82618990c..9853ef7e4 100644 --- a/packages/api/src/resolvers/highlight/index.ts +++ b/packages/api/src/resolvers/highlight/index.ts @@ -34,7 +34,8 @@ import { updateHighlight, } from '../../services/highlights' import { analytics } from '../../utils/analytics' -import { authorized, highlightDataToHighlight } from '../../utils/helpers' +import { highlightDataToHighlight } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' export const createHighlightResolver = authorized< CreateHighlightSuccess, diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index 31f7da898..a23a573e2 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -40,7 +40,8 @@ import { import { findLibraryItemById } from '../../services/library_item' import { analytics } from '../../utils/analytics' import { enqueueRecommendation } from '../../utils/createTask' -import { authorized, userDataToUser } from '../../utils/helpers' +import { userDataToUser } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' export const createGroupResolver = authorized< CreateGroupSuccess, diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 2b0e0b12f..b7e1507e4 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -44,11 +44,8 @@ import { unsubscribe } from '../../services/subscriptions' import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { enqueueRssFeedFetch } from '../../utils/createTask' -import { - authorized, - getAbsoluteUrl, - keysToCamelCase, -} from '../../utils/helpers' +import { getAbsoluteUrl, keysToCamelCase } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' import { parseFeed, parseOpml, RSS_PARSER_CONFIG } from '../../utils/parser' import { updateSubscription } from '../../services/update_subscription' diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 255efd2b5..c38019d8b 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -5,7 +5,8 @@ import { UpdatePageSuccess, } from '../../generated/graphql' import { updateLibraryItem } from '../../services/library_item' -import { authorized, libraryItemToArticle } from '../../utils/helpers' +import { libraryItemToArticle } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' export const updatePageResolver = authorized< UpdatePageSuccess, diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 9ea214584..b22395458 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -19,7 +19,9 @@ import { updateLibraryItem, } from '../../services/library_item' import { analytics } from '../../utils/analytics' -import { authorized, generateSlug } from '../../utils/helpers' +import { generateSlug } from '../../utils/helpers' +import { authorized } from '../../utils/gql-utils' + import { contentReaderForLibraryItem, generateUploadFilePathName, diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index c15000a66..8303d41ea 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -43,9 +43,10 @@ import { userRepository } from '../../repository/user' import { createUser } from '../../services/create_user' import { sendVerificationEmail } from '../../services/send_emails' import { softDeleteUser } from '../../services/user' -import { authorized, userDataToUser } from '../../utils/helpers' +import { userDataToUser } from '../../utils/helpers' import { validateUsername } from '../../utils/usernamePolicy' import { WithDataSourcesContext } from '../types' +import { authorized } from '../../utils/gql-utils' export const updateUserResolver = authorized< UpdateUserSuccess, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 85f341df2..b03608dcd 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -5,14 +5,12 @@ import { Highlight } from '../entity/highlight' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { User } from '../entity/user' import { homePageURL } from '../env' -import { - ArticleSavingRequestStatus, - Maybe, - PreparedDocumentInput, - SaveErrorCode, - SavePageInput, - SaveResult, -} from '../generated/graphql' +// import { +// ArticleSavingRequestStatus, +// PreparedDocumentInput, +// SaveErrorCode, +// SaveResult, +// } from '../generated/graphql' import { authTrx } from '../repository' import { enqueueThumbnailTask } from '../utils/createTask' import { @@ -30,6 +28,14 @@ import { createPageSaveRequest } from './create_page_save_request' import { createHighlight } from './highlights' import { createAndSaveLabelsInLibraryItem } from './labels' import { createLibraryItem, updateLibraryItem } from './library_item' +import { CreateLabelInput } from '../repository/label' +import { + ArticleSavingRequestStatus, + PreparedDocumentInput, + SaveErrorCode, + SavePageInput, + SaveResult, +} from '../generated/graphql' // where we can use APIs to fetch their underlying content. const FORCE_PUPPETEER_URLS = [ @@ -43,7 +49,7 @@ const ALREADY_PARSED_SOURCES = [ 'pocket', ] -const createSlug = (url: string, title?: Maybe | undefined) => { +const createSlug = (url: string, title?: string | null | undefined) => { const { pathname } = new URL(url) const croppedPathname = decodeURIComponent( pathname @@ -63,6 +69,15 @@ const shouldParseInBackend = (input: SavePageInput): boolean => { ) } +export const stringToRequestStatus = ( + str: string | undefined | null +): ArticleSavingRequestStatus | null => { + if (str && str in ArticleSavingRequestStatus) { + return str as ArticleSavingRequestStatus + } + return null +} + export const savePage = async ( input: SavePageInput, user: User @@ -95,7 +110,7 @@ export const savePage = async ( canonicalUrl: parseResult.canonicalUrl, savedAt: input.savedAt ? new Date(input.savedAt) : new Date(), publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, - state: input.state || undefined, + state: stringToRequestStatus(input.state) || undefined, rssFeedUrl: input.rssFeedUrl, folder: input.folder, }) @@ -109,7 +124,7 @@ export const savePage = async ( userId: user.id, url: itemToSave.originalUrl, articleSavingRequestId: clientRequestId || undefined, - state: input.state || undefined, + state: stringToRequestStatus(input.state) || undefined, labels: input.labels || undefined, folder: input.folder || undefined, })