From ac5acf7f21735e3590dc0d664377453b7039aff4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 22:34:55 +0800 Subject: [PATCH 1/4] fix: add rss label and feed url to the pdf in rss feed item --- packages/api/src/generated/graphql.ts | 3 + packages/api/src/generated/schema.graphql | 3 + packages/api/src/jobs/save_page.ts | 117 ++++++---------------- packages/api/src/schema.ts | 3 + packages/api/src/services/save_file.ts | 7 +- 5 files changed, 44 insertions(+), 89 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 33ecceb9b..9d908a941 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2283,8 +2283,11 @@ export type SaveFileInput = { clientRequestId: Scalars['ID']; folder?: InputMaybe; labels?: InputMaybe>; + publishedAt?: InputMaybe; + savedAt?: InputMaybe; source: Scalars['String']; state?: InputMaybe; + subscription?: InputMaybe; uploadFileId: Scalars['ID']; url: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 203d7b9bd..2df4f681e 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1718,8 +1718,11 @@ input SaveFileInput { clientRequestId: ID! folder: String labels: [CreateLabelInput!] + publishedAt: Date + savedAt: Date source: String! state: ArticleSavingRequestStatus + subscription: String uploadFileId: ID! url: String! } diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 0e87eb30c..1eebf1adb 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -8,6 +8,7 @@ import { } from '../generated/graphql' import { redisDataSource } from '../redis_data_source' import { userRepository } from '../repository/user' +import { saveFile } from '../services/save_file' import { savePage } from '../services/save_page' import { logger } from '../utils/logger' @@ -46,17 +47,6 @@ interface UploadFileResponse { } } -interface CreateArticleResponse { - data: { - createArticle: { - createdArticle: { - id: string - } - errorCodes: string[] - } - } -} - interface FetchResult { finalUrl: string title?: string @@ -174,57 +164,6 @@ const uploadPdf = async ( return uploadResult.id } -const sendCreateArticleMutation = async (userId: string, input: unknown) => { - const data = JSON.stringify({ - query: `mutation CreateArticle ($input: CreateArticleInput!){ - createArticle(input:$input){ - ... on CreateArticleSuccess{ - createdArticle{ - id - } - } - ... on CreateArticleError{ - errorCodes - } - } - }`, - variables: { - input, - }, - }) - - const auth = await signToken({ uid: userId }, JWT_SECRET) - try { - const response = await axios.post( - `${REST_BACKEND_ENDPOINT}/graphql`, - data, - { - headers: { - Cookie: `auth=${auth as string};`, - 'Content-Type': 'application/json', - }, - timeout: REQUEST_TIMEOUT, - } - ) - - if ( - response.data.data.createArticle.errorCodes && - response.data.data.createArticle.errorCodes.length > 0 - ) { - console.error( - 'error while creating article', - response.data.data.createArticle.errorCodes[0] - ) - return null - } - - return response.data.data.createArticle - } catch (error) { - console.error('error creating article', error) - return null - } -} - const sendImportStatusUpdate = async ( userId: string, taskId: string, @@ -298,25 +237,37 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { const { title, contentType } = fetchedResult let content = fetchedResult.content + const user = await userRepository.findById(userId) + if (!user) { + logger.error('Unable to save job, user can not be found.', { + userId, + url, + }) + // if the user is not found, we do not retry + return false + } + // for pdf content, we need to upload the pdf if (contentType === 'application/pdf') { - const encodedUrl = encodeURI(url) - const uploadFileId = await uploadPdf(url, userId, articleSavingRequestId) - const uploadedPdf = await sendCreateArticleMutation(userId, { - url: encodedUrl, - articleSavingRequestId, - uploadFileId, - state, - labels, - source, - folder, - rssFeedUrl, - savedAt, - publishedAt, - }) - if (!uploadedPdf) { - throw new Error('error while saving uploaded pdf') + + const result = await saveFile( + { + url, + uploadFileId, + state: state ? (state as ArticleSavingRequestStatus) : undefined, + labels, + source, + folder, + subscription: rssFeedUrl, + savedAt, + publishedAt, + clientRequestId: articleSavingRequestId, + }, + user + ) + if (result.__typename == 'SaveError') { + throw new Error(result.message || result.errorCodes[0]) } isSaved = true @@ -331,16 +282,6 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { state = ArticleSavingRequestStatus.Failed } - const user = await userRepository.findById(userId) - if (!user) { - logger.error('Unable to save job, user can not be found.', { - userId, - url, - }) - // if the user is not found, we do not retry - return false - } - // for non-pdf content, we need to save the page const result = await savePage( { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 877b4bfb4..d18b91012 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -545,6 +545,9 @@ const schema = gql` state: ArticleSavingRequestStatus labels: [CreateLabelInput!] folder: String + savedAt: Date + publishedAt: Date + subscription: String } input ParseResult { diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index e6c58148b..dd727ec38 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -14,6 +14,7 @@ export const saveFile = async ( const uploadFile = await findUploadFileById(input.uploadFileId) if (!uploadFile) { return { + __typename: 'SaveError', errorCodes: [SaveErrorCode.Unauthorized], } } @@ -24,6 +25,7 @@ export const saveFile = async ( if (!uploadFileData) { return { + __typename: 'SaveError', errorCodes: [SaveErrorCode.Unknown], } } @@ -34,6 +36,8 @@ export const saveFile = async ( { state: (input.state as unknown as LibraryItemState) || undefined, folder: input.folder || undefined, + savedAt: input.savedAt || undefined, + publishedAt: input.publishedAt || undefined, }, user.id ) @@ -43,7 +47,8 @@ export const saveFile = async ( await createAndSaveLabelsInLibraryItem( input.clientRequestId, user.id, - input.labels + input.labels, + input.subscription ) return { From a2bdddf8c9e8b572a4d9cda6a357af2969ac8c8e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 12:30:31 +0800 Subject: [PATCH 2/4] fix tests --- packages/api/src/services/save_file.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index dd727ec38..c0e2a5e75 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -36,8 +36,10 @@ export const saveFile = async ( { state: (input.state as unknown as LibraryItemState) || undefined, folder: input.folder || undefined, - savedAt: input.savedAt || undefined, - publishedAt: input.publishedAt || undefined, + savedAt: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt + ? new Date(input.publishedAt) + : undefined, }, user.id ) From 4f061546bfbfb3e4f94bb79027d32dfb985a15aa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 12:55:19 +0800 Subject: [PATCH 3/4] remove api call to upload file request --- packages/api/src/jobs/save_page.ts | 78 ++------- packages/api/src/resolvers/article/index.ts | 4 +- .../api/src/resolvers/upload_files/index.ts | 151 +---------------- packages/api/src/services/upload_file.ts | 155 ++++++++++++++++++ 4 files changed, 172 insertions(+), 216 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 1eebf1adb..82d5b246a 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -10,13 +10,13 @@ import { redisDataSource } from '../redis_data_source' import { userRepository } from '../repository/user' import { saveFile } from '../services/save_file' import { savePage } from '../services/save_page' +import { uploadFile } from '../services/upload_file' import { logger } from '../utils/logger' const signToken = promisify(jwt.sign) const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl const JWT_SECRET = env.server.jwtSecret -const REST_BACKEND_ENDPOINT = `${env.server.internalApiUrl}/api` const MAX_ATTEMPTS = 2 const REQUEST_TIMEOUT = 30000 // 30 seconds @@ -82,86 +82,32 @@ const uploadToSignedUrl = async ( } } -const getUploadIdAndSignedUrl = async ( - userId: string, - url: string, - articleSavingRequestId: string -) => { - const auth = await signToken({ uid: userId }, JWT_SECRET) - const data = JSON.stringify({ - query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { - uploadFileRequest(input:$input) { - ... on UploadFileRequestError { - errorCodes - } - ... on UploadFileRequestSuccess { - id - uploadSignedUrl - } - } - }`, - variables: { - input: { - url: encodeURI(url), - contentType: 'application/pdf', - clientRequestId: articleSavingRequestId, - }, - }, - }) - - try { - const response = await axios.post( - `${REST_BACKEND_ENDPOINT}/graphql`, - data, - { - headers: { - Cookie: `auth=${auth as string};`, - 'Content-Type': 'application/json', - }, - timeout: REQUEST_TIMEOUT, - } - ) - - if ( - response.data.data.uploadFileRequest.errorCodes && - response.data.data.uploadFileRequest.errorCodes?.length > 0 - ) { - console.error( - 'Error while getting upload id and signed url', - response.data.data.uploadFileRequest.errorCodes[0] - ) - return null - } - - return response.data.data.uploadFileRequest - } catch (e) { - console.error('error getting upload id and signed url', e) - return null - } -} - const uploadPdf = async ( url: string, userId: string, articleSavingRequestId: string ) => { - const uploadResult = await getUploadIdAndSignedUrl( - userId, - url, - articleSavingRequestId + const result = await uploadFile( + { + url, + contentType: 'application/pdf', + clientRequestId: articleSavingRequestId, + }, + userId ) - if (!uploadResult) { + if (!result.uploadSignedUrl) { throw new Error('error while getting upload id and signed url') } + const uploaded = await uploadToSignedUrl( - uploadResult.uploadSignedUrl, + result.uploadSignedUrl, 'application/pdf', url ) if (!uploaded) { throw new Error('error while uploading pdf') } - return uploadResult.id + return result.id } const sendImportStatusUpdate = async ( diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 03cce990e..80990d2f7 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -87,11 +87,13 @@ import { import { parsedContentToLibraryItem } from '../../services/save_page' import { findUploadFileById, + itemTypeForContentType, setFileUploadComplete, } from '../../services/upload_file' import { traceAs } from '../../tracing' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' +import { authorized } from '../../utils/gql-utils' import { cleanUrl, errorHandler, @@ -102,7 +104,6 @@ import { titleForFilePath, userDataToUser, } from '../../utils/helpers' -import { authorized } from '../../utils/gql-utils' import { contentConverter, getDistillerResult, @@ -111,7 +112,6 @@ import { parsePreparedContent, } from '../../utils/parser' import { getStorageFileDetails } from '../../utils/uploads' -import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { Markdown = 'markdown', diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index b22395458..74caee344 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -1,55 +1,17 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import normalizeUrl from 'normalize-url' -import path from 'path' -import { LibraryItemState } from '../../entity/library_item' -import { UploadFile } from '../../entity/upload_file' import { env } from '../../env' import { MutationUploadFileRequestArgs, - PageType, UploadFileRequestError, - UploadFileRequestErrorCode, UploadFileRequestSuccess, - UploadFileStatus, } from '../../generated/graphql' -import { validateUrl } from '../../services/create_page_save_request' -import { - createLibraryItem, - findLibraryItemByUrl, - updateLibraryItem, -} from '../../services/library_item' +import { uploadFile } from '../../services/upload_file' import { analytics } from '../../utils/analytics' -import { generateSlug } from '../../utils/helpers' import { authorized } from '../../utils/gql-utils' - -import { - contentReaderForLibraryItem, - generateUploadFilePathName, - generateUploadSignedUrl, -} from '../../utils/uploads' - -const isFileUrl = (url: string): boolean => { - const parsedUrl = new URL(url) - return parsedUrl.protocol == 'file:' -} - -export const itemTypeForContentType = (contentType: string) => { - if (contentType == 'application/epub+zip') { - return PageType.Book - } - return PageType.File -} - export const uploadFileRequestResolver = authorized< UploadFileRequestSuccess, UploadFileRequestError, MutationUploadFileRequestArgs ->(async (_, { input }, ctx) => { - const { authTrx, uid, log } = ctx - let uploadFileData: { id: string | null } = { - id: null, - } - +>(async (_, { input }, { uid }) => { analytics.track({ userId: uid, event: 'file_upload_request', @@ -59,112 +21,5 @@ export const uploadFileRequestResolver = authorized< }, }) - let title: string - let fileName: string - try { - const url = normalizeUrl(new URL(input.url).href, { - stripHash: true, - stripWWW: false, - }) - title = decodeURI(path.basename(new URL(url).pathname, '.pdf')) - fileName = decodeURI(path.basename(new URL(url).pathname)).replace( - /[^a-zA-Z0-9-_.]/g, - '' - ) - - if (!fileName) { - fileName = 'content.pdf' - } - - if (!isFileUrl(url)) { - try { - validateUrl(url) - } catch (error) { - log.info('illegal file input url', error) - return { - errorCodes: [UploadFileRequestErrorCode.BadInput], - } - } - } - } catch { - return { errorCodes: [UploadFileRequestErrorCode.BadInput] } - } - - uploadFileData = await authTrx((t) => - t.getRepository(UploadFile).save({ - url: input.url, - user: { id: uid }, - fileName, - status: UploadFileStatus.Initialized, - contentType: input.contentType, - }) - ) - - if (uploadFileData.id) { - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName( - uploadFileId, - fileName - ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - input.contentType - ) - - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) - }) - } - - let createdItemId: string | undefined = undefined - if (input.createPageEntry) { - // If we have a file:// URL, don't try to match it - // and create a copy of the item, just create a - // new item. - const item = await findLibraryItemByUrl(input.url, uid) - if (item) { - await updateLibraryItem( - item.id, - { - state: LibraryItemState.Processing, - }, - uid - ) - createdItemId = item.id - } else { - const itemType = itemTypeForContentType(input.contentType) - const uploadFileId = uploadFileData.id - const item = await createLibraryItem( - { - id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, - user: { id: uid }, - title, - readableContent: '', - itemType, - uploadFile: { id: uploadFileData.id }, - slug: generateSlug(uploadFilePathName), - state: LibraryItemState.Processing, - contentReader: contentReaderForLibraryItem(itemType, uploadFileId), - }, - uid - ) - createdItemId = item.id - } - } - - return { - id: uploadFileData.id, - uploadSignedUrl, - createdPageId: createdItemId, - } - } else { - return { errorCodes: [UploadFileRequestErrorCode.FailedCreate] } - } + return uploadFile(input, uid) }) diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index c281a3384..009157747 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -1,5 +1,39 @@ +import normalizeUrl from 'normalize-url' +import path from 'path' +import { LibraryItemState } from '../entity/library_item' import { UploadFile } from '../entity/upload_file' +import { + PageType, + UploadFileRequestErrorCode, + UploadFileRequestInput, + UploadFileStatus, +} from '../generated/graphql' import { authTrx, getRepository } from '../repository' +import { generateSlug } from '../utils/helpers' +import { logger } from '../utils/logger' +import { + contentReaderForLibraryItem, + generateUploadFilePathName, + generateUploadSignedUrl, +} from '../utils/uploads' +import { validateUrl } from './create_page_save_request' +import { + createLibraryItem, + findLibraryItemByUrl, + updateLibraryItem, +} from './library_item' + +const isFileUrl = (url: string): boolean => { + const parsedUrl = new URL(url) + return parsedUrl.protocol == 'file:' +} + +export const itemTypeForContentType = (contentType: string) => { + if (contentType == 'application/epub+zip') { + return PageType.Book + } + return PageType.File +} export const findUploadFileById = async (id: string) => { return getRepository(UploadFile).findOne({ @@ -22,3 +56,124 @@ export const setFileUploadComplete = async (id: string, userId?: string) => { userId ) } + +export const uploadFile = async ( + input: UploadFileRequestInput, + uid: string +) => { + let uploadFileData: { id: string | null } = { + id: null, + } + let title: string + let fileName: string + try { + const url = normalizeUrl(new URL(input.url).href, { + stripHash: true, + stripWWW: false, + }) + title = decodeURI(path.basename(new URL(url).pathname, '.pdf')) + fileName = decodeURI(path.basename(new URL(url).pathname)).replace( + /[^a-zA-Z0-9-_.]/g, + '' + ) + + if (!fileName) { + fileName = 'content.pdf' + } + + if (!isFileUrl(url)) { + try { + validateUrl(url) + } catch (error) { + logger.info('illegal file input url', error) + return { + errorCodes: [UploadFileRequestErrorCode.BadInput], + } + } + } + } catch { + return { + errorCodes: [UploadFileRequestErrorCode.BadInput], + } + } + + uploadFileData = await authTrx((t) => + t.getRepository(UploadFile).save({ + url: input.url, + user: { id: uid }, + fileName, + status: UploadFileStatus.Initialized, + contentType: input.contentType, + }) + ) + + if (uploadFileData.id) { + const uploadFileId = uploadFileData.id + const uploadFilePathName = generateUploadFilePathName( + uploadFileId, + fileName + ) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + input.contentType + ) + + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` + if (isFileUrl(input.url)) { + await authTrx(async (tx) => { + await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, + status: UploadFileStatus.Initialized, + }) + }) + } + + let createdItemId: string | undefined = undefined + if (input.createPageEntry) { + // If we have a file:// URL, don't try to match it + // and create a copy of the item, just create a + // new item. + const item = await findLibraryItemByUrl(input.url, uid) + if (item) { + await updateLibraryItem( + item.id, + { + state: LibraryItemState.Processing, + }, + uid + ) + createdItemId = item.id + } else { + const itemType = itemTypeForContentType(input.contentType) + const uploadFileId = uploadFileData.id + const item = await createLibraryItem( + { + id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + user: { id: uid }, + title, + readableContent: '', + itemType, + uploadFile: { id: uploadFileData.id }, + slug: generateSlug(uploadFilePathName), + state: LibraryItemState.Processing, + contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + }, + uid + ) + createdItemId = item.id + } + } + + return { + id: uploadFileData.id, + uploadSignedUrl, + createdPageId: createdItemId, + } + } else { + return { + errorCodes: [UploadFileRequestErrorCode.FailedCreate], + } + } +} From 58d384f4f653221d90b71aef522be7f3e4e74b23 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 14:11:07 +0800 Subject: [PATCH 4/4] fix pdf saving --- packages/api/src/jobs/save_page.ts | 20 ++-- packages/api/src/services/save_file.ts | 26 ++--- packages/api/src/services/upload_file.ts | 116 ++++++++------------ packages/api/test/resolvers/article.test.ts | 27 ++++- 4 files changed, 87 insertions(+), 102 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 82d5b246a..3dd702ab9 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -35,18 +35,6 @@ interface Data { taskId?: string } -interface UploadFileResponse { - data: { - uploadFileRequest: { - id: string - uploadSignedUrl: string - uploadFileId: string - createdPageId: string - errorCodes?: string[] - } - } -} - interface FetchResult { finalUrl: string title?: string @@ -63,6 +51,12 @@ const uploadToSignedUrl = async ( contentType: string, contentObjUrl: string ) => { + logger.info('uploading to signed url', { + uploadSignedUrl, + contentType, + contentObjUrl, + }) + try { const stream = await axios.get(contentObjUrl, { responseType: 'stream', @@ -92,6 +86,7 @@ const uploadPdf = async ( url, contentType: 'application/pdf', clientRequestId: articleSavingRequestId, + createPageEntry: true, }, userId ) @@ -116,6 +111,7 @@ const sendImportStatusUpdate = async ( isImported?: boolean ) => { try { + logger.info('sending import status update') const auth = await signToken({ uid: userId }, JWT_SECRET) await axios.post( diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index c0e2a5e75..a004f1558 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -30,20 +30,18 @@ export const saveFile = async ( } } - if (input.state || input.folder) { - await updateLibraryItem( - input.clientRequestId, - { - state: (input.state as unknown as LibraryItemState) || undefined, - folder: input.folder || undefined, - savedAt: input.savedAt ? new Date(input.savedAt) : undefined, - publishedAt: input.publishedAt - ? new Date(input.publishedAt) - : undefined, - }, - user.id - ) - } + await updateLibraryItem( + input.clientRequestId, + { + state: + (input.state as unknown as LibraryItemState) || + LibraryItemState.Succeeded, + folder: input.folder || undefined, + savedAt: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, + }, + user.id + ) // add labels to item await createAndSaveLabelsInLibraryItem( diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index 009157747..1df0ab97f 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -17,11 +17,7 @@ import { generateUploadSignedUrl, } from '../utils/uploads' import { validateUrl } from './create_page_save_request' -import { - createLibraryItem, - findLibraryItemByUrl, - updateLibraryItem, -} from './library_item' +import { createLibraryItem } from './library_item' const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) @@ -61,9 +57,6 @@ export const uploadFile = async ( input: UploadFileRequestInput, uid: string ) => { - let uploadFileData: { id: string | null } = { - id: null, - } let title: string let fileName: string try { @@ -97,7 +90,7 @@ export const uploadFile = async ( } } - uploadFileData = await authTrx((t) => + const uploadFileData = await authTrx((t) => t.getRepository(UploadFile).save({ url: input.url, user: { id: uid }, @@ -106,74 +99,53 @@ export const uploadFile = async ( contentType: input.contentType, }) ) + const uploadFileId = uploadFileData.id + const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + input.contentType + ) - if (uploadFileData.id) { - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName( - uploadFileId, - fileName - ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - input.contentType - ) - - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` + if (isFileUrl(input.url)) { + await authTrx(async (tx) => { + await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, + status: UploadFileStatus.Initialized, }) - } - - let createdItemId: string | undefined = undefined - if (input.createPageEntry) { - // If we have a file:// URL, don't try to match it - // and create a copy of the item, just create a - // new item. - const item = await findLibraryItemByUrl(input.url, uid) - if (item) { - await updateLibraryItem( - item.id, - { - state: LibraryItemState.Processing, - }, - uid - ) - createdItemId = item.id - } else { - const itemType = itemTypeForContentType(input.contentType) - const uploadFileId = uploadFileData.id - const item = await createLibraryItem( - { - id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, - user: { id: uid }, - title, - readableContent: '', - itemType, - uploadFile: { id: uploadFileData.id }, - slug: generateSlug(uploadFilePathName), - state: LibraryItemState.Processing, - contentReader: contentReaderForLibraryItem(itemType, uploadFileId), - }, - uid - ) - createdItemId = item.id - } - } + }) + } + const itemType = itemTypeForContentType(input.contentType) + if (input.createPageEntry) { + // If we have a file:// URL, don't try to match it + // and create a copy of the item, just create a + // new item. + const item = await createLibraryItem( + { + id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + user: { id: uid }, + title, + readableContent: '', + itemType, + uploadFile: { id: uploadFileData.id }, + slug: generateSlug(uploadFilePathName), + state: LibraryItemState.Processing, + contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + }, + uid + ) return { - id: uploadFileData.id, + id: uploadFileId, uploadSignedUrl, - createdPageId: createdItemId, - } - } else { - return { - errorCodes: [UploadFileRequestErrorCode.FailedCreate], + createdPageId: item.id, } } + + return { + id: uploadFileId, + uploadSignedUrl, + } } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 601b2a498..c25d1447f 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -218,14 +218,18 @@ const savePageQuery = ( ` } -const saveFileQuery = (url: string, uploadFileId: string) => { +const saveFileQuery = ( + clientRequestId: string, + url: string, + uploadFileId: string +) => { return ` mutation { saveFile ( input: { url: "${url}", source: "test", - clientRequestId: "${generateFakeUuid()}", + clientRequestId: "${clientRequestId}", uploadFileId: "${uploadFileId}", } ) { @@ -832,8 +836,23 @@ describe('Article API', () => { let query = '' let url = '' let uploadFileId = '' + let itemId = '' + + before(async () => { + const item = await createLibraryItem( + { + user: { id: user.id }, + originalUrl: 'https://blog.omnivore.app/setBookmarkArticle', + slug: 'test-with-omnivore', + readableContent: '

test

', + title: 'test title', + readingProgressBottomPercent: 100, + readingProgressTopPercent: 80, + }, + user.id + ) + itemId = item.id - before(() => { sinon.replace( uploads, 'getStorageFileDetails', @@ -842,7 +861,7 @@ describe('Article API', () => { }) beforeEach(() => { - query = saveFileQuery(url, uploadFileId) + query = saveFileQuery(itemId, url, uploadFileId) }) after(() => {