From 58d384f4f653221d90b71aef522be7f3e4e74b23 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 14:11:07 +0800 Subject: [PATCH] fix pdf saving --- packages/api/src/jobs/save_page.ts | 20 ++-- packages/api/src/services/save_file.ts | 26 ++--- packages/api/src/services/upload_file.ts | 116 ++++++++------------ packages/api/test/resolvers/article.test.ts | 27 ++++- 4 files changed, 87 insertions(+), 102 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 82d5b246a..3dd702ab9 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -35,18 +35,6 @@ interface Data { taskId?: string } -interface UploadFileResponse { - data: { - uploadFileRequest: { - id: string - uploadSignedUrl: string - uploadFileId: string - createdPageId: string - errorCodes?: string[] - } - } -} - interface FetchResult { finalUrl: string title?: string @@ -63,6 +51,12 @@ const uploadToSignedUrl = async ( contentType: string, contentObjUrl: string ) => { + logger.info('uploading to signed url', { + uploadSignedUrl, + contentType, + contentObjUrl, + }) + try { const stream = await axios.get(contentObjUrl, { responseType: 'stream', @@ -92,6 +86,7 @@ const uploadPdf = async ( url, contentType: 'application/pdf', clientRequestId: articleSavingRequestId, + createPageEntry: true, }, userId ) @@ -116,6 +111,7 @@ const sendImportStatusUpdate = async ( isImported?: boolean ) => { try { + logger.info('sending import status update') const auth = await signToken({ uid: userId }, JWT_SECRET) await axios.post( diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index c0e2a5e75..a004f1558 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -30,20 +30,18 @@ export const saveFile = async ( } } - if (input.state || input.folder) { - await updateLibraryItem( - input.clientRequestId, - { - state: (input.state as unknown as LibraryItemState) || undefined, - folder: input.folder || undefined, - savedAt: input.savedAt ? new Date(input.savedAt) : undefined, - publishedAt: input.publishedAt - ? new Date(input.publishedAt) - : undefined, - }, - user.id - ) - } + await updateLibraryItem( + input.clientRequestId, + { + state: + (input.state as unknown as LibraryItemState) || + LibraryItemState.Succeeded, + folder: input.folder || undefined, + savedAt: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, + }, + user.id + ) // add labels to item await createAndSaveLabelsInLibraryItem( diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index 009157747..1df0ab97f 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -17,11 +17,7 @@ import { generateUploadSignedUrl, } from '../utils/uploads' import { validateUrl } from './create_page_save_request' -import { - createLibraryItem, - findLibraryItemByUrl, - updateLibraryItem, -} from './library_item' +import { createLibraryItem } from './library_item' const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) @@ -61,9 +57,6 @@ export const uploadFile = async ( input: UploadFileRequestInput, uid: string ) => { - let uploadFileData: { id: string | null } = { - id: null, - } let title: string let fileName: string try { @@ -97,7 +90,7 @@ export const uploadFile = async ( } } - uploadFileData = await authTrx((t) => + const uploadFileData = await authTrx((t) => t.getRepository(UploadFile).save({ url: input.url, user: { id: uid }, @@ -106,74 +99,53 @@ export const uploadFile = async ( contentType: input.contentType, }) ) + const uploadFileId = uploadFileData.id + const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + input.contentType + ) - if (uploadFileData.id) { - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName( - uploadFileId, - fileName - ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - input.contentType - ) - - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` + if (isFileUrl(input.url)) { + await authTrx(async (tx) => { + await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, + status: UploadFileStatus.Initialized, }) - } - - let createdItemId: string | undefined = undefined - if (input.createPageEntry) { - // If we have a file:// URL, don't try to match it - // and create a copy of the item, just create a - // new item. - const item = await findLibraryItemByUrl(input.url, uid) - if (item) { - await updateLibraryItem( - item.id, - { - state: LibraryItemState.Processing, - }, - uid - ) - createdItemId = item.id - } else { - const itemType = itemTypeForContentType(input.contentType) - const uploadFileId = uploadFileData.id - const item = await createLibraryItem( - { - id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, - user: { id: uid }, - title, - readableContent: '', - itemType, - uploadFile: { id: uploadFileData.id }, - slug: generateSlug(uploadFilePathName), - state: LibraryItemState.Processing, - contentReader: contentReaderForLibraryItem(itemType, uploadFileId), - }, - uid - ) - createdItemId = item.id - } - } + }) + } + const itemType = itemTypeForContentType(input.contentType) + if (input.createPageEntry) { + // If we have a file:// URL, don't try to match it + // and create a copy of the item, just create a + // new item. + const item = await createLibraryItem( + { + id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + user: { id: uid }, + title, + readableContent: '', + itemType, + uploadFile: { id: uploadFileData.id }, + slug: generateSlug(uploadFilePathName), + state: LibraryItemState.Processing, + contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + }, + uid + ) return { - id: uploadFileData.id, + id: uploadFileId, uploadSignedUrl, - createdPageId: createdItemId, - } - } else { - return { - errorCodes: [UploadFileRequestErrorCode.FailedCreate], + createdPageId: item.id, } } + + return { + id: uploadFileId, + uploadSignedUrl, + } } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 601b2a498..c25d1447f 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -218,14 +218,18 @@ const savePageQuery = ( ` } -const saveFileQuery = (url: string, uploadFileId: string) => { +const saveFileQuery = ( + clientRequestId: string, + url: string, + uploadFileId: string +) => { return ` mutation { saveFile ( input: { url: "${url}", source: "test", - clientRequestId: "${generateFakeUuid()}", + clientRequestId: "${clientRequestId}", uploadFileId: "${uploadFileId}", } ) { @@ -832,8 +836,23 @@ describe('Article API', () => { let query = '' let url = '' let uploadFileId = '' + let itemId = '' + + before(async () => { + const item = await createLibraryItem( + { + user: { id: user.id }, + originalUrl: 'https://blog.omnivore.app/setBookmarkArticle', + slug: 'test-with-omnivore', + readableContent: '

test

', + title: 'test title', + readingProgressBottomPercent: 100, + readingProgressTopPercent: 80, + }, + user.id + ) + itemId = item.id - before(() => { sinon.replace( uploads, 'getStorageFileDetails', @@ -842,7 +861,7 @@ describe('Article API', () => { }) beforeEach(() => { - query = saveFileQuery(url, uploadFileId) + query = saveFileQuery(itemId, url, uploadFileId) }) after(() => {