diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 1eebf1adb..82d5b246a 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -10,13 +10,13 @@ import { redisDataSource } from '../redis_data_source' import { userRepository } from '../repository/user' import { saveFile } from '../services/save_file' import { savePage } from '../services/save_page' +import { uploadFile } from '../services/upload_file' import { logger } from '../utils/logger' const signToken = promisify(jwt.sign) const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl const JWT_SECRET = env.server.jwtSecret -const REST_BACKEND_ENDPOINT = `${env.server.internalApiUrl}/api` const MAX_ATTEMPTS = 2 const REQUEST_TIMEOUT = 30000 // 30 seconds @@ -82,86 +82,32 @@ const uploadToSignedUrl = async ( } } -const getUploadIdAndSignedUrl = async ( - userId: string, - url: string, - articleSavingRequestId: string -) => { - const auth = await signToken({ uid: userId }, JWT_SECRET) - const data = JSON.stringify({ - query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { - uploadFileRequest(input:$input) { - ... on UploadFileRequestError { - errorCodes - } - ... on UploadFileRequestSuccess { - id - uploadSignedUrl - } - } - }`, - variables: { - input: { - url: encodeURI(url), - contentType: 'application/pdf', - clientRequestId: articleSavingRequestId, - }, - }, - }) - - try { - const response = await axios.post( - `${REST_BACKEND_ENDPOINT}/graphql`, - data, - { - headers: { - Cookie: `auth=${auth as string};`, - 'Content-Type': 'application/json', - }, - timeout: REQUEST_TIMEOUT, - } - ) - - if ( - response.data.data.uploadFileRequest.errorCodes && - response.data.data.uploadFileRequest.errorCodes?.length > 0 - ) { - console.error( - 'Error while getting upload id and signed url', - response.data.data.uploadFileRequest.errorCodes[0] - ) - return null - } - - return response.data.data.uploadFileRequest - } catch (e) { - console.error('error getting upload id and signed url', e) - return null - } -} - const uploadPdf = async ( url: string, userId: string, articleSavingRequestId: string ) => { - const uploadResult = await getUploadIdAndSignedUrl( - userId, - url, - articleSavingRequestId + const result = await uploadFile( + { + url, + contentType: 'application/pdf', + clientRequestId: articleSavingRequestId, + }, + userId ) - if (!uploadResult) { + if (!result.uploadSignedUrl) { throw new Error('error while getting upload id and signed url') } + const uploaded = await uploadToSignedUrl( - uploadResult.uploadSignedUrl, + result.uploadSignedUrl, 'application/pdf', url ) if (!uploaded) { throw new Error('error while uploading pdf') } - return uploadResult.id + return result.id } const sendImportStatusUpdate = async ( diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 03cce990e..80990d2f7 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -87,11 +87,13 @@ import { import { parsedContentToLibraryItem } from '../../services/save_page' import { findUploadFileById, + itemTypeForContentType, setFileUploadComplete, } from '../../services/upload_file' import { traceAs } from '../../tracing' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' +import { authorized } from '../../utils/gql-utils' import { cleanUrl, errorHandler, @@ -102,7 +104,6 @@ import { titleForFilePath, userDataToUser, } from '../../utils/helpers' -import { authorized } from '../../utils/gql-utils' import { contentConverter, getDistillerResult, @@ -111,7 +112,6 @@ import { parsePreparedContent, } from '../../utils/parser' import { getStorageFileDetails } from '../../utils/uploads' -import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { Markdown = 'markdown', diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index b22395458..74caee344 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -1,55 +1,17 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import normalizeUrl from 'normalize-url' -import path from 'path' -import { LibraryItemState } from '../../entity/library_item' -import { UploadFile } from '../../entity/upload_file' import { env } from '../../env' import { MutationUploadFileRequestArgs, - PageType, UploadFileRequestError, - UploadFileRequestErrorCode, UploadFileRequestSuccess, - UploadFileStatus, } from '../../generated/graphql' -import { validateUrl } from '../../services/create_page_save_request' -import { - createLibraryItem, - findLibraryItemByUrl, - updateLibraryItem, -} from '../../services/library_item' +import { uploadFile } from '../../services/upload_file' import { analytics } from '../../utils/analytics' -import { generateSlug } from '../../utils/helpers' import { authorized } from '../../utils/gql-utils' - -import { - contentReaderForLibraryItem, - generateUploadFilePathName, - generateUploadSignedUrl, -} from '../../utils/uploads' - -const isFileUrl = (url: string): boolean => { - const parsedUrl = new URL(url) - return parsedUrl.protocol == 'file:' -} - -export const itemTypeForContentType = (contentType: string) => { - if (contentType == 'application/epub+zip') { - return PageType.Book - } - return PageType.File -} - export const uploadFileRequestResolver = authorized< UploadFileRequestSuccess, UploadFileRequestError, MutationUploadFileRequestArgs ->(async (_, { input }, ctx) => { - const { authTrx, uid, log } = ctx - let uploadFileData: { id: string | null } = { - id: null, - } - +>(async (_, { input }, { uid }) => { analytics.track({ userId: uid, event: 'file_upload_request', @@ -59,112 +21,5 @@ export const uploadFileRequestResolver = authorized< }, }) - let title: string - let fileName: string - try { - const url = normalizeUrl(new URL(input.url).href, { - stripHash: true, - stripWWW: false, - }) - title = decodeURI(path.basename(new URL(url).pathname, '.pdf')) - fileName = decodeURI(path.basename(new URL(url).pathname)).replace( - /[^a-zA-Z0-9-_.]/g, - '' - ) - - if (!fileName) { - fileName = 'content.pdf' - } - - if (!isFileUrl(url)) { - try { - validateUrl(url) - } catch (error) { - log.info('illegal file input url', error) - return { - errorCodes: [UploadFileRequestErrorCode.BadInput], - } - } - } - } catch { - return { errorCodes: [UploadFileRequestErrorCode.BadInput] } - } - - uploadFileData = await authTrx((t) => - t.getRepository(UploadFile).save({ - url: input.url, - user: { id: uid }, - fileName, - status: UploadFileStatus.Initialized, - contentType: input.contentType, - }) - ) - - if (uploadFileData.id) { - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName( - uploadFileId, - fileName - ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - input.contentType - ) - - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) - }) - } - - let createdItemId: string | undefined = undefined - if (input.createPageEntry) { - // If we have a file:// URL, don't try to match it - // and create a copy of the item, just create a - // new item. - const item = await findLibraryItemByUrl(input.url, uid) - if (item) { - await updateLibraryItem( - item.id, - { - state: LibraryItemState.Processing, - }, - uid - ) - createdItemId = item.id - } else { - const itemType = itemTypeForContentType(input.contentType) - const uploadFileId = uploadFileData.id - const item = await createLibraryItem( - { - id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, - user: { id: uid }, - title, - readableContent: '', - itemType, - uploadFile: { id: uploadFileData.id }, - slug: generateSlug(uploadFilePathName), - state: LibraryItemState.Processing, - contentReader: contentReaderForLibraryItem(itemType, uploadFileId), - }, - uid - ) - createdItemId = item.id - } - } - - return { - id: uploadFileData.id, - uploadSignedUrl, - createdPageId: createdItemId, - } - } else { - return { errorCodes: [UploadFileRequestErrorCode.FailedCreate] } - } + return uploadFile(input, uid) }) diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index c281a3384..009157747 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -1,5 +1,39 @@ +import normalizeUrl from 'normalize-url' +import path from 'path' +import { LibraryItemState } from '../entity/library_item' import { UploadFile } from '../entity/upload_file' +import { + PageType, + UploadFileRequestErrorCode, + UploadFileRequestInput, + UploadFileStatus, +} from '../generated/graphql' import { authTrx, getRepository } from '../repository' +import { generateSlug } from '../utils/helpers' +import { logger } from '../utils/logger' +import { + contentReaderForLibraryItem, + generateUploadFilePathName, + generateUploadSignedUrl, +} from '../utils/uploads' +import { validateUrl } from './create_page_save_request' +import { + createLibraryItem, + findLibraryItemByUrl, + updateLibraryItem, +} from './library_item' + +const isFileUrl = (url: string): boolean => { + const parsedUrl = new URL(url) + return parsedUrl.protocol == 'file:' +} + +export const itemTypeForContentType = (contentType: string) => { + if (contentType == 'application/epub+zip') { + return PageType.Book + } + return PageType.File +} export const findUploadFileById = async (id: string) => { return getRepository(UploadFile).findOne({ @@ -22,3 +56,124 @@ export const setFileUploadComplete = async (id: string, userId?: string) => { userId ) } + +export const uploadFile = async ( + input: UploadFileRequestInput, + uid: string +) => { + let uploadFileData: { id: string | null } = { + id: null, + } + let title: string + let fileName: string + try { + const url = normalizeUrl(new URL(input.url).href, { + stripHash: true, + stripWWW: false, + }) + title = decodeURI(path.basename(new URL(url).pathname, '.pdf')) + fileName = decodeURI(path.basename(new URL(url).pathname)).replace( + /[^a-zA-Z0-9-_.]/g, + '' + ) + + if (!fileName) { + fileName = 'content.pdf' + } + + if (!isFileUrl(url)) { + try { + validateUrl(url) + } catch (error) { + logger.info('illegal file input url', error) + return { + errorCodes: [UploadFileRequestErrorCode.BadInput], + } + } + } + } catch { + return { + errorCodes: [UploadFileRequestErrorCode.BadInput], + } + } + + uploadFileData = await authTrx((t) => + t.getRepository(UploadFile).save({ + url: input.url, + user: { id: uid }, + fileName, + status: UploadFileStatus.Initialized, + contentType: input.contentType, + }) + ) + + if (uploadFileData.id) { + const uploadFileId = uploadFileData.id + const uploadFilePathName = generateUploadFilePathName( + uploadFileId, + fileName + ) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + input.contentType + ) + + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` + if (isFileUrl(input.url)) { + await authTrx(async (tx) => { + await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, + status: UploadFileStatus.Initialized, + }) + }) + } + + let createdItemId: string | undefined = undefined + if (input.createPageEntry) { + // If we have a file:// URL, don't try to match it + // and create a copy of the item, just create a + // new item. + const item = await findLibraryItemByUrl(input.url, uid) + if (item) { + await updateLibraryItem( + item.id, + { + state: LibraryItemState.Processing, + }, + uid + ) + createdItemId = item.id + } else { + const itemType = itemTypeForContentType(input.contentType) + const uploadFileId = uploadFileData.id + const item = await createLibraryItem( + { + id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + user: { id: uid }, + title, + readableContent: '', + itemType, + uploadFile: { id: uploadFileData.id }, + slug: generateSlug(uploadFilePathName), + state: LibraryItemState.Processing, + contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + }, + uid + ) + createdItemId = item.id + } + } + + return { + id: uploadFileData.id, + uploadSignedUrl, + createdPageId: createdItemId, + } + } else { + return { + errorCodes: [UploadFileRequestErrorCode.FailedCreate], + } + } +}