remove api call to upload file request

packages/api/src/jobs/save_page.ts

@@ -10,13 +10,13 @@ import { redisDataSource } from '../redis_data_source'
 import { userRepository } from '../repository/user'
 import { saveFile } from '../services/save_file'
 import { savePage } from '../services/save_page'
+import { uploadFile } from '../services/upload_file'
 import { logger } from '../utils/logger'
 
 const signToken = promisify(jwt.sign)
 
 const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl
 const JWT_SECRET = env.server.jwtSecret
-const REST_BACKEND_ENDPOINT = `${env.server.internalApiUrl}/api`
 
 const MAX_ATTEMPTS = 2
 const REQUEST_TIMEOUT = 30000 // 30 seconds
@@ -82,86 +82,32 @@ const uploadToSignedUrl = async (
   }
 }
 
-const getUploadIdAndSignedUrl = async (
-  userId: string,
-  url: string,
-  articleSavingRequestId: string
-) => {
-  const auth = await signToken({ uid: userId }, JWT_SECRET)
-  const data = JSON.stringify({
-    query: `mutation UploadFileRequest($input: UploadFileRequestInput!) {
-      uploadFileRequest(input:$input) {
-        ... on UploadFileRequestError {
-          errorCodes
-        }
-        ... on UploadFileRequestSuccess {
-          id
-          uploadSignedUrl
-        }
-      }
-    }`,
-    variables: {
-      input: {
-        url: encodeURI(url),
-        contentType: 'application/pdf',
-        clientRequestId: articleSavingRequestId,
-      },
-    },
-  })
-
-  try {
-    const response = await axios.post<UploadFileResponse>(
-      `${REST_BACKEND_ENDPOINT}/graphql`,
-      data,
-      {
-        headers: {
-          Cookie: `auth=${auth as string};`,
-          'Content-Type': 'application/json',
-        },
-        timeout: REQUEST_TIMEOUT,
-      }
-    )
-
-    if (
-      response.data.data.uploadFileRequest.errorCodes &&
-      response.data.data.uploadFileRequest.errorCodes?.length > 0
-    ) {
-      console.error(
-        'Error while getting upload id and signed url',
-        response.data.data.uploadFileRequest.errorCodes[0]
-      )
-      return null
-    }
-
-    return response.data.data.uploadFileRequest
-  } catch (e) {
-    console.error('error getting upload id and signed url', e)
-    return null
-  }
-}
-
 const uploadPdf = async (
   url: string,
   userId: string,
   articleSavingRequestId: string
 ) => {
-  const uploadResult = await getUploadIdAndSignedUrl(
-    userId,
-    url,
-    articleSavingRequestId
+  const result = await uploadFile(
+    {
+      url,
+      contentType: 'application/pdf',
+      clientRequestId: articleSavingRequestId,
+    },
+    userId
   )
-  if (!uploadResult) {
+  if (!result.uploadSignedUrl) {
     throw new Error('error while getting upload id and signed url')
   }
+
   const uploaded = await uploadToSignedUrl(
-    uploadResult.uploadSignedUrl,
+    result.uploadSignedUrl,
     'application/pdf',
     url
   )
   if (!uploaded) {
     throw new Error('error while uploading pdf')
   }
-  return uploadResult.id
+  return result.id
 }
 
 const sendImportStatusUpdate = async (

packages/api/src/resolvers/article/index.ts

@@ -87,11 +87,13 @@ import {
 import { parsedContentToLibraryItem } from '../../services/save_page'
 import {
   findUploadFileById,
+  itemTypeForContentType,
   setFileUploadComplete,
 } from '../../services/upload_file'
 import { traceAs } from '../../tracing'
 import { analytics } from '../../utils/analytics'
 import { isSiteBlockedForParse } from '../../utils/blocked'
+import { authorized } from '../../utils/gql-utils'
 import {
   cleanUrl,
   errorHandler,
@@ -102,7 +104,6 @@ import {
   titleForFilePath,
   userDataToUser,
 } from '../../utils/helpers'
-import { authorized } from '../../utils/gql-utils'
 import {
   contentConverter,
   getDistillerResult,
@@ -111,7 +112,6 @@ import {
   parsePreparedContent,
 } from '../../utils/parser'
 import { getStorageFileDetails } from '../../utils/uploads'
-import { itemTypeForContentType } from '../upload_files'
 
 export enum ArticleFormat {
   Markdown = 'markdown',

packages/api/src/resolvers/upload_files/index.ts

@@ -1,55 +1,17 @@
-/* eslint-disable @typescript-eslint/no-unused-vars */
-import normalizeUrl from 'normalize-url'
-import path from 'path'
-import { LibraryItemState } from '../../entity/library_item'
-import { UploadFile } from '../../entity/upload_file'
 import { env } from '../../env'
 import {
   MutationUploadFileRequestArgs,
-  PageType,
   UploadFileRequestError,
-  UploadFileRequestErrorCode,
   UploadFileRequestSuccess,
-  UploadFileStatus,
 } from '../../generated/graphql'
-import { validateUrl } from '../../services/create_page_save_request'
-import {
-  createLibraryItem,
-  findLibraryItemByUrl,
-  updateLibraryItem,
-} from '../../services/library_item'
+import { uploadFile } from '../../services/upload_file'
 import { analytics } from '../../utils/analytics'
-import { generateSlug } from '../../utils/helpers'
 import { authorized } from '../../utils/gql-utils'
-
-import {
-  contentReaderForLibraryItem,
-  generateUploadFilePathName,
-  generateUploadSignedUrl,
-} from '../../utils/uploads'
-
-const isFileUrl = (url: string): boolean => {
-  const parsedUrl = new URL(url)
-  return parsedUrl.protocol == 'file:'
-}
-
-export const itemTypeForContentType = (contentType: string) => {
-  if (contentType == 'application/epub+zip') {
-    return PageType.Book
-  }
-  return PageType.File
-}
-
 export const uploadFileRequestResolver = authorized<
   UploadFileRequestSuccess,
   UploadFileRequestError,
   MutationUploadFileRequestArgs
->(async (_, { input }, ctx) => {
-  const { authTrx, uid, log } = ctx
-  let uploadFileData: { id: string | null } = {
-    id: null,
-  }
-
+>(async (_, { input }, { uid }) => {
   analytics.track({
     userId: uid,
     event: 'file_upload_request',
@@ -59,112 +21,5 @@ export const uploadFileRequestResolver = authorized<
     },
   })
 
-  let title: string
-  let fileName: string
-  try {
-    const url = normalizeUrl(new URL(input.url).href, {
-      stripHash: true,
-      stripWWW: false,
-    })
-    title = decodeURI(path.basename(new URL(url).pathname, '.pdf'))
-    fileName = decodeURI(path.basename(new URL(url).pathname)).replace(
-      /[^a-zA-Z0-9-_.]/g,
-      ''
-    )
-
-    if (!fileName) {
-      fileName = 'content.pdf'
-    }
-
-    if (!isFileUrl(url)) {
-      try {
-        validateUrl(url)
-      } catch (error) {
-        log.info('illegal file input url', error)
-        return {
-          errorCodes: [UploadFileRequestErrorCode.BadInput],
-        }
-      }
-    }
-  } catch {
-    return { errorCodes: [UploadFileRequestErrorCode.BadInput] }
-  }
-
-  uploadFileData = await authTrx((t) =>
-    t.getRepository(UploadFile).save({
-      url: input.url,
-      user: { id: uid },
-      fileName,
-      status: UploadFileStatus.Initialized,
-      contentType: input.contentType,
-    })
-  )
-
-  if (uploadFileData.id) {
-    const uploadFileId = uploadFileData.id
-    const uploadFilePathName = generateUploadFilePathName(
-      uploadFileId,
-      fileName
-    )
-    const uploadSignedUrl = await generateUploadSignedUrl(
-      uploadFilePathName,
-      input.contentType
-    )
-
-    // If this is a file URL, we swap in a special URL
-    const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}`
-    if (isFileUrl(input.url)) {
-      await authTrx(async (tx) => {
-        await tx.getRepository(UploadFile).update(uploadFileId, {
-          url: attachmentUrl,
-          status: UploadFileStatus.Initialized,
-        })
-      })
-    }
-
-    let createdItemId: string | undefined = undefined
-    if (input.createPageEntry) {
-      // If we have a file:// URL, don't try to match it
-      // and create a copy of the item, just create a
-      // new item.
-      const item = await findLibraryItemByUrl(input.url, uid)
-      if (item) {
-        await updateLibraryItem(
-          item.id,
-          {
-            state: LibraryItemState.Processing,
-          },
-          uid
-        )
-        createdItemId = item.id
-      } else {
-        const itemType = itemTypeForContentType(input.contentType)
-        const uploadFileId = uploadFileData.id
-        const item = await createLibraryItem(
-          {
-            id: input.clientRequestId || undefined,
-            originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url,
-            user: { id: uid },
-            title,
-            readableContent: '',
-            itemType,
-            uploadFile: { id: uploadFileData.id },
-            slug: generateSlug(uploadFilePathName),
-            state: LibraryItemState.Processing,
-            contentReader: contentReaderForLibraryItem(itemType, uploadFileId),
-          },
-          uid
-        )
-        createdItemId = item.id
-      }
-    }
-
-    return {
-      id: uploadFileData.id,
-      uploadSignedUrl,
-      createdPageId: createdItemId,
-    }
-  } else {
-    return { errorCodes: [UploadFileRequestErrorCode.FailedCreate] }
-  }
+  return uploadFile(input, uid)
 })

packages/api/src/services/upload_file.ts

@@ -1,5 +1,39 @@
+import normalizeUrl from 'normalize-url'
+import path from 'path'
+import { LibraryItemState } from '../entity/library_item'
 import { UploadFile } from '../entity/upload_file'
+import {
+  PageType,
+  UploadFileRequestErrorCode,
+  UploadFileRequestInput,
+  UploadFileStatus,
+} from '../generated/graphql'
 import { authTrx, getRepository } from '../repository'
+import { generateSlug } from '../utils/helpers'
+import { logger } from '../utils/logger'
+import {
+  contentReaderForLibraryItem,
+  generateUploadFilePathName,
+  generateUploadSignedUrl,
+} from '../utils/uploads'
+import { validateUrl } from './create_page_save_request'
+import {
+  createLibraryItem,
+  findLibraryItemByUrl,
+  updateLibraryItem,
+} from './library_item'
+
+const isFileUrl = (url: string): boolean => {
+  const parsedUrl = new URL(url)
+  return parsedUrl.protocol == 'file:'
+}
+
+export const itemTypeForContentType = (contentType: string) => {
+  if (contentType == 'application/epub+zip') {
+    return PageType.Book
+  }
+  return PageType.File
+}
 
 export const findUploadFileById = async (id: string) => {
   return getRepository(UploadFile).findOne({
@@ -22,3 +56,124 @@ export const setFileUploadComplete = async (id: string, userId?: string) => {
     userId
   )
 }
+
+export const uploadFile = async (
+  input: UploadFileRequestInput,
+  uid: string
+) => {
+  let uploadFileData: { id: string | null } = {
+    id: null,
+  }
+  let title: string
+  let fileName: string
+  try {
+    const url = normalizeUrl(new URL(input.url).href, {
+      stripHash: true,
+      stripWWW: false,
+    })
+    title = decodeURI(path.basename(new URL(url).pathname, '.pdf'))
+    fileName = decodeURI(path.basename(new URL(url).pathname)).replace(
+      /[^a-zA-Z0-9-_.]/g,
+      ''
+    )
+
+    if (!fileName) {
+      fileName = 'content.pdf'
+    }
+
+    if (!isFileUrl(url)) {
+      try {
+        validateUrl(url)
+      } catch (error) {
+        logger.info('illegal file input url', error)
+        return {
+          errorCodes: [UploadFileRequestErrorCode.BadInput],
+        }
+      }
+    }
+  } catch {
+    return {
+      errorCodes: [UploadFileRequestErrorCode.BadInput],
+    }
+  }
+
+  uploadFileData = await authTrx((t) =>
+    t.getRepository(UploadFile).save({
+      url: input.url,
+      user: { id: uid },
+      fileName,
+      status: UploadFileStatus.Initialized,
+      contentType: input.contentType,
+    })
+  )
+
+  if (uploadFileData.id) {
+    const uploadFileId = uploadFileData.id
+    const uploadFilePathName = generateUploadFilePathName(
+      uploadFileId,
+      fileName
+    )
+    const uploadSignedUrl = await generateUploadSignedUrl(
+      uploadFilePathName,
+      input.contentType
+    )
+
+    // If this is a file URL, we swap in a special URL
+    const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}`
+    if (isFileUrl(input.url)) {
+      await authTrx(async (tx) => {
+        await tx.getRepository(UploadFile).update(uploadFileId, {
+          url: attachmentUrl,
+          status: UploadFileStatus.Initialized,
+        })
+      })
+    }
+
+    let createdItemId: string | undefined = undefined
+    if (input.createPageEntry) {
+      // If we have a file:// URL, don't try to match it
+      // and create a copy of the item, just create a
+      // new item.
+      const item = await findLibraryItemByUrl(input.url, uid)
+      if (item) {
+        await updateLibraryItem(
+          item.id,
+          {
+            state: LibraryItemState.Processing,
+          },
+          uid
+        )
+        createdItemId = item.id
+      } else {
+        const itemType = itemTypeForContentType(input.contentType)
+        const uploadFileId = uploadFileData.id
+        const item = await createLibraryItem(
+          {
+            id: input.clientRequestId || undefined,
+            originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url,
+            user: { id: uid },
+            title,
+            readableContent: '',
+            itemType,
+            uploadFile: { id: uploadFileData.id },
+            slug: generateSlug(uploadFilePathName),
+            state: LibraryItemState.Processing,
+            contentReader: contentReaderForLibraryItem(itemType, uploadFileId),
+          },
+          uid
+        )
+        createdItemId = item.id
+      }
+    }
+
+    return {
+      id: uploadFileData.id,
+      uploadSignedUrl,
+      createdPageId: createdItemId,
+    }
+  } else {
+    return {
+      errorCodes: [UploadFileRequestErrorCode.FailedCreate],
+    }
+  }
+}