Merge pull request #64 from omnivore-app/feature/handle-pdf-attachment

add pdf attachment to library

packages/api/src/routers/svc/pdf_attachments.ts

@@ -0,0 +1,177 @@
+import express from 'express'
+import { env } from '../../env'
+import * as jwt from 'jsonwebtoken'
+import { PageType, UploadFileStatus } from '../../generated/graphql'
+import {
+  generateUploadFilePathName,
+  generateUploadSignedUrl,
+  getStorageFileDetails,
+  makeStorageFilePublic,
+} from '../../utils/uploads'
+import { initModels } from '../../server'
+import { kx } from '../../datalayer/knex_config'
+import { analytics } from '../../utils/analytics'
+import { getNewsletterEmail } from '../../services/newsletters'
+
+export function pdfAttachmentsRouter() {
+  const router = express.Router()
+
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  router.post('/upload', async (req, res) => {
+    console.log('pdf-attachments/upload')
+
+    const { email, fileName } = req.body as {
+      email: string
+      fileName: string
+    }
+
+    const token = req?.headers?.authorization
+    if (!token || !jwt.verify(token, env.server.jwtSecret)) {
+      return res.status(401).send('UNAUTHORIZED')
+    }
+
+    const newsletterEmail = await getNewsletterEmail(email)
+    if (!newsletterEmail || !newsletterEmail.user) {
+      return res.status(401).send('UNAUTHORIZED')
+    }
+
+    const user = newsletterEmail.user
+
+    analytics.track({
+      userId: user.id,
+      event: 'pdf-attachment-upload',
+      properties: {
+        env: env.server.apiEnv,
+      },
+    })
+
+    try {
+      const contentType = 'application/pdf'
+      const models = initModels(kx, false)
+      const uploadFileData = await models.uploadFile.create({
+        url: '',
+        userId: user.id,
+        fileName: fileName,
+        status: UploadFileStatus.Initialized,
+        contentType: contentType,
+      })
+
+      if (uploadFileData.id) {
+        const uploadFilePathName = generateUploadFilePathName(
+          uploadFileData.id,
+          fileName
+        )
+        const uploadSignedUrl =
+          env.server.apiEnv === 'prod'
+            ? await generateUploadSignedUrl(uploadFilePathName, contentType)
+            : 'http://localhost:3000/uploads/' + uploadFilePathName
+        res.send({
+          id: uploadFileData.id,
+          url: uploadSignedUrl,
+        })
+      } else {
+        res.status(400).send('BAD REQUEST')
+      }
+    } catch (err) {
+      console.error(err)
+      return res.status(500).send('INTERNAL_SERVER_ERROR')
+    }
+  })
+
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  router.post('/create-article', async (req, res) => {
+    console.log('pdf-attachments/create-article')
+
+    const { email, uploadFileId } = req.body as {
+      email: string
+      uploadFileId: string
+    }
+
+    const token = req?.headers?.authorization
+    if (!token || !jwt.verify(token, env.server.jwtSecret)) {
+      return res.status(401).send('UNAUTHORIZED')
+    }
+
+    const newsletterEmail = await getNewsletterEmail(email)
+    if (!newsletterEmail || !newsletterEmail.user) {
+      return res.status(401).send('UNAUTHORIZED')
+    }
+
+    const user = newsletterEmail.user
+
+    analytics.track({
+      userId: user.id,
+      event: 'pdf-attachment-create-article',
+      properties: {
+        env: env.server.apiEnv,
+      },
+    })
+
+    try {
+      const models = initModels(kx, false)
+      const uploadFile = await models.uploadFile.getWhere({
+        id: uploadFileId,
+        userId: user.id,
+      })
+      if (!uploadFile) {
+        return res.status(400).send('BAD REQUEST')
+      }
+
+      const uploadFileDetails =
+        env.server.apiEnv === 'prod'
+          ? await getStorageFileDetails(uploadFileId, uploadFile.fileName)
+          : { md5Hash: '', size: 0 }
+      const uploadFileHash = uploadFileDetails.md5Hash
+      const pageType = PageType.File
+
+      const saveTime = new Date()
+      const articleToSave = {
+        url: '',
+        pageType: pageType,
+        hash: uploadFileHash,
+        uploadFileId: uploadFileId,
+        title: uploadFile.fileName,
+        content: '',
+      }
+
+      const uploadFileData = await models.uploadFile.setFileUploadComplete(
+        uploadFileId
+      )
+      if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) {
+        return res.status(400).send('BAD REQUEST')
+      }
+
+      const uploadFileUrlOverride =
+        env.server.apiEnv === 'prod'
+          ? await makeStorageFilePublic(
+              uploadFileData.id,
+              uploadFileData.fileName
+            )
+          : 'http://localhost:3000/uploads/' +
+            uploadFileData.id +
+            '/' +
+            uploadFileData.fileName
+
+      const link = await kx.transaction(async (tx) => {
+        const articleRecord = await models.article.create(articleToSave, tx)
+        return models.userArticle.create(
+          {
+            userId: user.id,
+            slug: '',
+            savedAt: saveTime,
+            articleId: articleRecord.id,
+            articleUrl: uploadFileUrlOverride,
+            articleHash: articleRecord.hash,
+          },
+          tx
+        )
+      })
+      res.send({ id: link.id })
+    } catch (err) {
+      console.log(err)
+      res.status(500).send(err)
+    }
+  })
+
+  return router
+}

packages/api/src/server.ts

@@ -37,6 +37,7 @@ import { emailsServiceRouter } from './routers/svc/emails'
 import ReminderModel from './datalayer/reminders'
 import { remindersServiceRouter } from './routers/svc/reminders'
 import { ApolloServer } from 'apollo-server-express'
+import { pdfAttachmentsRouter } from './routers/svc/pdf_attachments'
 
 const PORT = process.env.PORT || 4000
 
@@ -97,6 +98,7 @@ export const createApp = (): {
   app.use('/svc/pubsub/newsletters', newsletterServiceRouter())
   app.use('/svc/pubsub/emails', emailsServiceRouter())
   app.use('/svc/reminders', remindersServiceRouter())
+  app.use('/svc/pdf-attachments', pdfAttachmentsRouter())
 
   if (env.dev.isLocal) {
     app.use('/local/debug', localDebugRouter())

packages/api/test/db.ts

@@ -196,3 +196,7 @@ export const getDeviceToken = async (
 export const getUser = async (id: string): Promise<User | undefined> => {
   return getRepository(User).findOne(id)
 }
+
+export const getLink = async (id: string): Promise<Link | undefined> => {
+  return getRepository(Link).findOne(id)
+}

packages/api/test/routers/pdf_attachments.test.ts

@@ -0,0 +1,80 @@
+import {
+  createTestNewsletterEmail,
+  createTestUser,
+  deleteTestUser,
+  getLink,
+} from '../db'
+import { request } from '../util'
+import { User } from '../../src/entity/user'
+import 'mocha'
+import * as jwt from 'jsonwebtoken'
+import { expect } from 'chai'
+
+describe('PDF attachments Router', () => {
+  const username = 'fakeUser'
+  const newsletterEmail = 'fakeEmail'
+
+  let user: User
+  let authToken: string
+
+  before(async () => {
+    // create test user and login
+    user = await createTestUser(username)
+
+    await createTestNewsletterEmail(user, newsletterEmail)
+    authToken = jwt.sign(newsletterEmail, process.env.JWT_SECRET || '')
+  })
+
+  after(async () => {
+    // clean up
+    await deleteTestUser(username)
+  })
+
+  describe('upload', () => {
+    it('create upload file request and return id and url', async () => {
+      const testFile = 'testFile.pdf'
+
+      const res = await request
+        .post('/svc/pdf-attachments/upload')
+        .set('Authorization', `${authToken}`)
+        .send({
+          email: newsletterEmail,
+          fileName: testFile,
+        })
+        .expect(200)
+
+      expect(res.body.id).to.be.a('string')
+      expect(res.body.url).to.be.a('string')
+    })
+  })
+
+  describe('create article', () => {
+    it('create article with uploaded file id and url', async () => {
+      // upload file first
+      const testFile = 'testFile.pdf'
+      const res = await request
+        .post('/svc/pdf-attachments/upload')
+        .set('Authorization', `${authToken}`)
+        .send({
+          email: newsletterEmail,
+          fileName: testFile,
+        })
+      const uploadFileId = res.body.id
+
+      // create article
+      const res2 = await request
+        .post('/svc/pdf-attachments/create-article')
+        .send({
+          email: newsletterEmail,
+          uploadFileId: uploadFileId,
+        })
+        .set('Authorization', `${authToken}`)
+        .expect(200)
+
+      expect(res2.body.id).to.be.a('string')
+      const link = await getLink(res2.body.id)
+
+      expect(link).to.exist
+    })
+  })
+})

packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql

@@ -0,0 +1,14 @@
+-- Type: DO
+-- Name: add_delete_cascade_on_upload_files
+-- Description: Add delete cascade on user_id field on upload_files table
+
+BEGIN;
+
+ALTER TABLE omnivore.upload_files
+    DROP CONSTRAINT upload_files_user_id_fkey,
+    ADD CONSTRAINT upload_files_user_id_fkey
+        FOREIGN KEY (user_id)
+        REFERENCES omnivore.user(id)
+        ON DELETE CASCADE;
+
+COMMIT;

packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql

@@ -0,0 +1,13 @@
+-- Type: UNDO
+-- Name: add_delete_cascade_on_upload_files
+-- Description: Add delete cascade on user_id field on upload_files table
+
+BEGIN;
+
+ALTER TABLE omnivore.upload_files
+    DROP CONSTRAINT upload_files_user_id_fkey,
+    ADD CONSTRAINT upload_files_user_id_fkey
+        FOREIGN KEY (user_id)
+        REFERENCES omnivore.user(id);
+
+COMMIT;

packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql

@@ -0,0 +1,14 @@
+-- Type: DO
+-- Name: add_delete_cascade_on_upload_file_id_to_pages
+-- Description: Add delete cascade on upload_file_id field on pages table
+
+BEGIN;
+
+ALTER TABLE omnivore.pages
+    DROP CONSTRAINT article_upload_file_id_fkey,
+    ADD CONSTRAINT pages_upload_file_id_fkey
+        FOREIGN KEY (upload_file_id)
+        REFERENCES omnivore.upload_files (id)
+        ON DELETE CASCADE;
+
+COMMIT;

packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql

@@ -0,0 +1,13 @@
+-- Type: UNDO
+-- Name: add_delete_cascade_on_upload_file_id_to_pages
+-- Description: Add delete cascade on upload_file_id field on pages table
+
+BEGIN;
+
+ALTER TABLE omnivore.pages
+    DROP CONSTRAINT pages_upload_file_id_fkey,
+    ADD CONSTRAINT article_upload_file_id_fkey
+        FOREIGN KEY (upload_file_id)
+        REFERENCES omnivore.upload_files (id);
+
+COMMIT;

packages/inbound-email-handler/package.json

@@ -22,7 +22,6 @@
   "devDependencies": {
     "@types/json-bigint": "^1.0.1",
     "@types/node": "^14.11.2",
-    "@types/quoted-printable": "^1.0.0",
     "eslint-plugin-prettier": "^4.0.0"
   },
   "dependencies": {
@@ -31,8 +30,8 @@
     "@sendgrid/client": "^7.6.0",
     "@sentry/serverless": "^6.16.1",
     "axios": "^0.26.0",
+    "jsonwebtoken": "^8.5.1",
     "parse-headers": "^2.0.4",
-    "parse-multipart-data": "^1.2.1",
-    "quoted-printable": "^1.0.1"
+    "parse-multipart-data": "^1.2.1"
   }
 }

packages/inbound-email-handler/src/index.ts

@@ -13,6 +13,7 @@ import {
   isNewsletter,
 } from './newsletter'
 import { PubSub } from '@google-cloud/pubsub'
+import { handlePdfAttachment } from './pdf'
 
 const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
 const pubsub = new PubSub()
@@ -23,9 +24,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
     const parsed: Record<string, string> = {}
 
     for (const part of parts) {
-      const { name, data } = part
+      const { name, data, type, filename } = part
       if (name && data) {
         parsed[name] = data.toString()
+      } else if (type === 'application/pdf' && data) {
+        parsed['pdf-attachment-data'] = data.toString()
+        parsed['pdf-attachment-filename'] = filename
+          ? filename
+          : 'attachment.pdf'
       } else {
         console.log('no data or name for ', part)
       }
@@ -76,7 +82,15 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         if (isConfirmationEmail(from)) {
           console.log('handleConfirmation', from, recipientAddress)
           await handleConfirmation(recipientAddress, subject)
+        } else if (parsed['pdf-attachment-filename']) {
+          console.log('handle PDF attachment', from, recipientAddress)
+          await handlePdfAttachment(
+            recipientAddress,
+            parsed['pdf-attachment-filename'],
+            parsed['pdf-attachment-data']
+          )
         }
+
         // queue non-newsletter emails
         await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
           json: {

packages/inbound-email-handler/src/pdf.ts

@@ -0,0 +1,101 @@
+import axios, { AxiosResponse } from 'axios'
+import { promisify } from 'util'
+import * as jwt from 'jsonwebtoken'
+
+const signToken = promisify(jwt.sign)
+
+type UploadResponse = {
+  id: string
+  url: string
+}
+
+export const handlePdfAttachment = async (
+  email: string,
+  fileName: string,
+  data: string
+): Promise<void> => {
+  console.log('handlePdfAttachment', email, fileName)
+
+  try {
+    const uploadResult = await getUploadIdAndSignedUrl(email, fileName)
+    if (!uploadResult.url || !uploadResult.id) {
+      console.log('failed to create upload request', uploadResult)
+      return
+    }
+    await uploadToSignedUrl(uploadResult.url, data)
+    await createArticle(email, uploadResult.id)
+  } catch (error) {
+    console.error('handlePdfAttachment error', error)
+  }
+}
+
+const getUploadIdAndSignedUrl = async (
+  email: string,
+  fileName: string
+): Promise<UploadResponse> => {
+  if (process.env.JWT_SECRET === undefined) {
+    throw new Error('JWT_SECRET is not defined')
+  }
+  const auth = await signToken(email, process.env.JWT_SECRET)
+  const data = {
+    fileName,
+    email,
+  }
+
+  if (process.env.REST_BACKEND_ENDPOINT === undefined) {
+    throw new Error('REST_BACKEND_ENDPOINT is not defined')
+  }
+  const response = await axios.post(
+    `${process.env.REST_BACKEND_ENDPOINT}/svc/pdf-attachments/upload`,
+    data,
+    {
+      headers: {
+        Authorization: `${auth as string}`,
+        'Content-Type': 'application/json',
+      },
+    }
+  )
+  return response.data as UploadResponse
+}
+
+const uploadToSignedUrl = async (
+  uploadUrl: string,
+  data: string
+): Promise<AxiosResponse> => {
+  return axios.put(uploadUrl, data, {
+    headers: {
+      'Content-Type': 'application/pdf',
+    },
+    maxBodyLength: 1000000000,
+    maxContentLength: 100000000,
+  })
+}
+
+const createArticle = async (
+  email: string,
+  uploadFileId: string
+): Promise<AxiosResponse> => {
+  const data = {
+    email,
+    uploadFileId,
+  }
+
+  if (process.env.JWT_SECRET === undefined) {
+    throw new Error('JWT_SECRET is not defined')
+  }
+  const auth = await signToken(email, process.env.JWT_SECRET)
+
+  if (process.env.REST_BACKEND_ENDPOINT === undefined) {
+    throw new Error('REST_BACKEND_ENDPOINT is not defined')
+  }
+  return axios.post(
+    `${process.env.REST_BACKEND_ENDPOINT}/svc/pdf-attachments/create-article`,
+    data,
+    {
+      headers: {
+        Authorization: `${auth as string}`,
+        'Content-Type': 'application/json',
+      },
+    }
+  )
+}

yarn.lock

@@ -4711,11 +4711,6 @@
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
   integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
 
-"@types/quoted-printable@^1.0.0":
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/@types/quoted-printable/-/quoted-printable-1.0.0.tgz#903f81f5d81698d361c176b5b52970cd2bc9868e"
-  integrity sha512-hgFjmHmgT5M8SvDVe+tMhiUb3xViwqkEAM/sTpWCpO0B2Z7RGAgwiQaxPcLVk4KLiZmqj7BMXZvaQQdX6uPM6A==
-
 "@types/range-parser@*":
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
@@ -14332,13 +14327,6 @@ quick-lru@^4.0.1:
   resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f"
   integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==
 
-quoted-printable@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/quoted-printable/-/quoted-printable-1.0.1.tgz#9eebf5eb3d11eef022b264fd2d2b6b2bb3b84cc3"
-  integrity sha1-nuv16z0R7vAismT9LStrK7O4TMM=
-  dependencies:
-    utf8 "^2.1.0"
-
 randombytes@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a"
@@ -16777,11 +16765,6 @@ user-home@^1.1.1:
   resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190"
   integrity sha1-K1viOjK2Onyd640PKNSFcko98ZA=
 
-utf8@^2.1.0:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96"
-  integrity sha1-H6DZJw6b6FDZsFAn9jUZv0ZFfZY=
-
 util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"