diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts new file mode 100644 index 000000000..6ec01d68a --- /dev/null +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -0,0 +1,177 @@ +import express from 'express' +import { env } from '../../env' +import * as jwt from 'jsonwebtoken' +import { PageType, UploadFileStatus } from '../../generated/graphql' +import { + generateUploadFilePathName, + generateUploadSignedUrl, + getStorageFileDetails, + makeStorageFilePublic, +} from '../../utils/uploads' +import { initModels } from '../../server' +import { kx } from '../../datalayer/knex_config' +import { analytics } from '../../utils/analytics' +import { getNewsletterEmail } from '../../services/newsletters' + +export function pdfAttachmentsRouter() { + const router = express.Router() + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + router.post('/upload', async (req, res) => { + console.log('pdf-attachments/upload') + + const { email, fileName } = req.body as { + email: string + fileName: string + } + + const token = req?.headers?.authorization + if (!token || !jwt.verify(token, env.server.jwtSecret)) { + return res.status(401).send('UNAUTHORIZED') + } + + const newsletterEmail = await getNewsletterEmail(email) + if (!newsletterEmail || !newsletterEmail.user) { + return res.status(401).send('UNAUTHORIZED') + } + + const user = newsletterEmail.user + + analytics.track({ + userId: user.id, + event: 'pdf-attachment-upload', + properties: { + env: env.server.apiEnv, + }, + }) + + try { + const contentType = 'application/pdf' + const models = initModels(kx, false) + const uploadFileData = await models.uploadFile.create({ + url: '', + userId: user.id, + fileName: fileName, + status: UploadFileStatus.Initialized, + contentType: contentType, + }) + + if (uploadFileData.id) { + const uploadFilePathName = generateUploadFilePathName( + uploadFileData.id, + fileName + ) + const uploadSignedUrl = + env.server.apiEnv === 'prod' + ? await generateUploadSignedUrl(uploadFilePathName, contentType) + : 'http://localhost:3000/uploads/' + uploadFilePathName + res.send({ + id: uploadFileData.id, + url: uploadSignedUrl, + }) + } else { + res.status(400).send('BAD REQUEST') + } + } catch (err) { + console.error(err) + return res.status(500).send('INTERNAL_SERVER_ERROR') + } + }) + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + router.post('/create-article', async (req, res) => { + console.log('pdf-attachments/create-article') + + const { email, uploadFileId } = req.body as { + email: string + uploadFileId: string + } + + const token = req?.headers?.authorization + if (!token || !jwt.verify(token, env.server.jwtSecret)) { + return res.status(401).send('UNAUTHORIZED') + } + + const newsletterEmail = await getNewsletterEmail(email) + if (!newsletterEmail || !newsletterEmail.user) { + return res.status(401).send('UNAUTHORIZED') + } + + const user = newsletterEmail.user + + analytics.track({ + userId: user.id, + event: 'pdf-attachment-create-article', + properties: { + env: env.server.apiEnv, + }, + }) + + try { + const models = initModels(kx, false) + const uploadFile = await models.uploadFile.getWhere({ + id: uploadFileId, + userId: user.id, + }) + if (!uploadFile) { + return res.status(400).send('BAD REQUEST') + } + + const uploadFileDetails = + env.server.apiEnv === 'prod' + ? await getStorageFileDetails(uploadFileId, uploadFile.fileName) + : { md5Hash: '', size: 0 } + const uploadFileHash = uploadFileDetails.md5Hash + const pageType = PageType.File + + const saveTime = new Date() + const articleToSave = { + url: '', + pageType: pageType, + hash: uploadFileHash, + uploadFileId: uploadFileId, + title: uploadFile.fileName, + content: '', + } + + const uploadFileData = await models.uploadFile.setFileUploadComplete( + uploadFileId + ) + if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { + return res.status(400).send('BAD REQUEST') + } + + const uploadFileUrlOverride = + env.server.apiEnv === 'prod' + ? await makeStorageFilePublic( + uploadFileData.id, + uploadFileData.fileName + ) + : 'http://localhost:3000/uploads/' + + uploadFileData.id + + '/' + + uploadFileData.fileName + + const link = await kx.transaction(async (tx) => { + const articleRecord = await models.article.create(articleToSave, tx) + return models.userArticle.create( + { + userId: user.id, + slug: '', + savedAt: saveTime, + articleId: articleRecord.id, + articleUrl: uploadFileUrlOverride, + articleHash: articleRecord.hash, + }, + tx + ) + }) + res.send({ id: link.id }) + } catch (err) { + console.log(err) + res.status(500).send(err) + } + }) + + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index c72e0f853..b0352da67 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -37,6 +37,7 @@ import { emailsServiceRouter } from './routers/svc/emails' import ReminderModel from './datalayer/reminders' import { remindersServiceRouter } from './routers/svc/reminders' import { ApolloServer } from 'apollo-server-express' +import { pdfAttachmentsRouter } from './routers/svc/pdf_attachments' const PORT = process.env.PORT || 4000 @@ -97,6 +98,7 @@ export const createApp = (): { app.use('/svc/pubsub/newsletters', newsletterServiceRouter()) app.use('/svc/pubsub/emails', emailsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) + app.use('/svc/pdf-attachments', pdfAttachmentsRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index d405472aa..68596e748 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -196,3 +196,7 @@ export const getDeviceToken = async ( export const getUser = async (id: string): Promise => { return getRepository(User).findOne(id) } + +export const getLink = async (id: string): Promise => { + return getRepository(Link).findOne(id) +} diff --git a/packages/api/test/routers/pdf_attachments.test.ts b/packages/api/test/routers/pdf_attachments.test.ts new file mode 100644 index 000000000..6e9756f41 --- /dev/null +++ b/packages/api/test/routers/pdf_attachments.test.ts @@ -0,0 +1,80 @@ +import { + createTestNewsletterEmail, + createTestUser, + deleteTestUser, + getLink, +} from '../db' +import { request } from '../util' +import { User } from '../../src/entity/user' +import 'mocha' +import * as jwt from 'jsonwebtoken' +import { expect } from 'chai' + +describe('PDF attachments Router', () => { + const username = 'fakeUser' + const newsletterEmail = 'fakeEmail' + + let user: User + let authToken: string + + before(async () => { + // create test user and login + user = await createTestUser(username) + + await createTestNewsletterEmail(user, newsletterEmail) + authToken = jwt.sign(newsletterEmail, process.env.JWT_SECRET || '') + }) + + after(async () => { + // clean up + await deleteTestUser(username) + }) + + describe('upload', () => { + it('create upload file request and return id and url', async () => { + const testFile = 'testFile.pdf' + + const res = await request + .post('/svc/pdf-attachments/upload') + .set('Authorization', `${authToken}`) + .send({ + email: newsletterEmail, + fileName: testFile, + }) + .expect(200) + + expect(res.body.id).to.be.a('string') + expect(res.body.url).to.be.a('string') + }) + }) + + describe('create article', () => { + it('create article with uploaded file id and url', async () => { + // upload file first + const testFile = 'testFile.pdf' + const res = await request + .post('/svc/pdf-attachments/upload') + .set('Authorization', `${authToken}`) + .send({ + email: newsletterEmail, + fileName: testFile, + }) + const uploadFileId = res.body.id + + // create article + const res2 = await request + .post('/svc/pdf-attachments/create-article') + .send({ + email: newsletterEmail, + uploadFileId: uploadFileId, + }) + .set('Authorization', `${authToken}`) + .expect(200) + + expect(res2.body.id).to.be.a('string') + const link = await getLink(res2.body.id) + + expect(link).to.exist + }) + }) +}) diff --git a/packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql b/packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql new file mode 100755 index 000000000..5cbdc49cd --- /dev/null +++ b/packages/db/migrations/0068.do.add_delete_cascade_on_upload_files.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: add_delete_cascade_on_upload_files +-- Description: Add delete cascade on user_id field on upload_files table + +BEGIN; + +ALTER TABLE omnivore.upload_files + DROP CONSTRAINT upload_files_user_id_fkey, + ADD CONSTRAINT upload_files_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES omnivore.user(id) + ON DELETE CASCADE; + +COMMIT; diff --git a/packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql b/packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql new file mode 100755 index 000000000..a4c0727a7 --- /dev/null +++ b/packages/db/migrations/0068.undo.add_delete_cascade_on_upload_files.sql @@ -0,0 +1,13 @@ +-- Type: UNDO +-- Name: add_delete_cascade_on_upload_files +-- Description: Add delete cascade on user_id field on upload_files table + +BEGIN; + +ALTER TABLE omnivore.upload_files + DROP CONSTRAINT upload_files_user_id_fkey, + ADD CONSTRAINT upload_files_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES omnivore.user(id); + +COMMIT; diff --git a/packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql b/packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql new file mode 100755 index 000000000..0249065f3 --- /dev/null +++ b/packages/db/migrations/0069.do.add_delete_cascade_on_upload_file_id_to_pages.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: add_delete_cascade_on_upload_file_id_to_pages +-- Description: Add delete cascade on upload_file_id field on pages table + +BEGIN; + +ALTER TABLE omnivore.pages + DROP CONSTRAINT article_upload_file_id_fkey, + ADD CONSTRAINT pages_upload_file_id_fkey + FOREIGN KEY (upload_file_id) + REFERENCES omnivore.upload_files (id) + ON DELETE CASCADE; + +COMMIT; diff --git a/packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql b/packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql new file mode 100755 index 000000000..d9006dcf0 --- /dev/null +++ b/packages/db/migrations/0069.undo.add_delete_cascade_on_upload_file_id_to_pages.sql @@ -0,0 +1,13 @@ +-- Type: UNDO +-- Name: add_delete_cascade_on_upload_file_id_to_pages +-- Description: Add delete cascade on upload_file_id field on pages table + +BEGIN; + +ALTER TABLE omnivore.pages + DROP CONSTRAINT pages_upload_file_id_fkey, + ADD CONSTRAINT article_upload_file_id_fkey + FOREIGN KEY (upload_file_id) + REFERENCES omnivore.upload_files (id); + +COMMIT; diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json index 704439c60..a3b9231db 100644 --- a/packages/inbound-email-handler/package.json +++ b/packages/inbound-email-handler/package.json @@ -22,7 +22,6 @@ "devDependencies": { "@types/json-bigint": "^1.0.1", "@types/node": "^14.11.2", - "@types/quoted-printable": "^1.0.0", "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { @@ -31,8 +30,8 @@ "@sendgrid/client": "^7.6.0", "@sentry/serverless": "^6.16.1", "axios": "^0.26.0", + "jsonwebtoken": "^8.5.1", "parse-headers": "^2.0.4", - "parse-multipart-data": "^1.2.1", - "quoted-printable": "^1.0.1" + "parse-multipart-data": "^1.2.1" } } diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 121c1194a..5b79b2cfc 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -13,6 +13,7 @@ import { isNewsletter, } from './newsletter' import { PubSub } from '@google-cloud/pubsub' +import { handlePdfAttachment } from './pdf' const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived' const pubsub = new PubSub() @@ -23,9 +24,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( const parsed: Record = {} for (const part of parts) { - const { name, data } = part + const { name, data, type, filename } = part if (name && data) { parsed[name] = data.toString() + } else if (type === 'application/pdf' && data) { + parsed['pdf-attachment-data'] = data.toString() + parsed['pdf-attachment-filename'] = filename + ? filename + : 'attachment.pdf' } else { console.log('no data or name for ', part) } @@ -76,7 +82,15 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( if (isConfirmationEmail(from)) { console.log('handleConfirmation', from, recipientAddress) await handleConfirmation(recipientAddress, subject) + } else if (parsed['pdf-attachment-filename']) { + console.log('handle PDF attachment', from, recipientAddress) + await handlePdfAttachment( + recipientAddress, + parsed['pdf-attachment-filename'], + parsed['pdf-attachment-data'] + ) } + // queue non-newsletter emails await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({ json: { diff --git a/packages/inbound-email-handler/src/pdf.ts b/packages/inbound-email-handler/src/pdf.ts new file mode 100644 index 000000000..45bd1bed3 --- /dev/null +++ b/packages/inbound-email-handler/src/pdf.ts @@ -0,0 +1,101 @@ +import axios, { AxiosResponse } from 'axios' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' + +const signToken = promisify(jwt.sign) + +type UploadResponse = { + id: string + url: string +} + +export const handlePdfAttachment = async ( + email: string, + fileName: string, + data: string +): Promise => { + console.log('handlePdfAttachment', email, fileName) + + try { + const uploadResult = await getUploadIdAndSignedUrl(email, fileName) + if (!uploadResult.url || !uploadResult.id) { + console.log('failed to create upload request', uploadResult) + return + } + await uploadToSignedUrl(uploadResult.url, data) + await createArticle(email, uploadResult.id) + } catch (error) { + console.error('handlePdfAttachment error', error) + } +} + +const getUploadIdAndSignedUrl = async ( + email: string, + fileName: string +): Promise => { + if (process.env.JWT_SECRET === undefined) { + throw new Error('JWT_SECRET is not defined') + } + const auth = await signToken(email, process.env.JWT_SECRET) + const data = { + fileName, + email, + } + + if (process.env.REST_BACKEND_ENDPOINT === undefined) { + throw new Error('REST_BACKEND_ENDPOINT is not defined') + } + const response = await axios.post( + `${process.env.REST_BACKEND_ENDPOINT}/svc/pdf-attachments/upload`, + data, + { + headers: { + Authorization: `${auth as string}`, + 'Content-Type': 'application/json', + }, + } + ) + return response.data as UploadResponse +} + +const uploadToSignedUrl = async ( + uploadUrl: string, + data: string +): Promise => { + return axios.put(uploadUrl, data, { + headers: { + 'Content-Type': 'application/pdf', + }, + maxBodyLength: 1000000000, + maxContentLength: 100000000, + }) +} + +const createArticle = async ( + email: string, + uploadFileId: string +): Promise => { + const data = { + email, + uploadFileId, + } + + if (process.env.JWT_SECRET === undefined) { + throw new Error('JWT_SECRET is not defined') + } + const auth = await signToken(email, process.env.JWT_SECRET) + + if (process.env.REST_BACKEND_ENDPOINT === undefined) { + throw new Error('REST_BACKEND_ENDPOINT is not defined') + } + return axios.post( + `${process.env.REST_BACKEND_ENDPOINT}/svc/pdf-attachments/create-article`, + data, + { + headers: { + Authorization: `${auth as string}`, + 'Content-Type': 'application/json', + }, + } + ) +} diff --git a/yarn.lock b/yarn.lock index cee55bb8d..b80fb2343 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4711,11 +4711,6 @@ resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== -"@types/quoted-printable@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@types/quoted-printable/-/quoted-printable-1.0.0.tgz#903f81f5d81698d361c176b5b52970cd2bc9868e" - integrity sha512-hgFjmHmgT5M8SvDVe+tMhiUb3xViwqkEAM/sTpWCpO0B2Z7RGAgwiQaxPcLVk4KLiZmqj7BMXZvaQQdX6uPM6A== - "@types/range-parser@*": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" @@ -14332,13 +14327,6 @@ quick-lru@^4.0.1: resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== -quoted-printable@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/quoted-printable/-/quoted-printable-1.0.1.tgz#9eebf5eb3d11eef022b264fd2d2b6b2bb3b84cc3" - integrity sha1-nuv16z0R7vAismT9LStrK7O4TMM= - dependencies: - utf8 "^2.1.0" - randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -16777,11 +16765,6 @@ user-home@^1.1.1: resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190" integrity sha1-K1viOjK2Onyd640PKNSFcko98ZA= -utf8@^2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96" - integrity sha1-H6DZJw6b6FDZsFAn9jUZv0ZFfZY= - util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"