From d89a307aca008d2eb3d7a9f47d199e2bd5fd52d1 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 4 Jan 2023 18:15:26 +0800 Subject: [PATCH 1/8] Send email to user after a bulk import completes --- packages/api/src/routers/user_router.ts | 63 +++++++++++++++++++++++++ packages/api/src/server.ts | 2 + packages/import-handler/src/index.ts | 53 ++++++++++++++++++--- packages/import-handler/src/task.ts | 6 ++- 4 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 packages/api/src/routers/user_router.ts diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts new file mode 100644 index 000000000..13f668bc1 --- /dev/null +++ b/packages/api/src/routers/user_router.ts @@ -0,0 +1,63 @@ +import express from 'express' +import { sendEmail } from '../utils/sendEmail' +import { env } from '../env' +import { buildLogger } from '../utils/logger' +import { getRepository } from '../entity/utils' +import { User } from '../entity/user' +import { getClaimsByToken } from '../utils/auth' + +const logger = buildLogger('app.dispatch') + +export function userRouter() { + const router = express.Router() + + router.post('/email', async (req, res) => { + logger.info('email to-user router') + + const token = req?.cookies?.auth || req?.headers?.authorization + const claims = await getClaimsByToken(token) + if (!claims) { + res.status(401).send('UNAUTHORIZED') + return + } + + const from = process.env.SENDER_MESSAGE + const { body, subject } = req.body as { + body?: string + subject?: string + } + + if (!subject || !body || !from) { + console.log(subject, body, from) + res.status(400).send('Bad Request') + return + } + + try { + const user = await getRepository(User).findOneBy({ id: claims.uid }) + if (!user) { + res.status(400).send('Bad Request') + return + } + + const result = await sendEmail({ + from: env.sender.message, + to: user.email, + subject: subject, + text: body, + }) + + if (!result) { + logger.error('Email not sent to user') + res.status(500).send('Failed to send email') + return + } + + res.status(200).send('Email sent to user') + } catch (e) { + logger.info(e) + } + }) + + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index adae47e87..bb15c3d3e 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -48,6 +48,7 @@ import { integrationsServiceRouter } from './routers/svc/integrations' import { textToSpeechRouter } from './routers/text_to_speech' import * as httpContext from 'express-http-context' import { notificationRouter } from './routers/notification_router' +import { userRouter } from './routers/user_router' const PORT = process.env.PORT || 4000 @@ -129,6 +130,7 @@ export const createApp = (): { app.use('/api/auth', authRouter()) app.use('/api/page', pageRouter()) + app.use('/api/user', userRouter()) app.use('/api/article', articleRouter()) app.use('/api/mobile-auth', mobileAuthRouter()) app.use('/api/text-to-speech', textToSpeechRouter()) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 6815f8ff3..054af1ee0 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -10,6 +10,12 @@ import { Stream } from 'node:stream' import { v4 as uuid } from 'uuid' import { createCloudTask } from './task' +import axios, { AxiosResponse } from 'axios' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' + +const signToken = promisify(jwt.sign) + const storage = new Storage() interface StorageEventData { @@ -50,6 +56,30 @@ const importURL = async ( }) } +const importCompletedTask = async (userId: string, urlsEnqueued: number) => { + if (!process.env.JWT_SECRET) { + throw 'Envrionment not setup correctly' + } + + const exp = Math.floor(Date.now() / 1000) + 60 * 60 * 24 // 1 day + const authToken = await signToken( + { uid: userId, exp }, + process.env.JWT_SECRET + ) + const headers = { + Authorization: `auth=${authToken}`, + } + + createCloudTask( + { + userId, + subject: 'Your Omnivore import has completed processing', + body: `${urlsEnqueued} URLs have been pcoessed and should be available in your library.`, + }, + headers + ) +} + const handlerForFile = (name: string): importHandlerFunc | undefined => { const fileName = path.parse(name).name if (fileName.startsWith('MATTER')) { @@ -79,21 +109,30 @@ export const importHandler: EventFunction = async (event, context) => { return } + const regex = new RegExp('imports/(.*?)/') + const groups = regex.exec(data.name) + if (!groups || groups.length < 2) { + console.log('could not match file pattern: ', data.name) + return + } + const userId = [...groups][1] + if (!userId) { + console.log('could not extract userId from file name') + return + } + + let countImported = 0 await handler(stream, async (url): Promise => { try { // Imports are stored in the format imports//-.csv - const regex = new RegExp('imports/(.*?)/') - const groups = regex.exec(data.name) - if (!groups || groups.length < 2) { - console.log('could not match file pattern: ', data.name) - return - } - const userId = [...groups][1] const result = await importURL(userId, url, 'csv-importer') console.log('import url result', result) + countImported = countImported + 1 } catch (err) { console.log('error importing url', err) } }) + + await importCompletedTask(userId, countImported) } } diff --git a/packages/import-handler/src/task.ts b/packages/import-handler/src/task.ts index 2b2dc3e21..5d11a3440 100644 --- a/packages/import-handler/src/task.ts +++ b/packages/import-handler/src/task.ts @@ -10,7 +10,10 @@ type TaskPayload = { const cloudTask = new CloudTasksClient() -export const createCloudTask = async (payload: TaskPayload) => { +export const createCloudTask = async ( + payload: unknown, + requestHeaders?: Record +) => { const queue = 'omnivore-import-queue' const location = process.env.GCP_LOCATION const project = process.env.GCP_PROJECT_ID @@ -40,6 +43,7 @@ export const createCloudTask = async (payload: TaskPayload) => { url: taskHandlerUrl, headers: { 'Content-Type': 'application/json', + ...requestHeaders, }, body, ...(serviceAccountEmail From dd9d00ec8d49c8e8ff25e0c4c544000a1ae47e2c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 4 Jan 2023 18:30:28 +0800 Subject: [PATCH 2/8] Linting fixes, wait on email send --- packages/import-handler/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 054af1ee0..ff7b63ae0 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -62,15 +62,15 @@ const importCompletedTask = async (userId: string, urlsEnqueued: number) => { } const exp = Math.floor(Date.now() / 1000) + 60 * 60 * 24 // 1 day - const authToken = await signToken( + const authToken = (await signToken( { uid: userId, exp }, process.env.JWT_SECRET - ) + )) as string const headers = { Authorization: `auth=${authToken}`, } - createCloudTask( + return createCloudTask( { userId, subject: 'Your Omnivore import has completed processing', From 7b2bc21b16236547921328de27d0e1fb04f5a8a4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 4 Jan 2023 21:45:41 +0800 Subject: [PATCH 3/8] Add cors config, handle exception response --- packages/api/src/routers/article_router.ts | 3 --- packages/api/src/routers/user_router.ts | 18 +++++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index f54222b36..f20a4d29a 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -1,8 +1,5 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ import express from 'express' import { CreateArticleErrorCode } from '../generated/graphql' import { isSiteBlockedForParse } from '../utils/blocked' diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index 13f668bc1..09ab98004 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { sendEmail } from '../utils/sendEmail' import { env } from '../env' @@ -5,57 +9,53 @@ import { buildLogger } from '../utils/logger' import { getRepository } from '../entity/utils' import { User } from '../entity/user' import { getClaimsByToken } from '../utils/auth' +import { corsConfig } from '../utils/corsConfig' +import cors from 'cors' const logger = buildLogger('app.dispatch') export function userRouter() { const router = express.Router() - router.post('/email', async (req, res) => { + router.post('/email', cors(corsConfig), async (req, res) => { logger.info('email to-user router') - - const token = req?.cookies?.auth || req?.headers?.authorization + const token = req?.headers?.authorization const claims = await getClaimsByToken(token) if (!claims) { res.status(401).send('UNAUTHORIZED') return } - const from = process.env.SENDER_MESSAGE const { body, subject } = req.body as { body?: string subject?: string } - if (!subject || !body || !from) { console.log(subject, body, from) res.status(400).send('Bad Request') return } - try { const user = await getRepository(User).findOneBy({ id: claims.uid }) if (!user) { res.status(400).send('Bad Request') return } - const result = await sendEmail({ from: env.sender.message, to: user.email, subject: subject, text: body, }) - if (!result) { logger.error('Email not sent to user') res.status(500).send('Failed to send email') return } - res.status(200).send('Email sent to user') } catch (e) { logger.info(e) + res.status(500).send('Email sent to user') } }) From cfa3b5de89697dcd8c5291349215afe6e08646ba Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 5 Jan 2023 10:25:10 +0800 Subject: [PATCH 4/8] Remove unneeded linting rules --- packages/api/src/routers/user_router.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index 09ab98004..227b0ef74 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -1,7 +1,5 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { sendEmail } from '../utils/sendEmail' import { env } from '../env' From 37dc0336fa3afb9bb2ae148b9248e04484ac97ea Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 5 Jan 2023 11:13:58 +0800 Subject: [PATCH 5/8] Set Authorization to authToken --- packages/import-handler/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index ff7b63ae0..aa0dcb168 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -67,7 +67,7 @@ const importCompletedTask = async (userId: string, urlsEnqueued: number) => { process.env.JWT_SECRET )) as string const headers = { - Authorization: `auth=${authToken}`, + Authorization: authToken, } return createCloudTask( From f177a0d1db3459cc615dc06513a98f12eab75d63 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 5 Jan 2023 11:14:57 +0800 Subject: [PATCH 6/8] Remove unused type --- packages/import-handler/src/task.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/import-handler/src/task.ts b/packages/import-handler/src/task.ts index 5d11a3440..da6c971b3 100644 --- a/packages/import-handler/src/task.ts +++ b/packages/import-handler/src/task.ts @@ -1,13 +1,6 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ import { CloudTasksClient, protos } from '@google-cloud/tasks' -type TaskPayload = { - url: string - userId: string - saveRequestId: string - source: string -} - const cloudTask = new CloudTasksClient() export const createCloudTask = async ( From 750fdd6b4542eabe124ba4421c5dceda6f23be4b Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 5 Jan 2023 11:48:54 +0800 Subject: [PATCH 7/8] Remove unused packages --- packages/import-handler/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index a4567dfb2..48c341c43 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -29,8 +29,6 @@ "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^3.0.5", "@types/express": "^4.17.13", - "axios": "^0.27.2", - "concurrently": "^7.0.0", "csv-parser": "^3.0.0", "nodemon": "^2.0.15" } From 687d126bd37b416771b4e36c91bc6a8f6b1725c5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 5 Jan 2023 11:49:16 +0800 Subject: [PATCH 8/8] Fix the task URLs, send an email if no URLs are parsed for import --- packages/import-handler/src/index.ts | 42 +++++++++++++++++++--------- packages/import-handler/src/task.ts | 11 +++++++- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index aa0dcb168..4012827cc 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -8,9 +8,8 @@ import * as path from 'path' import { importMatterHistory } from './matterHistory' import { Stream } from 'node:stream' import { v4 as uuid } from 'uuid' -import { createCloudTask } from './task' +import { CONTENT_FETCH_URL, createCloudTask, EMAIL_USER_URL } from './task' -import axios, { AxiosResponse } from 'axios' import { promisify } from 'util' import * as jwt from 'jsonwebtoken' @@ -48,7 +47,7 @@ const importURL = async ( url: URL, source: string ): Promise => { - return createCloudTask({ + return createCloudTask(CONTENT_FETCH_URL, { userId, source, url: url.toString(), @@ -56,7 +55,7 @@ const importURL = async ( }) } -const importCompletedTask = async (userId: string, urlsEnqueued: number) => { +const createEmailCloudTask = async (userId: string, payload: unknown) => { if (!process.env.JWT_SECRET) { throw 'Envrionment not setup correctly' } @@ -70,14 +69,25 @@ const importCompletedTask = async (userId: string, urlsEnqueued: number) => { Authorization: authToken, } - return createCloudTask( - { - userId, - subject: 'Your Omnivore import has completed processing', - body: `${urlsEnqueued} URLs have been pcoessed and should be available in your library.`, - }, - headers - ) + return createCloudTask(EMAIL_USER_URL, payload, headers) +} + +const sendImportFailedEmail = async (userId: string) => { + return createEmailCloudTask(userId, { + subject: 'Your Omnivore import failed.', + body: `There was an error importing your file. Please ensure you uploaded the correct file type, if you need help, please email feedback@omnivore.app`, + }) +} + +const sendImportCompletedEmail = async ( + userId: string, + urlsEnqueued: number, + urlsFailed: number +) => { + return createEmailCloudTask(userId, { + subject: 'Your Omnivore import has completed processing', + body: `${urlsEnqueued} URLs have been pcoessed and should be available in your library. ${urlsFailed} URLs failed to be parsed.`, + }) } const handlerForFile = (name: string): importHandlerFunc | undefined => { @@ -121,6 +131,7 @@ export const importHandler: EventFunction = async (event, context) => { return } + let countFailed = 0 let countImported = 0 await handler(stream, async (url): Promise => { try { @@ -130,9 +141,14 @@ export const importHandler: EventFunction = async (event, context) => { countImported = countImported + 1 } catch (err) { console.log('error importing url', err) + countFailed = countFailed + 1 } }) - await importCompletedTask(userId, countImported) + if (countImported < 1) { + await sendImportFailedEmail(userId) + } else { + await sendImportCompletedEmail(userId, countImported, countFailed) + } } } diff --git a/packages/import-handler/src/task.ts b/packages/import-handler/src/task.ts index da6c971b3..e63be6bd3 100644 --- a/packages/import-handler/src/task.ts +++ b/packages/import-handler/src/task.ts @@ -3,14 +3,23 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks' const cloudTask = new CloudTasksClient() +export const EMAIL_USER_URL = (() => { + if (!process.env.INTERNAL_SVC_ENDPOINT) { + throw `Environment not configured correctly, no SVC endpoint` + } + return (process.env.INTERNAL_SVC_ENDPOINT ?? '') + '/api/user/email' +})() + +export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL + export const createCloudTask = async ( + taskHandlerUrl: string | undefined, payload: unknown, requestHeaders?: Record ) => { const queue = 'omnivore-import-queue' const location = process.env.GCP_LOCATION const project = process.env.GCP_PROJECT_ID - const taskHandlerUrl = process.env.CONTENT_FETCH_GCF_URL if (!project || !location || !queue || !taskHandlerUrl) { throw `Environment not configured: ${project}, ${location}, ${queue}, ${taskHandlerUrl}`