From 6f11ccacb133e7ea651be190830174bf76a901fb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 26 Jul 2022 22:08:10 +0800 Subject: [PATCH] Save article from forwarding emails --- packages/api/src/routers/svc/emails.ts | 86 ++++++++++++------- packages/api/src/routers/svc/newsletters.ts | 2 +- .../api/src/services/save_newsletter_email.ts | 5 +- packages/api/src/utils/parser.ts | 8 ++ packages/api/test/routers/auth.test.ts | 5 +- packages/api/test/utils/parser.test.ts | 19 ++++ packages/inbound-email-handler/src/index.ts | 16 ++-- 7 files changed, 101 insertions(+), 40 deletions(-) diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index 01498f83a..0c166cdf6 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -1,12 +1,22 @@ import express from 'express' -import { readPushSubscription } from '../../datalayer/pubsub' +import { + createPubSubClient, + readPushSubscription, +} from '../../datalayer/pubsub' import { sendEmail } from '../../utils/sendEmail' import { analytics } from '../../utils/analytics' import { getNewsletterEmail } from '../../services/newsletters' import { env } from '../../env' -import { v4 as uuid } from 'uuid' -import { findNewsletterUrl, isProbablyNewsletter } from '../../utils/parser' +import { + findNewsletterUrl, + generateUniqueUrl, + getTitleFromEmailSubject, + isProbablyArticle, + isProbablyNewsletter, +} from '../../utils/parser' import { saveNewsletterEmail } from '../../services/save_newsletter_email' +import { saveEmail } from '../../services/save_email' +import { buildLogger } from '../../utils/logger' interface ForwardEmailMessage { from: string @@ -17,15 +27,17 @@ interface ForwardEmailMessage { unsubHttpUrl?: string } +const logger = buildLogger('app.dispatch') + export function emailsServiceRouter() { const router = express.Router() // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/forward', async (req, res) => { - console.log('forward') + logger.info('email forward router') const { message, expired } = readPushSubscription(req) - console.log('pubsub message:', message, 'expired:', expired) + logger.info('pubsub message:', message, 'expired:', expired) if (!message) { res.status(400).send('Bad Request') @@ -33,7 +45,7 @@ export function emailsServiceRouter() { } if (expired) { - console.log('discards expired message:', message) + logger.log('discards expired message:', message) res.status(200).send('Expired') return } @@ -48,39 +60,55 @@ export function emailsServiceRouter() { !('subject' in data) || !('html' in data) ) { - console.log('Invalid message') + logger.info('Invalid message') res.status(400).send('Bad Request') return } - if (await isProbablyNewsletter(data.html)) { - console.log('handling as newsletter', data) - await saveNewsletterEmail({ - email: data.to, - title: data.subject, - content: data.html, - author: data.from, - url: - (await findNewsletterUrl(data.html)) || - 'https://omnivore.app/no_url?q' + uuid(), - unsubMailTo: data.unsubMailTo, - unsubHttpUrl: data.unsubHttpUrl, - }) - res.status(200).send('Newsletter') - return - } - // get user from newsletter email const newsletterEmail = await getNewsletterEmail(data.to) if (!newsletterEmail) { - console.log('newsletter email not found', data.to) + logger.info('newsletter email not found', data.to) res.status(200).send('Not Found') return } + const user = newsletterEmail.user + const ctx = { pubsub: createPubSubClient(), uid: user.id } + + if (await isProbablyNewsletter(data.html)) { + logger.info('handling as newsletter', data) + await saveNewsletterEmail( + { + email: data.to, + title: data.subject, + content: data.html, + author: data.from, + url: (await findNewsletterUrl(data.html)) || generateUniqueUrl(), + unsubMailTo: data.unsubMailTo, + unsubHttpUrl: data.unsubHttpUrl, + newsletterEmail, + }, + ctx + ) + res.status(200).send('Newsletter') + return + } + + if (await isProbablyArticle(data.from, data.subject)) { + logger.info('handling as article', data) + await saveEmail(ctx, { + title: getTitleFromEmailSubject(data.subject), + author: data.from, + url: generateUniqueUrl(), + originalContent: data.html, + }) + res.status(200).send('Article') + return + } analytics.track({ - userId: newsletterEmail.user.id, + userId: user.id, event: 'non_newsletter_email_received', properties: { env: env.server.apiEnv, @@ -90,21 +118,21 @@ export function emailsServiceRouter() { // forward non-newsletter emails to the registered email address const result = await sendEmail({ from: env.sender.message, - to: newsletterEmail.user.email, + to: user.email, subject: `Fwd: ${data.subject}`, html: data.html, replyTo: data.from, }) if (!result) { - console.log('Email not forwarded', data) + logger.info('Email not forwarded', data) res.status(200).send('Failed to send email') return } res.status(200).send('Email forwarded') } catch (e) { - console.log(e) + logger.info(e) if (e instanceof SyntaxError) { // when message is not a valid json string res.status(400).send(e) diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index 463da1747..f76536098 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -96,7 +96,7 @@ export function newsletterServiceRouter() { const result = await saveNewsletterEmail(data) if (!result) { - console.log('Error createing newsletter link from data', data) + console.log('Error creating newsletter link from data', data) res.status(500).send('Error creating newsletter link') return } diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts index ec06d6a9e..ecf5177dd 100644 --- a/packages/api/src/services/save_newsletter_email.ts +++ b/packages/api/src/services/save_newsletter_email.ts @@ -11,6 +11,7 @@ import { getDeviceTokensByUserId } from './user_device_tokens' import { Page } from '../elastic/types' import { addLabelToPage } from './labels' import { saveSubscription } from './subscriptions' +import { NewsletterEmail } from '../entity/newsletter_email' interface NewsletterMessage { email: string @@ -20,6 +21,7 @@ interface NewsletterMessage { author: string unsubMailTo?: string unsubHttpUrl?: string + newsletterEmail?: NewsletterEmail } // Returns true if the link was created successfully. Can still fail to @@ -29,7 +31,8 @@ export const saveNewsletterEmail = async ( ctx?: SaveContext ): Promise => { // get user from newsletter email - const newsletterEmail = await getNewsletterEmail(data.email) + const newsletterEmail = + data.newsletterEmail || (await getNewsletterEmail(data.email)) if (!newsletterEmail) { console.log('newsletter email not found', data.email) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 7a31782fa..89fe4b018 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -18,6 +18,7 @@ import { parseHTML } from 'linkedom' import { getRepository } from '../entity/utils' import { User } from '../entity/user' import { ILike } from 'typeorm' +import { v4 as uuid } from 'uuid' const logger = buildLogger('utils.parse') @@ -559,3 +560,10 @@ export const isProbablyArticle = async ( }) return !!user || subject.includes(ARTICLE_PREFIX) } + +export const generateUniqueUrl = () => 'https://omnivore.app/no_url?q=' + uuid() + +export const getTitleFromEmailSubject = (subject: string) => { + const title = subject.replace(ARTICLE_PREFIX, '') + return title.trim() +} diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 526ccce02..37b3190a9 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -1,6 +1,5 @@ import { createTestUser, deleteTestUser } from '../db' import { generateFakeUuid, request } from '../util' -import { expect } from 'chai' import { StatusType } from '../../src/datalayer/user/model' import { getRepository } from '../../src/entity/utils' import { User } from '../../src/entity/user' @@ -13,6 +12,10 @@ import { generateVerificationToken, hashPassword, } from '../../src/utils/auth' +import sinonChai from 'sinon-chai' +import chai, { expect } from 'chai' + +chai.use(sinonChai) describe('auth router', () => { const route = '/api/auth' diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 9daa0cd2a..5c79e8619 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -5,6 +5,8 @@ import 'chai/register-should' import fs from 'fs' import { findNewsletterUrl, + generateUniqueUrl, + getTitleFromEmailSubject, isProbablyArticle, isProbablyNewsletter, parsePageMetadata, @@ -160,3 +162,20 @@ describe('isProbablyArticle', () => { expect(await isProbablyArticle('test-email', subject)).to.be.true }) }) + +describe('generateUniqueUrl', () => { + it('generates a unique URL', () => { + const url1 = generateUniqueUrl() + const url2 = generateUniqueUrl() + + expect(url1).to.not.eql(url2) + }) +}) + +describe('getTitleFromEmailSubject', () => { + it('returns the title from the email subject', () => { + const title = 'test subject' + const subject = `omnivore: ${title}` + expect(getTitleFromEmailSubject(subject)).to.eql(title) + }) +}) diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index df87c1af8..5f3b887b3 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -119,11 +119,11 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( // queue non-newsletter emails await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({ json: { - from: from, + from, to: recipientAddress, - subject: subject, - html: html, - text: text, + subject, + html, + text, unsubMailTo: unsubscribe.mailTo, unsubHttpUrl: unsubscribe.httpUrl, }, @@ -140,11 +140,11 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( // queue error emails await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({ json: { - from: from, + from, to: recipientAddress, - subject: subject, - html: html, - text: text, + subject, + html, + text, }, }) }