Save article from forwarding emails

packages/api/src/routers/svc/emails.ts

@@ -1,12 +1,22 @@
 import express from 'express'
-import { readPushSubscription } from '../../datalayer/pubsub'
+import {
+  createPubSubClient,
+  readPushSubscription,
+} from '../../datalayer/pubsub'
 import { sendEmail } from '../../utils/sendEmail'
 import { analytics } from '../../utils/analytics'
 import { getNewsletterEmail } from '../../services/newsletters'
 import { env } from '../../env'
-import { v4 as uuid } from 'uuid'
-import { findNewsletterUrl, isProbablyNewsletter } from '../../utils/parser'
+import {
+  findNewsletterUrl,
+  generateUniqueUrl,
+  getTitleFromEmailSubject,
+  isProbablyArticle,
+  isProbablyNewsletter,
+} from '../../utils/parser'
 import { saveNewsletterEmail } from '../../services/save_newsletter_email'
+import { saveEmail } from '../../services/save_email'
+import { buildLogger } from '../../utils/logger'
 
 interface ForwardEmailMessage {
   from: string
@@ -17,15 +27,17 @@ interface ForwardEmailMessage {
   unsubHttpUrl?: string
 }
 
+const logger = buildLogger('app.dispatch')
+
 export function emailsServiceRouter() {
   const router = express.Router()
 
   // eslint-disable-next-line @typescript-eslint/no-misused-promises
   router.post('/forward', async (req, res) => {
-    console.log('forward')
+    logger.info('email forward router')
 
     const { message, expired } = readPushSubscription(req)
-    console.log('pubsub message:', message, 'expired:', expired)
+    logger.info('pubsub message:', message, 'expired:', expired)
 
     if (!message) {
       res.status(400).send('Bad Request')
@@ -33,7 +45,7 @@ export function emailsServiceRouter() {
     }
 
     if (expired) {
-      console.log('discards expired message:', message)
+      logger.log('discards expired message:', message)
       res.status(200).send('Expired')
       return
     }
@@ -48,39 +60,55 @@ export function emailsServiceRouter() {
         !('subject' in data) ||
         !('html' in data)
       ) {
-        console.log('Invalid message')
+        logger.info('Invalid message')
         res.status(400).send('Bad Request')
         return
       }
 
-      if (await isProbablyNewsletter(data.html)) {
-        console.log('handling as newsletter', data)
-        await saveNewsletterEmail({
-          email: data.to,
-          title: data.subject,
-          content: data.html,
-          author: data.from,
-          url:
-            (await findNewsletterUrl(data.html)) ||
-            'https://omnivore.app/no_url?q' + uuid(),
-          unsubMailTo: data.unsubMailTo,
-          unsubHttpUrl: data.unsubHttpUrl,
-        })
-        res.status(200).send('Newsletter')
-        return
-      }
-
       // get user from newsletter email
       const newsletterEmail = await getNewsletterEmail(data.to)
 
       if (!newsletterEmail) {
-        console.log('newsletter email not found', data.to)
+        logger.info('newsletter email not found', data.to)
         res.status(200).send('Not Found')
         return
       }
+      const user = newsletterEmail.user
+      const ctx = { pubsub: createPubSubClient(), uid: user.id }
+
+      if (await isProbablyNewsletter(data.html)) {
+        logger.info('handling as newsletter', data)
+        await saveNewsletterEmail(
+          {
+            email: data.to,
+            title: data.subject,
+            content: data.html,
+            author: data.from,
+            url: (await findNewsletterUrl(data.html)) || generateUniqueUrl(),
+            unsubMailTo: data.unsubMailTo,
+            unsubHttpUrl: data.unsubHttpUrl,
+            newsletterEmail,
+          },
+          ctx
+        )
+        res.status(200).send('Newsletter')
+        return
+      }
+
+      if (await isProbablyArticle(data.from, data.subject)) {
+        logger.info('handling as article', data)
+        await saveEmail(ctx, {
+          title: getTitleFromEmailSubject(data.subject),
+          author: data.from,
+          url: generateUniqueUrl(),
+          originalContent: data.html,
+        })
+        res.status(200).send('Article')
+        return
+      }
 
       analytics.track({
-        userId: newsletterEmail.user.id,
+        userId: user.id,
         event: 'non_newsletter_email_received',
         properties: {
           env: env.server.apiEnv,
@@ -90,21 +118,21 @@ export function emailsServiceRouter() {
       // forward non-newsletter emails to the registered email address
       const result = await sendEmail({
         from: env.sender.message,
-        to: newsletterEmail.user.email,
+        to: user.email,
         subject: `Fwd: ${data.subject}`,
         html: data.html,
         replyTo: data.from,
       })
 
       if (!result) {
-        console.log('Email not forwarded', data)
+        logger.info('Email not forwarded', data)
         res.status(200).send('Failed to send email')
         return
       }
 
       res.status(200).send('Email forwarded')
     } catch (e) {
-      console.log(e)
+      logger.info(e)
       if (e instanceof SyntaxError) {
         // when message is not a valid json string
         res.status(400).send(e)

packages/api/src/routers/svc/newsletters.ts

@@ -96,7 +96,7 @@ export function newsletterServiceRouter() {
 
       const result = await saveNewsletterEmail(data)
       if (!result) {
-        console.log('Error createing newsletter link from data', data)
+        console.log('Error creating newsletter link from data', data)
         res.status(500).send('Error creating newsletter link')
         return
       }

packages/api/src/services/save_newsletter_email.ts

@@ -11,6 +11,7 @@ import { getDeviceTokensByUserId } from './user_device_tokens'
 import { Page } from '../elastic/types'
 import { addLabelToPage } from './labels'
 import { saveSubscription } from './subscriptions'
+import { NewsletterEmail } from '../entity/newsletter_email'
 
 interface NewsletterMessage {
   email: string
@@ -20,6 +21,7 @@ interface NewsletterMessage {
   author: string
   unsubMailTo?: string
   unsubHttpUrl?: string
+  newsletterEmail?: NewsletterEmail
 }
 
 // Returns true if the link was created successfully. Can still fail to
@@ -29,7 +31,8 @@ export const saveNewsletterEmail = async (
   ctx?: SaveContext
 ): Promise<boolean> => {
   // get user from newsletter email
-  const newsletterEmail = await getNewsletterEmail(data.email)
+  const newsletterEmail =
+    data.newsletterEmail || (await getNewsletterEmail(data.email))
 
   if (!newsletterEmail) {
     console.log('newsletter email not found', data.email)

packages/api/src/utils/parser.ts

@@ -18,6 +18,7 @@ import { parseHTML } from 'linkedom'
 import { getRepository } from '../entity/utils'
 import { User } from '../entity/user'
 import { ILike } from 'typeorm'
+import { v4 as uuid } from 'uuid'
 
 const logger = buildLogger('utils.parse')
 
@@ -559,3 +560,10 @@ export const isProbablyArticle = async (
   })
   return !!user || subject.includes(ARTICLE_PREFIX)
 }
+
+export const generateUniqueUrl = () => 'https://omnivore.app/no_url?q=' + uuid()
+
+export const getTitleFromEmailSubject = (subject: string) => {
+  const title = subject.replace(ARTICLE_PREFIX, '')
+  return title.trim()
+}

packages/api/test/routers/auth.test.ts

@@ -1,6 +1,5 @@
 import { createTestUser, deleteTestUser } from '../db'
 import { generateFakeUuid, request } from '../util'
-import { expect } from 'chai'
 import { StatusType } from '../../src/datalayer/user/model'
 import { getRepository } from '../../src/entity/utils'
 import { User } from '../../src/entity/user'
@@ -13,6 +12,10 @@ import {
   generateVerificationToken,
   hashPassword,
 } from '../../src/utils/auth'
+import sinonChai from 'sinon-chai'
+import chai, { expect } from 'chai'
+
+chai.use(sinonChai)
 
 describe('auth router', () => {
   const route = '/api/auth'

packages/api/test/utils/parser.test.ts

@@ -5,6 +5,8 @@ import 'chai/register-should'
 import fs from 'fs'
 import {
   findNewsletterUrl,
+  generateUniqueUrl,
+  getTitleFromEmailSubject,
   isProbablyArticle,
   isProbablyNewsletter,
   parsePageMetadata,
@@ -160,3 +162,20 @@ describe('isProbablyArticle', () => {
     expect(await isProbablyArticle('test-email', subject)).to.be.true
   })
 })
+
+describe('generateUniqueUrl', () => {
+  it('generates a unique URL', () => {
+    const url1 = generateUniqueUrl()
+    const url2 = generateUniqueUrl()
+
+    expect(url1).to.not.eql(url2)
+  })
+})
+
+describe('getTitleFromEmailSubject', () => {
+  it('returns the title from the email subject', () => {
+    const title = 'test subject'
+    const subject = `omnivore: ${title}`
+    expect(getTitleFromEmailSubject(subject)).to.eql(title)
+  })
+})

packages/inbound-email-handler/src/index.ts

@@ -119,11 +119,11 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         // queue non-newsletter emails
         await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
           json: {
-            from: from,
+            from,
             to: recipientAddress,
-            subject: subject,
-            html: html,
-            text: text,
+            subject,
+            html,
+            text,
             unsubMailTo: unsubscribe.mailTo,
             unsubHttpUrl: unsubscribe.httpUrl,
           },
@@ -140,11 +140,11 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         // queue error emails
         await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
           json: {
-            from: from,
+            from,
             to: recipientAddress,
-            subject: subject,
-            html: html,
-            text: text,
+            subject,
+            html,
+            text,
           },
         })
       }