diff --git a/packages/api/index_settings.json b/packages/api/index_settings.json index 2e540dcde..8f7ff2630 100644 --- a/packages/api/index_settings.json +++ b/packages/api/index_settings.json @@ -97,6 +97,9 @@ }, "siteName": { "type": "text" + }, + "subscription": { + "type": "keyword" } } } diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index c550367a8..192a4acf0 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -191,6 +191,9 @@ export interface Page { _id?: string siteIcon?: string highlights?: Highlight[] + subscription?: string + unsubMailTo?: string + unsubHttpUrl?: string } export interface SearchItem { diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 6f04b5d6c..28bc0886d 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -20,6 +20,8 @@ export type SaveEmailInput = { url: string title: string author: string + unsubMailTo?: string + unsubHttpUrl?: string } export const saveEmail = async ( @@ -66,6 +68,9 @@ export const saveEmail = async ( createdAt: new Date(), readingProgressAnchorIndex: 0, readingProgressPercent: 0, + subscription: input.author, + unsubHttpUrl: input.unsubHttpUrl, + unsubMailTo: input.unsubMailTo, } const page = await getPageByParam({ userId: ctx.uid, url: articleToSave.url }) diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts index df879df1c..c68e8147f 100644 --- a/packages/api/src/services/save_newsletter_email.ts +++ b/packages/api/src/services/save_newsletter_email.ts @@ -10,6 +10,7 @@ import { SaveContext, saveEmail, SaveEmailInput } from './save_email' import { getDeviceTokensByUserId } from './user_device_tokens' import { Page } from '../elastic/types' import { addLabelToPage } from './labels' +import { saveSubscription } from './subscriptions' interface NewsletterMessage { email: string @@ -17,6 +18,8 @@ interface NewsletterMessage { url: string title: string author: string + unsubMailTo?: string + unsubHttpUrl?: string } // Returns true if the link was created successfully. Can still fail to @@ -54,6 +57,8 @@ export const saveNewsletterEmail = async ( originalContent: data.content, title: data.title, author: data.author, + unsubMailTo: data.unsubMailTo, + unsubHttpUrl: data.unsubHttpUrl, } const page = await saveEmail(saveCtx, input) @@ -62,14 +67,23 @@ export const saveNewsletterEmail = async ( return false } - // add newsletters label to page + // creates or updates subscription + const subscription = await saveSubscription( + newsletterEmail.user.id, + data.author, + data.unsubMailTo, + data.unsubHttpUrl + ) + console.log('subscription', subscription) + + // adds newsletters label to page const result = await addLabelToPage(saveCtx, page.id, { name: 'Newsletter', color: '#07D2D1', }) console.log('newsletter label added:', result) - // send push notification + // sends push notification const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id) if (!deviceTokens) { diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts new file mode 100644 index 000000000..9b0c36da0 --- /dev/null +++ b/packages/api/src/services/subscriptions.ts @@ -0,0 +1,33 @@ +import { Subscription } from '../entity/subscription' +import { getRepository } from '../entity/utils' +import { SubscriptionStatus } from '../generated/graphql' + +export const saveSubscription = async ( + userId: string, + name: string, + unsubscribeMailTo?: string, + unsubscribeHttpUrl?: string +): Promise => { + const subscription = await getRepository(Subscription).findOneBy({ + name, + user: { id: userId }, + }) + + if (subscription) { + // if subscription already exists, updates updatedAt + subscription.updatedAt = new Date() + subscription.status = SubscriptionStatus.Active + unsubscribeMailTo && (subscription.unsubscribeMailTo = unsubscribeMailTo) + unsubscribeHttpUrl && (subscription.unsubscribeHttpUrl = unsubscribeHttpUrl) + return getRepository(Subscription).save(subscription) + } + + // create new subscription + return getRepository(Subscription).save({ + name, + user: { id: userId }, + status: SubscriptionStatus.Active, + unsubscribeHttpUrl, + unsubscribeMailTo, + }) +} diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 71c1d018b..3b9cd48b5 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -29,12 +29,12 @@ const NEWSLETTER_HANDLERS = [ ] export const getNewsletterHandler = ( - rawUrl: string, + postHeader: string, from: string, - unSubRawUrl: string + unSubHeader: string ): NewsletterHandler | undefined => { return NEWSLETTER_HANDLERS.find((h) => { - return h.isNewsletter(rawUrl, from, unSubRawUrl) + return h.isNewsletter(postHeader, from, unSubHeader) }) } @@ -72,22 +72,29 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( const recipientAddress = forwardedAddress ? forwardedAddress.toString() : parsed.to - const rawUrl = headers['list-post'] ? headers['list-post'].toString() : '' - const unSubRawUrl = headers['list-unsubscribe'] + const postHeader = headers['list-post'] + ? headers['list-post'].toString() + : '' + const unSubHeader = headers['list-unsubscribe'] ? headers['list-unsubscribe'].toString() : '' // check if it is a forwarding confirmation email or newsletter - const newsletterHandler = getNewsletterHandler(rawUrl, from, unSubRawUrl) + const newsletterHandler = getNewsletterHandler( + postHeader, + from, + unSubHeader + ) try { if (newsletterHandler) { console.log('handleNewsletter', from, recipientAddress) await newsletterHandler.handleNewsletter( recipientAddress, html, - rawUrl, + postHeader, subject, - from + from, + unSubHeader ) } else { console.log('non-newsletter email from:', from, recipientAddress) diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts index 1e96a70bd..d20795f92 100644 --- a/packages/inbound-email-handler/src/newsletter.ts +++ b/packages/inbound-email-handler/src/newsletter.ts @@ -9,19 +9,26 @@ const EMAIL_FORWARDING_SENDER_ADDRESSES = [ 'Gmail Team ', ] const CONFIRMATION_CODE_PATTERN = /^\(#\d+\)/ +const UNSUBSCRIBE_HTTP_URL_PATTERN = /<(https?:\/\/[^>]*)>/ +const UNSUBSCRIBE_MAIL_TO_PATTERN = /]*)>/ + +interface Unsubscribe { + mailTo?: string + httpUrl?: string +} export class NewsletterHandler { protected senderRegex = /NEWSLETTER_SENDER_REGEX/ protected urlRegex = /NEWSLETTER_URL_REGEX/ protected defaultUrl = 'NEWSLETTER_DEFAULT_URL' - isNewsletter(rawUrl: string, from: string, unSubRawUrl: string): boolean { + isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { // Axios newsletter is from const re = new RegExp(this.senderRegex) - return re.test(from) && (!!rawUrl || !!unSubRawUrl) + return re.test(from) && (!!postHeader || !!unSubHeader) } - getNewsletterUrl(_rawUrl: string, html: string): string | undefined { + parseNewsletterUrl(_postHeader: string, html: string): string | undefined { // get newsletter url from html const matches = html.match(this.urlRegex) if (matches) { @@ -30,7 +37,7 @@ export class NewsletterHandler { return undefined } - getAuthor(from: string): string { + parseAuthor(from: string): string { // get author name from email // e.g. 'Jackson Harper from Omnivore App ' // or 'Mike Allen ' @@ -41,14 +48,24 @@ export class NewsletterHandler { return from } + parseUnsubscribe(unSubHeader: string): Unsubscribe { + // parse list-unsubscribe header + // e.g. List-Unsubscribe: , + return { + mailTo: unSubHeader.match(UNSUBSCRIBE_MAIL_TO_PATTERN)?.[1], + httpUrl: unSubHeader.match(UNSUBSCRIBE_HTTP_URL_PATTERN)?.[1], + } + } + async handleNewsletter( email: string, html: string, - rawUrl: string, + postHeader: string, title: string, - from: string + from: string, + unSubHeader: string ): Promise { - console.log('handleNewsletter', email, rawUrl, title, from) + console.log('handleNewsletter', email, postHeader, title, from) if (!email || !html || !title || !from) { console.log('invalid newsletter email') @@ -58,17 +75,20 @@ export class NewsletterHandler { // fallback to default url if newsletter url does not exist // assign a random uuid to the default url to avoid duplicate url const url = - this.getNewsletterUrl(rawUrl, html) || + this.parseNewsletterUrl(postHeader, html) || `${this.defaultUrl}?source=newsletters&id=${uuidv4()}` - const author = this.getAuthor(from) || 'Unknown' - + const author = this.parseAuthor(from) || 'Unknown' + const unsubscribe = this.parseUnsubscribe(unSubHeader) const message = { - email: email, + email, content: html, - url: url, - title: title, - author: author, + url, + title, + author, + unsubMailTo: unsubscribe.mailTo || '', + unsubHttpUrl: unsubscribe.httpUrl || '', } + return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message) } } diff --git a/packages/inbound-email-handler/src/substack-handler.ts b/packages/inbound-email-handler/src/substack-handler.ts index 4aa888383..9c676f25c 100644 --- a/packages/inbound-email-handler/src/substack-handler.ts +++ b/packages/inbound-email-handler/src/substack-handler.ts @@ -7,15 +7,19 @@ export class SubstackHandler extends NewsletterHandler { this.defaultUrl = 'https://www.substack.com' } - getNewsletterUrl(rawUrl: string, _html: string): string | undefined { + getNewsletterUrl(postHeader: string, _html: string): string | undefined { // raw SubStack newsletter url is like // we need to get the real url from the raw url - return addressparser(rawUrl).length > 0 - ? addressparser(rawUrl)[0].name + return addressparser(postHeader).length > 0 + ? addressparser(postHeader)[0].name : undefined } - isNewsletter(rawUrl: string, _from: string, _unSubRawUrl: string): boolean { - return !!rawUrl + isNewsletter( + postHeader: string, + _from: string, + _unSubHeader: string + ): boolean { + return !!postHeader } } diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts index 9710e7407..81d6f31d6 100644 --- a/packages/inbound-email-handler/test/newsletter.test.ts +++ b/packages/inbound-email-handler/test/newsletter.test.ts @@ -92,7 +92,7 @@ describe('Newsletter email test', () => { const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app' const html = `View in browser at ${url}` - expect(new AxiosHandler().getNewsletterUrl('', html)).to.equal(url) + expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url) }) it('returns url when email is from Bloomberg', () => { @@ -103,7 +103,7 @@ describe('Newsletter email test', () => { ` - expect(new BloombergHandler().getNewsletterUrl('', html)).to.equal(url) + expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url) }) it('returns url when email is from Golang Weekly', () => { @@ -112,21 +112,42 @@ describe('Newsletter email test', () => { Read on the Web ` - expect(new GolangHandler().getNewsletterUrl('', html)).to.equal(url) + expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url) }) }) describe('get author from email address', () => { it('returns author when email is from Substack', () => { const from = 'Jackson Harper from Omnivore App ' - expect(new NewsletterHandler().getAuthor(from)).to.equal( + expect(new NewsletterHandler().parseAuthor(from)).to.equal( 'Jackson Harper from Omnivore App' ) }) it('returns author when email is from Axios', () => { const from = 'Mike Allen ' - expect(new NewsletterHandler().getAuthor(from)).to.equal('Mike Allen') + expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen') + }) + }) + + describe('get unsubscribe from header', () => { + const mailTo = 'unsub@omnivore.com' + const httpUrl = 'https://omnivore.com/unsubscribe' + + it('returns mail to address if exists', () => { + const header = `, ` + + expect(new NewsletterHandler().parseUnsubscribe(header).mailTo).to.equal( + mailTo + ) + }) + + it('returns http url if exists', () => { + const header = `<${httpUrl}>` + + expect(new NewsletterHandler().parseUnsubscribe(header).httpUrl).to.equal( + httpUrl + ) }) }) })