[feat] Add an imap watcher for emails. (#4536)

[feat] Add an imap watcher for emails.

packages/imap-mail-watcher/.eslintrc

@@ -0,0 +1,13 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-unsafe-argument": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/strictNullChecks": "off",
+    "@typescript-eslint/no-unsafe-member-access": "off",
+    "@typescript-eslint/no-unsafe-assignment": "off"
+  }
+}

packages/imap-mail-watcher/.gitignore

@@ -0,0 +1,131 @@
+.idea/
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*

packages/imap-mail-watcher/Dockerfile

@@ -0,0 +1,40 @@
+ FROM node:22.12 as builder
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/imap-mail-watcher/src ./packages/imap-mail-watcher/src
+COPY /packages/imap-mail-watcher/package.json ./packages/imap-mail-watcher/package.json
+COPY /packages/imap-mail-watcher/tsconfig.json ./packages/imap-mail-watcher/tsconfig.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/imap-mail-watcher build
+
+FROM node:22.12 as runner
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/imap-mail-watcher/dist /app/packages/imap-mail-watcher/dist
+COPY --from=builder /app/packages/imap-mail-watcher/package.json /app/packages/imap-mail-watcher/package.json
+COPY --from=builder /app/packages/imap-mail-watcher/node_modules /app/packages/imap-mail-watcher/node_modules
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+CMD ["yarn", "workspace", "@omnivore/imap-mail-watcher", "start"]

packages/imap-mail-watcher/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@omnivore/imap-mail-watcher",
+  "version": "0.0.1",
+  "scripts": {
+    "build": "tsc",
+    "dev": "ts-node-dev --files src/index.ts",
+    "start": "node dist/watcher.js",
+    "lint": "eslint src --ext ts,js,tsx,jsx",
+    "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
+    "test:typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@omnivore/utils": "1.0.0",
+    "axios": "^1.7.7",
+    "imapflow": "^1.0.181",
+    "mailparser": "^3.7.1"
+  },
+  "devDependencies": {
+    "@types/imapflow": "^1.0.19",
+    "@types/axios": "^0.14.4",
+    "@types/express": "^5.0.0",
+    "rxjs": "^7.8.1",
+    "@types/html-to-text": "^9.0.2",
+    "@types/jsdom": "^21.1.3",
+    "@types/mailparser": "^3.4.5",
+    "@types/node": "^22.10.7",
+    "@types/pg": "^8.10.5",
+    "@types/pg-format": "^1.0.3",
+    "@types/urlsafe-base64": "^1.0.28",
+    "@types/uuid": "^9.0.1",
+    "@types/voca": "^1.4.3",
+    "ts-node": "^10.9.1",
+    "tslib": "^2.6.2",
+    "typescript": "^5.7.3"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

packages/imap-mail-watcher/src/env.ts

@@ -0,0 +1,53 @@
+interface WatcherEnv {
+  imap: {
+    host: string
+    port: number
+    auth: {
+      user: string
+      password: string
+    }
+  }
+  omnivoreEmail: string
+  apiKey: string
+  apiEndpoint: string,
+  waitTime: number
+}
+
+const envParser =
+  (env: { [key: string]: string | undefined }) =>
+  (varName: string, throwOnUndefined = false): string | undefined => {
+    const value = env[varName]
+    if (typeof value === 'string' && value) {
+      return value
+    }
+
+    if (throwOnUndefined) {
+      throw new Error(
+        `Missing ${varName} with a non-empty value in process environment`
+      )
+    }
+
+    return
+  }
+
+export function getEnv(): WatcherEnv {
+  const parse = envParser(process.env)
+  const imap = {
+    auth: {
+      user: parse('IMAP_USER')!,
+      password: parse('IMAP_PASSWORD')!,
+    },
+    host: parse('IMAP_HOST')!,
+    port: Number(parse('IMAP_PORT')!),
+  }
+
+  return {
+    apiKey: parse('WATCHER_API_KEY')!,
+    apiEndpoint: parse('WATCHER_API_ENDPOINT')!,
+    omnivoreEmail: parse('OMNIVORE_EMAIL')!,
+    waitTime: Number(parse('WAIT_TIME')),
+    imap,
+  }
+}
+
+export const env = getEnv()

packages/imap-mail-watcher/src/lib/emailApi.ts

@@ -0,0 +1,26 @@
+import axios from 'axios'
+import { env } from '../env'
+import { ParsedMail } from 'mailparser'
+import { EmailContents } from '../types/EmailContents'
+
+export const sendToEmailApi = (data: EmailContents) => {
+  console.log(`Sending mail with subject: ${data.subject} to ${data.to}`)
+  return axios.post(`${env.apiEndpoint}/mail`, data, {
+    headers: {
+      ['x-api-key']: env.apiKey,
+      'Content-Type': 'application/json',
+    },
+    timeout: 5000,
+  })
+}
+
+export const convertToMailObject = (it: ParsedMail): EmailContents => {
+  return {
+    from: it.from?.value[0]?.address || '',
+    to: env.omnivoreEmail,
+    subject: it.subject || '',
+    html: it.html || '',
+    text: it.text || '',
+    headers: it.headers,
+  }
+}

packages/imap-mail-watcher/src/lib/emailObserver.ts

@@ -0,0 +1,58 @@
+import { Observable } from 'rxjs'
+import { FetchMessageObject, ImapFlow, MailboxLockObject } from 'imapflow'
+import { env } from '../env'
+
+const client: ImapFlow = new ImapFlow({
+  host: env.imap.host,
+  port: env.imap.port,
+  secure: true,
+  auth: {
+    user: env.imap.auth.user,
+    pass: env.imap.auth.password,
+  },
+})
+
+export const emailObserver$ = new Observable<FetchMessageObject>(
+  (subscriber) => {
+    let loop = true
+    let lock: MailboxLockObject | null = null
+
+    process.nextTick(async () => {
+      while (loop) {
+        if (!client.usable) {
+          await client.connect()
+        }
+
+        if (!lock) {
+          lock = await client.getMailboxLock('INBOX')
+        }
+
+        // Retrieve all the mails that have yet to be seen.
+        const messages = await client.fetchAll(
+          { seen: false },
+          {
+            envelope: true,
+            source: true,
+            uid: true,
+          }
+        )
+
+        for (const message of messages) {
+          subscriber.next(message)
+          // Once we are done with this message, set it to seen.
+          await client.messageFlagsSet(
+            { uid: message.uid.toString(), seen: false },
+            ['\\Seen']
+          )
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, env.waitTime))
+      }
+    })
+
+    return () => {
+      loop = false
+      lock?.release()
+    }
+  }
+)

packages/imap-mail-watcher/src/types/EmailContents.ts

@@ -0,0 +1,20 @@
+import { HeaderValue } from 'mailparser'
+
+export type EmailContents = {
+  from: string
+  to: string
+  subject: string
+  html: string
+  text: string
+  headers: Map<string, HeaderValue>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  confirmationCode?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+}

packages/imap-mail-watcher/src/watcher.ts

@@ -0,0 +1,17 @@
+import { map, mergeMap } from 'rxjs'
+import { FetchMessageObject } from 'imapflow'
+import { emailObserver$ } from './lib/emailObserver'
+import { simpleParser } from 'mailparser'
+import { convertToMailObject, sendToEmailApi } from './lib/emailApi'
+
+void (() => {
+  emailObserver$
+    .pipe(
+      mergeMap((email: FetchMessageObject) =>
+        simpleParser(email.source.toString())
+      ),
+      map(convertToMailObject),
+      mergeMap(sendToEmailApi)
+    )
+    .subscribe()
+})()

packages/imap-mail-watcher/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./../../tsconfig.json",
+  "compileOnSave": false,
+  "include": ["./src/**/*"],
+  "compilerOptions": {
+    "outDir": "dist",
+    "typeRoots": ["./../../node_modules/pgvector/types"]
+  }
+}

packages/imap-mail-watcher/tslint.json

@@ -0,0 +1,51 @@
+{
+  "extends": "tslint:recommended",
+  "rulesDirectory": ["codelyzer"],
+  "rules": {
+    "array-type": false,
+    "arrow-parens": false,
+    "deprecation": {
+      "severity": "warn"
+    },
+    "import-blacklist": [true, "rxjs/Rx"],
+    "interface-name": false,
+    "max-classes-per-file": false,
+    "max-line-length": [true, 140],
+    "member-access": false,
+    "member-ordering": [
+      true,
+      {
+        "order": [
+          "static-field",
+          "instance-field",
+          "static-method",
+          "instance-method"
+        ]
+      }
+    ],
+    "no-consecutive-blank-lines": false,
+    "no-console": [true, "debug", "info", "time", "timeEnd", "trace"],
+    "no-empty": false,
+    "no-inferrable-types": [true, "ignore-params"],
+    "no-non-null-assertion": true,
+    "no-redundant-jsdoc": true,
+    "no-switch-case-fall-through": true,
+    "no-use-before-declare": true,
+    "no-var-requires": false,
+    "object-literal-key-quotes": [true, "as-needed"],
+    "object-literal-sort-keys": false,
+    "ordered-imports": false,
+    "quotemark": [true, "single"],
+    "trailing-comma": false,
+    "no-output-on-prefix": true,
+    "no-inputs-metadata-property": true,
+    "no-outputs-metadata-property": true,
+    "no-host-metadata-property": true,
+    "no-input-rename": true,
+    "no-output-rename": true,
+    "use-life-cycle-interface": true,
+    "use-pipe-transform-interface": true,
+    "component-class-suffix": true,
+    "directive-class-suffix": true
+  }
+}

self-hosting/GUIDE.md

@@ -4,6 +4,7 @@
 - [Nginx Reverse Proxy](#nginx-reverse-proxy)
 - [Cloudflare Tunnel](#cloudflare-tunnel)
 - [Email](#email)
+- - [IMap Watcher](#imap-watcher)
 - - [Self Hosted Mail Server](#docker-mailserver-and-mail-watcher)
 - - [Third Party Services](#third-party-services)
 - [YouTube Transcripts](#youtube-transcripts)
@@ -159,6 +160,61 @@ We will go over
 
 6. New newsletters will be automatically delivered to your Omnivore inbox.
 
+### IMap Watcher
+
+One of the easiest ways to get this functionality up and running is to use an existing email box, such as gmail. 
+
+We have included a docker file `self-hosting/docker-compose/mail/imap-parser/docker-compose-imap`. 
+
+There are a few environment variables that need to be set to make this work.
+
+| Environment Variable | Description                                                                                          | .env example                      |
+|----------------------|------------------------------------------------------------------------------------------------------|-----------------------------------|
+| WATCHER_API_KEY      | The API Key for the Watcher Service.                                                                 | api-key                           |
+| WATCHER_API_ENDPOINT | The URL of the Watcher Server.                                                                       | https://omnivore-watch.domain.tld |
+| IMAP_USER            | The IMAP User, in the gmail case this will be your email                                             | email-address@emailserver.com     |
+| IMAP_PASSWORD        | For gmail, this will be an application password. for other services this will be your email-password | password                          |
+| IMAP_HOST            | The IMAP Host, for gmail this will be imap.gmail.com                                                 | imap.host.com                     |
+| IMAP_PORT            | The IMAP Port, usually 993                                                                           | 993                               |
+| OMNIVORE_EMAIL       | The email address that Omnivore Creates                                                              | uuid@omnivore.tld                 |
+
+We will show how to set this up with a gmail account below. The steps should be similar for different email services. 
+
+#### GMail: How to
+
+##### Step 1. Create an Omnivore Email
+![Email](../docs/guides/images/create-new-email.png)
+
+This is the email address that you will use for the .env.imap file, `OMNIVORE_EMAIL`
+
+##### Step 2. Enable imap on GMail. 
+Note: For this step, I would recommend creating a separate email account rather than using your own email account. This functionality works by tracking which emails have already been opened, and automatically opens emails. 
+
+![Email Imap](../docs/guides/images/enable-imap.png)
+
+This is located in your gmail settings. 
+
+##### Step 3. Enable Application Passwords for Email. 
+For gmail, the password we need to use is an application password. In order to use these, we first have to enable multi-factor authentication for this account. 
+
+![2fa](../docs/guides/images/enable-2fa.png)
+
+Then follow the link here: https://myaccount.google.com/apppasswords to create an application password. 
+
+![app-pass](../docs/guides/images/create-app-password.png)
+
+![app-pass](../docs/guides/images/create-app-password-2.png)
+
+This will be the password for the `IMAP_PASSWORD`. 
+
+#### Step 4. Run docker compose up 
+`cd self-hosting/docker-compose/mail/imap-parser`
+`docker compose -f docker-compose-imap.yml build`
+`docker compose -f docker-compose-imap.yml up`
+
+#### Step 5. Emails are sent to Omnivore 
+![inc-gmail](../docs/guides/images/incoming-gmail.png)
+
 ### Docker-mailserver and mail-watcher
 
 One way to get this functionality back is to host your own mail server. In this example we will only be using this mail server as an incoming mailbox to receive emails. I would not recommend this method, as it's largely more effort  than it is worth.  

self-hosting/docker-compose/mail/imap-parser/.env.imap

@@ -0,0 +1,10 @@
+#MAIL
+WATCHER_API_KEY=mail-api-key
+WATCHER_API_ENDPOINT=https://omnivore-watch.omnivore.tld
+IMAP_HOST=imap.gmail.com
+IMAP_PORT=993
+IMAP_USER=email@address.com
+IMAP_PASSWORD=password
+OMNIVORE_EMAIL=uuid@omnivore.tld
+WAIT_TIME=600000
+

self-hosting/docker-compose/mail/imap-parser/docker-compose-imap.yml

@@ -0,0 +1,8 @@
+services:
+  imap-watcher:
+    build:
+      context: ../../../../
+      dockerfile: ./packages/imap-mail-watcher/Dockerfile
+    container_name: "omnivore-imap-watch"
+    env_file:
+      - .env.imap

self-hosting/docker-compose/mail/mail-server/docker-compose-mail.yml

@@ -50,7 +50,7 @@ services:
       "
   watcher:
     build:
-      context: ../../../
+      context: ../../../../
       dockerfile: ./packages/local-mail-watcher/Dockerfile-watcher
     container_name: "omnivore-mail-watch"
     volumes: