[feat] Add an imap watcher for emails. (#4536)

[feat] Add an imap watcher for emails.

packages/imap-mail-watcher/.eslintrc

@@ -0,0 +1,13 @@
+{
+  "extends": "../../.eslintrc",
+  "parserOptions": {
+    "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-unsafe-argument": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/strictNullChecks": "off",
+    "@typescript-eslint/no-unsafe-member-access": "off",
+    "@typescript-eslint/no-unsafe-assignment": "off"
+  }
+}

packages/imap-mail-watcher/.gitignore

@@ -0,0 +1,131 @@
+.idea/
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*

packages/imap-mail-watcher/Dockerfile

@@ -0,0 +1,40 @@
+ FROM node:22.12 as builder
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/imap-mail-watcher/src ./packages/imap-mail-watcher/src
+COPY /packages/imap-mail-watcher/package.json ./packages/imap-mail-watcher/package.json
+COPY /packages/imap-mail-watcher/tsconfig.json ./packages/imap-mail-watcher/tsconfig.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/imap-mail-watcher build
+
+FROM node:22.12 as runner
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/imap-mail-watcher/dist /app/packages/imap-mail-watcher/dist
+COPY --from=builder /app/packages/imap-mail-watcher/package.json /app/packages/imap-mail-watcher/package.json
+COPY --from=builder /app/packages/imap-mail-watcher/node_modules /app/packages/imap-mail-watcher/node_modules
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+
+CMD ["yarn", "workspace", "@omnivore/imap-mail-watcher", "start"]

packages/imap-mail-watcher/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@omnivore/imap-mail-watcher",
+  "version": "0.0.1",
+  "scripts": {
+    "build": "tsc",
+    "dev": "ts-node-dev --files src/index.ts",
+    "start": "node dist/watcher.js",
+    "lint": "eslint src --ext ts,js,tsx,jsx",
+    "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
+    "test:typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@omnivore/utils": "1.0.0",
+    "axios": "^1.7.7",
+    "imapflow": "^1.0.181",
+    "mailparser": "^3.7.1"
+  },
+  "devDependencies": {
+    "@types/imapflow": "^1.0.19",
+    "@types/axios": "^0.14.4",
+    "@types/express": "^5.0.0",
+    "rxjs": "^7.8.1",
+    "@types/html-to-text": "^9.0.2",
+    "@types/jsdom": "^21.1.3",
+    "@types/mailparser": "^3.4.5",
+    "@types/node": "^22.10.7",
+    "@types/pg": "^8.10.5",
+    "@types/pg-format": "^1.0.3",
+    "@types/urlsafe-base64": "^1.0.28",
+    "@types/uuid": "^9.0.1",
+    "@types/voca": "^1.4.3",
+    "ts-node": "^10.9.1",
+    "tslib": "^2.6.2",
+    "typescript": "^5.7.3"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

packages/imap-mail-watcher/src/env.ts

@@ -0,0 +1,53 @@
+interface WatcherEnv {
+  imap: {
+    host: string
+    port: number
+    auth: {
+      user: string
+      password: string
+    }
+  }
+  omnivoreEmail: string
+  apiKey: string
+  apiEndpoint: string,
+  waitTime: number
+}
+
+const envParser =
+  (env: { [key: string]: string | undefined }) =>
+  (varName: string, throwOnUndefined = false): string | undefined => {
+    const value = env[varName]
+    if (typeof value === 'string' && value) {
+      return value
+    }
+
+    if (throwOnUndefined) {
+      throw new Error(
+        `Missing ${varName} with a non-empty value in process environment`
+      )
+    }
+
+    return
+  }
+
+export function getEnv(): WatcherEnv {
+  const parse = envParser(process.env)
+  const imap = {
+    auth: {
+      user: parse('IMAP_USER')!,
+      password: parse('IMAP_PASSWORD')!,
+    },
+    host: parse('IMAP_HOST')!,
+    port: Number(parse('IMAP_PORT')!),
+  }
+
+  return {
+    apiKey: parse('WATCHER_API_KEY')!,
+    apiEndpoint: parse('WATCHER_API_ENDPOINT')!,
+    omnivoreEmail: parse('OMNIVORE_EMAIL')!,
+    waitTime: Number(parse('WAIT_TIME')),
+    imap,
+  }
+}
+
+export const env = getEnv()

packages/imap-mail-watcher/src/lib/emailApi.ts

@@ -0,0 +1,26 @@
+import axios from 'axios'
+import { env } from '../env'
+import { ParsedMail } from 'mailparser'
+import { EmailContents } from '../types/EmailContents'
+
+export const sendToEmailApi = (data: EmailContents) => {
+  console.log(`Sending mail with subject: ${data.subject} to ${data.to}`)
+  return axios.post(`${env.apiEndpoint}/mail`, data, {
+    headers: {
+      ['x-api-key']: env.apiKey,
+      'Content-Type': 'application/json',
+    },
+    timeout: 5000,
+  })
+}
+
+export const convertToMailObject = (it: ParsedMail): EmailContents => {
+  return {
+    from: it.from?.value[0]?.address || '',
+    to: env.omnivoreEmail,
+    subject: it.subject || '',
+    html: it.html || '',
+    text: it.text || '',
+    headers: it.headers,
+  }
+}

packages/imap-mail-watcher/src/lib/emailObserver.ts

@@ -0,0 +1,58 @@
+import { Observable } from 'rxjs'
+import { FetchMessageObject, ImapFlow, MailboxLockObject } from 'imapflow'
+import { env } from '../env'
+
+const client: ImapFlow = new ImapFlow({
+  host: env.imap.host,
+  port: env.imap.port,
+  secure: true,
+  auth: {
+    user: env.imap.auth.user,
+    pass: env.imap.auth.password,
+  },
+})
+
+export const emailObserver$ = new Observable<FetchMessageObject>(
+  (subscriber) => {
+    let loop = true
+    let lock: MailboxLockObject | null = null
+
+    process.nextTick(async () => {
+      while (loop) {
+        if (!client.usable) {
+          await client.connect()
+        }
+
+        if (!lock) {
+          lock = await client.getMailboxLock('INBOX')
+        }
+
+        // Retrieve all the mails that have yet to be seen.
+        const messages = await client.fetchAll(
+          { seen: false },
+          {
+            envelope: true,
+            source: true,
+            uid: true,
+          }
+        )
+
+        for (const message of messages) {
+          subscriber.next(message)
+          // Once we are done with this message, set it to seen.
+          await client.messageFlagsSet(
+            { uid: message.uid.toString(), seen: false },
+            ['\\Seen']
+          )
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, env.waitTime))
+      }
+    })
+
+    return () => {
+      loop = false
+      lock?.release()
+    }
+  }
+)

packages/imap-mail-watcher/src/types/EmailContents.ts

@@ -0,0 +1,20 @@
+import { HeaderValue } from 'mailparser'
+
+export type EmailContents = {
+  from: string
+  to: string
+  subject: string
+  html: string
+  text: string
+  headers: Map<string, HeaderValue>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  confirmationCode?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+}

packages/imap-mail-watcher/src/watcher.ts

@@ -0,0 +1,17 @@
+import { map, mergeMap } from 'rxjs'
+import { FetchMessageObject } from 'imapflow'
+import { emailObserver$ } from './lib/emailObserver'
+import { simpleParser } from 'mailparser'
+import { convertToMailObject, sendToEmailApi } from './lib/emailApi'
+
+void (() => {
+  emailObserver$
+    .pipe(
+      mergeMap((email: FetchMessageObject) =>
+        simpleParser(email.source.toString())
+      ),
+      map(convertToMailObject),
+      mergeMap(sendToEmailApi)
+    )
+    .subscribe()
+})()

packages/imap-mail-watcher/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./../../tsconfig.json",
+  "compileOnSave": false,
+  "include": ["./src/**/*"],
+  "compilerOptions": {
+    "outDir": "dist",
+    "typeRoots": ["./../../node_modules/pgvector/types"]
+  }
+}

packages/imap-mail-watcher/tslint.json

@@ -0,0 +1,51 @@
+{
+  "extends": "tslint:recommended",
+  "rulesDirectory": ["codelyzer"],
+  "rules": {
+    "array-type": false,
+    "arrow-parens": false,
+    "deprecation": {
+      "severity": "warn"
+    },
+    "import-blacklist": [true, "rxjs/Rx"],
+    "interface-name": false,
+    "max-classes-per-file": false,
+    "max-line-length": [true, 140],
+    "member-access": false,
+    "member-ordering": [
+      true,
+      {
+        "order": [
+          "static-field",
+          "instance-field",
+          "static-method",
+          "instance-method"
+        ]
+      }
+    ],
+    "no-consecutive-blank-lines": false,
+    "no-console": [true, "debug", "info", "time", "timeEnd", "trace"],
+    "no-empty": false,
+    "no-inferrable-types": [true, "ignore-params"],
+    "no-non-null-assertion": true,
+    "no-redundant-jsdoc": true,
+    "no-switch-case-fall-through": true,
+    "no-use-before-declare": true,
+    "no-var-requires": false,
+    "object-literal-key-quotes": [true, "as-needed"],
+    "object-literal-sort-keys": false,
+    "ordered-imports": false,
+    "quotemark": [true, "single"],
+    "trailing-comma": false,
+    "no-output-on-prefix": true,
+    "no-inputs-metadata-property": true,
+    "no-outputs-metadata-property": true,
+    "no-host-metadata-property": true,
+    "no-input-rename": true,
+    "no-output-rename": true,
+    "use-life-cycle-interface": true,
+    "use-pipe-transform-interface": true,
+    "component-class-suffix": true,
+    "directive-class-suffix": true
+  }
+}