diff --git a/docs/guides/images/create-app-password-2.png b/docs/guides/images/create-app-password-2.png new file mode 100644 index 000000000..fc487e325 Binary files /dev/null and b/docs/guides/images/create-app-password-2.png differ diff --git a/docs/guides/images/create-app-password.png b/docs/guides/images/create-app-password.png new file mode 100644 index 000000000..b91e916dd Binary files /dev/null and b/docs/guides/images/create-app-password.png differ diff --git a/docs/guides/images/enable-2fa.png b/docs/guides/images/enable-2fa.png new file mode 100644 index 000000000..5ef3c7003 Binary files /dev/null and b/docs/guides/images/enable-2fa.png differ diff --git a/docs/guides/images/enable-imap.png b/docs/guides/images/enable-imap.png new file mode 100644 index 000000000..bc2b83011 Binary files /dev/null and b/docs/guides/images/enable-imap.png differ diff --git a/docs/guides/images/incoming-gmail.png b/docs/guides/images/incoming-gmail.png new file mode 100644 index 000000000..65cff4d58 Binary files /dev/null and b/docs/guides/images/incoming-gmail.png differ diff --git a/packages/imap-mail-watcher/.eslintrc b/packages/imap-mail-watcher/.eslintrc new file mode 100644 index 000000000..30bce838f --- /dev/null +++ b/packages/imap-mail-watcher/.eslintrc @@ -0,0 +1,13 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + }, + "rules": { + "@typescript-eslint/no-unsafe-argument": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/strictNullChecks": "off", + "@typescript-eslint/no-unsafe-member-access": "off", + "@typescript-eslint/no-unsafe-assignment": "off" + } +} diff --git a/packages/imap-mail-watcher/.gitignore b/packages/imap-mail-watcher/.gitignore new file mode 100644 index 000000000..b442f8ba9 --- /dev/null +++ b/packages/imap-mail-watcher/.gitignore @@ -0,0 +1,131 @@ +.idea/ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* \ No newline at end of file diff --git a/packages/imap-mail-watcher/Dockerfile b/packages/imap-mail-watcher/Dockerfile new file mode 100644 index 000000000..f21c6d95d --- /dev/null +++ b/packages/imap-mail-watcher/Dockerfile @@ -0,0 +1,40 @@ + FROM node:22.12 as builder + +WORKDIR /app + +RUN apt-get update && apt-get install -y g++ make python3 + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .prettierrc . +COPY .eslintrc . + +COPY /packages/imap-mail-watcher/src ./packages/imap-mail-watcher/src +COPY /packages/imap-mail-watcher/package.json ./packages/imap-mail-watcher/package.json +COPY /packages/imap-mail-watcher/tsconfig.json ./packages/imap-mail-watcher/tsconfig.json +COPY /packages/utils/package.json ./packages/utils/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/utils ./packages/utils + +RUN yarn workspace @omnivore/utils build +RUN yarn workspace @omnivore/imap-mail-watcher build + +FROM node:22.12 as runner + +WORKDIR /app + +ENV NODE_ENV production + +COPY --from=builder /app/packages/imap-mail-watcher/dist /app/packages/imap-mail-watcher/dist +COPY --from=builder /app/packages/imap-mail-watcher/package.json /app/packages/imap-mail-watcher/package.json +COPY --from=builder /app/packages/imap-mail-watcher/node_modules /app/packages/imap-mail-watcher/node_modules +COPY --from=builder /app/packages/utils/ /app/packages/utils/ + + +COPY --from=builder /app/node_modules /app/node_modules +COPY --from=builder /app/package.json /app/package.json + +CMD ["yarn", "workspace", "@omnivore/imap-mail-watcher", "start"] diff --git a/packages/imap-mail-watcher/package.json b/packages/imap-mail-watcher/package.json new file mode 100644 index 000000000..9ae8c0d3e --- /dev/null +++ b/packages/imap-mail-watcher/package.json @@ -0,0 +1,39 @@ +{ + "name": "@omnivore/imap-mail-watcher", + "version": "0.0.1", + "scripts": { + "build": "tsc", + "dev": "ts-node-dev --files src/index.ts", + "start": "node dist/watcher.js", + "lint": "eslint src --ext ts,js,tsx,jsx", + "lint:fix": "eslint src --fix --ext ts,js,tsx,jsx", + "test:typecheck": "tsc --noEmit" + }, + "dependencies": { + "@omnivore/utils": "1.0.0", + "axios": "^1.7.7", + "imapflow": "^1.0.181", + "mailparser": "^3.7.1" + }, + "devDependencies": { + "@types/imapflow": "^1.0.19", + "@types/axios": "^0.14.4", + "@types/express": "^5.0.0", + "rxjs": "^7.8.1", + "@types/html-to-text": "^9.0.2", + "@types/jsdom": "^21.1.3", + "@types/mailparser": "^3.4.5", + "@types/node": "^22.10.7", + "@types/pg": "^8.10.5", + "@types/pg-format": "^1.0.3", + "@types/urlsafe-base64": "^1.0.28", + "@types/uuid": "^9.0.1", + "@types/voca": "^1.4.3", + "ts-node": "^10.9.1", + "tslib": "^2.6.2", + "typescript": "^5.7.3" + }, + "volta": { + "extends": "../../package.json" + } +} diff --git a/packages/imap-mail-watcher/src/env.ts b/packages/imap-mail-watcher/src/env.ts new file mode 100755 index 000000000..997dc65d5 --- /dev/null +++ b/packages/imap-mail-watcher/src/env.ts @@ -0,0 +1,53 @@ +interface WatcherEnv { + imap: { + host: string + port: number + auth: { + user: string + password: string + } + } + omnivoreEmail: string + apiKey: string + apiEndpoint: string, + waitTime: number +} + +const envParser = + (env: { [key: string]: string | undefined }) => + (varName: string, throwOnUndefined = false): string | undefined => { + const value = env[varName] + if (typeof value === 'string' && value) { + return value + } + + if (throwOnUndefined) { + throw new Error( + `Missing ${varName} with a non-empty value in process environment` + ) + } + + return + } + +export function getEnv(): WatcherEnv { + const parse = envParser(process.env) + const imap = { + auth: { + user: parse('IMAP_USER')!, + password: parse('IMAP_PASSWORD')!, + }, + host: parse('IMAP_HOST')!, + port: Number(parse('IMAP_PORT')!), + } + + return { + apiKey: parse('WATCHER_API_KEY')!, + apiEndpoint: parse('WATCHER_API_ENDPOINT')!, + omnivoreEmail: parse('OMNIVORE_EMAIL')!, + waitTime: Number(parse('WAIT_TIME')), + imap, + } +} + +export const env = getEnv() diff --git a/packages/imap-mail-watcher/src/lib/emailApi.ts b/packages/imap-mail-watcher/src/lib/emailApi.ts new file mode 100644 index 000000000..223e46c35 --- /dev/null +++ b/packages/imap-mail-watcher/src/lib/emailApi.ts @@ -0,0 +1,26 @@ +import axios from 'axios' +import { env } from '../env' +import { ParsedMail } from 'mailparser' +import { EmailContents } from '../types/EmailContents' + +export const sendToEmailApi = (data: EmailContents) => { + console.log(`Sending mail with subject: ${data.subject} to ${data.to}`) + return axios.post(`${env.apiEndpoint}/mail`, data, { + headers: { + ['x-api-key']: env.apiKey, + 'Content-Type': 'application/json', + }, + timeout: 5000, + }) +} + +export const convertToMailObject = (it: ParsedMail): EmailContents => { + return { + from: it.from?.value[0]?.address || '', + to: env.omnivoreEmail, + subject: it.subject || '', + html: it.html || '', + text: it.text || '', + headers: it.headers, + } +} diff --git a/packages/imap-mail-watcher/src/lib/emailObserver.ts b/packages/imap-mail-watcher/src/lib/emailObserver.ts new file mode 100644 index 000000000..ecb8a055e --- /dev/null +++ b/packages/imap-mail-watcher/src/lib/emailObserver.ts @@ -0,0 +1,58 @@ +import { Observable } from 'rxjs' +import { FetchMessageObject, ImapFlow, MailboxLockObject } from 'imapflow' +import { env } from '../env' + +const client: ImapFlow = new ImapFlow({ + host: env.imap.host, + port: env.imap.port, + secure: true, + auth: { + user: env.imap.auth.user, + pass: env.imap.auth.password, + }, +}) + +export const emailObserver$ = new Observable( + (subscriber) => { + let loop = true + let lock: MailboxLockObject | null = null + + process.nextTick(async () => { + while (loop) { + if (!client.usable) { + await client.connect() + } + + if (!lock) { + lock = await client.getMailboxLock('INBOX') + } + + // Retrieve all the mails that have yet to be seen. + const messages = await client.fetchAll( + { seen: false }, + { + envelope: true, + source: true, + uid: true, + } + ) + + for (const message of messages) { + subscriber.next(message) + // Once we are done with this message, set it to seen. + await client.messageFlagsSet( + { uid: message.uid.toString(), seen: false }, + ['\\Seen'] + ) + } + + await new Promise((resolve) => setTimeout(resolve, env.waitTime)) + } + }) + + return () => { + loop = false + lock?.release() + } + } +) diff --git a/packages/imap-mail-watcher/src/types/EmailContents.ts b/packages/imap-mail-watcher/src/types/EmailContents.ts new file mode 100644 index 000000000..1ddef34f2 --- /dev/null +++ b/packages/imap-mail-watcher/src/types/EmailContents.ts @@ -0,0 +1,20 @@ +import { HeaderValue } from 'mailparser' + +export type EmailContents = { + from: string + to: string + subject: string + html: string + text: string + headers: Map + unsubMailTo?: string + unsubHttpUrl?: string + forwardedFrom?: string + replyTo?: string + confirmationCode?: string + uploadFile?: { + fileName: string + contentType: string + id: string + } +} diff --git a/packages/imap-mail-watcher/src/watcher.ts b/packages/imap-mail-watcher/src/watcher.ts new file mode 100644 index 000000000..e2728e236 --- /dev/null +++ b/packages/imap-mail-watcher/src/watcher.ts @@ -0,0 +1,17 @@ +import { map, mergeMap } from 'rxjs' +import { FetchMessageObject } from 'imapflow' +import { emailObserver$ } from './lib/emailObserver' +import { simpleParser } from 'mailparser' +import { convertToMailObject, sendToEmailApi } from './lib/emailApi' + +void (() => { + emailObserver$ + .pipe( + mergeMap((email: FetchMessageObject) => + simpleParser(email.source.toString()) + ), + map(convertToMailObject), + mergeMap(sendToEmailApi) + ) + .subscribe() +})() diff --git a/packages/imap-mail-watcher/tsconfig.json b/packages/imap-mail-watcher/tsconfig.json new file mode 100644 index 000000000..8d6ee874e --- /dev/null +++ b/packages/imap-mail-watcher/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "./../../tsconfig.json", + "compileOnSave": false, + "include": ["./src/**/*"], + "compilerOptions": { + "outDir": "dist", + "typeRoots": ["./../../node_modules/pgvector/types"] + } +} diff --git a/packages/imap-mail-watcher/tslint.json b/packages/imap-mail-watcher/tslint.json new file mode 100644 index 000000000..db7169cd9 --- /dev/null +++ b/packages/imap-mail-watcher/tslint.json @@ -0,0 +1,51 @@ +{ + "extends": "tslint:recommended", + "rulesDirectory": ["codelyzer"], + "rules": { + "array-type": false, + "arrow-parens": false, + "deprecation": { + "severity": "warn" + }, + "import-blacklist": [true, "rxjs/Rx"], + "interface-name": false, + "max-classes-per-file": false, + "max-line-length": [true, 140], + "member-access": false, + "member-ordering": [ + true, + { + "order": [ + "static-field", + "instance-field", + "static-method", + "instance-method" + ] + } + ], + "no-consecutive-blank-lines": false, + "no-console": [true, "debug", "info", "time", "timeEnd", "trace"], + "no-empty": false, + "no-inferrable-types": [true, "ignore-params"], + "no-non-null-assertion": true, + "no-redundant-jsdoc": true, + "no-switch-case-fall-through": true, + "no-use-before-declare": true, + "no-var-requires": false, + "object-literal-key-quotes": [true, "as-needed"], + "object-literal-sort-keys": false, + "ordered-imports": false, + "quotemark": [true, "single"], + "trailing-comma": false, + "no-output-on-prefix": true, + "no-inputs-metadata-property": true, + "no-outputs-metadata-property": true, + "no-host-metadata-property": true, + "no-input-rename": true, + "no-output-rename": true, + "use-life-cycle-interface": true, + "use-pipe-transform-interface": true, + "component-class-suffix": true, + "directive-class-suffix": true + } +} diff --git a/self-hosting/GUIDE.md b/self-hosting/GUIDE.md index 0c7ee9738..8cf317b8d 100644 --- a/self-hosting/GUIDE.md +++ b/self-hosting/GUIDE.md @@ -4,6 +4,7 @@ - [Nginx Reverse Proxy](#nginx-reverse-proxy) - [Cloudflare Tunnel](#cloudflare-tunnel) - [Email](#email) +- - [IMap Watcher](#imap-watcher) - - [Self Hosted Mail Server](#docker-mailserver-and-mail-watcher) - - [Third Party Services](#third-party-services) - [YouTube Transcripts](#youtube-transcripts) @@ -159,6 +160,61 @@ We will go over 6. New newsletters will be automatically delivered to your Omnivore inbox. +### IMap Watcher + +One of the easiest ways to get this functionality up and running is to use an existing email box, such as gmail. + +We have included a docker file `self-hosting/docker-compose/mail/imap-parser/docker-compose-imap`. + +There are a few environment variables that need to be set to make this work. + +| Environment Variable | Description | .env example | +|----------------------|------------------------------------------------------------------------------------------------------|-----------------------------------| +| WATCHER_API_KEY | The API Key for the Watcher Service. | api-key | +| WATCHER_API_ENDPOINT | The URL of the Watcher Server. | https://omnivore-watch.domain.tld | +| IMAP_USER | The IMAP User, in the gmail case this will be your email | email-address@emailserver.com | +| IMAP_PASSWORD | For gmail, this will be an application password. for other services this will be your email-password | password | +| IMAP_HOST | The IMAP Host, for gmail this will be imap.gmail.com | imap.host.com | +| IMAP_PORT | The IMAP Port, usually 993 | 993 | +| OMNIVORE_EMAIL | The email address that Omnivore Creates | uuid@omnivore.tld | + +We will show how to set this up with a gmail account below. The steps should be similar for different email services. + +#### GMail: How to + +##### Step 1. Create an Omnivore Email +![Email](../docs/guides/images/create-new-email.png) + +This is the email address that you will use for the .env.imap file, `OMNIVORE_EMAIL` + +##### Step 2. Enable imap on GMail. +Note: For this step, I would recommend creating a separate email account rather than using your own email account. This functionality works by tracking which emails have already been opened, and automatically opens emails. + +![Email Imap](../docs/guides/images/enable-imap.png) + +This is located in your gmail settings. + +##### Step 3. Enable Application Passwords for Email. +For gmail, the password we need to use is an application password. In order to use these, we first have to enable multi-factor authentication for this account. + +![2fa](../docs/guides/images/enable-2fa.png) + +Then follow the link here: https://myaccount.google.com/apppasswords to create an application password. + +![app-pass](../docs/guides/images/create-app-password.png) + +![app-pass](../docs/guides/images/create-app-password-2.png) + +This will be the password for the `IMAP_PASSWORD`. + +#### Step 4. Run docker compose up +`cd self-hosting/docker-compose/mail/imap-parser` +`docker compose -f docker-compose-imap.yml build` +`docker compose -f docker-compose-imap.yml up` + +#### Step 5. Emails are sent to Omnivore +![inc-gmail](../docs/guides/images/incoming-gmail.png) + ### Docker-mailserver and mail-watcher One way to get this functionality back is to host your own mail server. In this example we will only be using this mail server as an incoming mailbox to receive emails. I would not recommend this method, as it's largely more effort than it is worth. diff --git a/self-hosting/docker-compose/mail/imap-parser/.env.imap b/self-hosting/docker-compose/mail/imap-parser/.env.imap new file mode 100644 index 000000000..18b2c6518 --- /dev/null +++ b/self-hosting/docker-compose/mail/imap-parser/.env.imap @@ -0,0 +1,10 @@ +#MAIL +WATCHER_API_KEY=mail-api-key +WATCHER_API_ENDPOINT=https://omnivore-watch.omnivore.tld +IMAP_HOST=imap.gmail.com +IMAP_PORT=993 +IMAP_USER=email@address.com +IMAP_PASSWORD=password +OMNIVORE_EMAIL=uuid@omnivore.tld +WAIT_TIME=600000 + diff --git a/self-hosting/docker-compose/mail/imap-parser/docker-compose-imap.yml b/self-hosting/docker-compose/mail/imap-parser/docker-compose-imap.yml new file mode 100644 index 000000000..6fa52ccd2 --- /dev/null +++ b/self-hosting/docker-compose/mail/imap-parser/docker-compose-imap.yml @@ -0,0 +1,8 @@ +services: + imap-watcher: + build: + context: ../../../../ + dockerfile: ./packages/imap-mail-watcher/Dockerfile + container_name: "omnivore-imap-watch" + env_file: + - .env.imap diff --git a/self-hosting/docker-compose/mail/.env.mail b/self-hosting/docker-compose/mail/mail-server/.env.mail similarity index 100% rename from self-hosting/docker-compose/mail/.env.mail rename to self-hosting/docker-compose/mail/mail-server/.env.mail diff --git a/self-hosting/docker-compose/mail/docker-compose-mail.yml b/self-hosting/docker-compose/mail/mail-server/docker-compose-mail.yml similarity index 98% rename from self-hosting/docker-compose/mail/docker-compose-mail.yml rename to self-hosting/docker-compose/mail/mail-server/docker-compose-mail.yml index b4a6855dc..3b229b5a9 100644 --- a/self-hosting/docker-compose/mail/docker-compose-mail.yml +++ b/self-hosting/docker-compose/mail/mail-server/docker-compose-mail.yml @@ -50,7 +50,7 @@ services: " watcher: build: - context: ../../../ + context: ../../../../ dockerfile: ./packages/local-mail-watcher/Dockerfile-watcher container_name: "omnivore-mail-watch" volumes: diff --git a/self-hosting/docker-compose/mail/mailserver.env b/self-hosting/docker-compose/mail/mail-server/mailserver.env similarity index 100% rename from self-hosting/docker-compose/mail/mailserver.env rename to self-hosting/docker-compose/mail/mail-server/mailserver.env