diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index 0c3118544..869de8aa2 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -100,7 +100,7 @@ const getBrowserPromise = (async () => { '--window-size=1920,1080', ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH , + executablePath: process.env.CHROMIUM_PATH, headless: !!process.env.LAUNCH_HEADLESS, timeout: 120000, // 2 minutes }); diff --git a/packages/inbound-email-handler/Dockerfile b/packages/inbound-email-handler/Dockerfile index e6cf56c11..364c786bc 100644 --- a/packages/inbound-email-handler/Dockerfile +++ b/packages/inbound-email-handler/Dockerfile @@ -9,6 +9,7 @@ COPY tsconfig.json . COPY .prettierrc . COPY .eslintrc . +COPY /packages/inbound-email-handler/package.json ./packages/inbound-email-handler/package.json COPY /packages/content-handler/package.json ./packages/content-handler/package.json RUN yarn install --pure-lockfile @@ -16,6 +17,7 @@ RUN yarn install --pure-lockfile ADD /packages/inbound-email-handler ./packages/inbound-email-handler ADD /packages/content-handler ./packages/content-handler RUN yarn workspace @omnivore/content-handler build +RUN yarn workspace @omnivore/inbound-email-handler build # After building, fetch the production dependencies RUN rm -rf /app/packages/inbound-email-handler/node_modules diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json index b11c89e31..0fef8043a 100644 --- a/packages/inbound-email-handler/package.json +++ b/packages/inbound-email-handler/package.json @@ -24,6 +24,7 @@ "@types/json-bigint": "^1.0.1", "@types/node": "^14.11.2", "@types/rfc2047": "^2.0.1", + "chai": "^4.3.6", "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { diff --git a/packages/inbound-email-handler/tsconfig.json b/packages/inbound-email-handler/tsconfig.json index f450acf38..5220d6b3f 100644 --- a/packages/inbound-email-handler/tsconfig.json +++ b/packages/inbound-email-handler/tsconfig.json @@ -5,5 +5,5 @@ "rootDir": ".", "lib": ["dom"] }, - "include": ["src", "test"] + "include": ["src"] } diff --git a/packages/puppeteer-parse/Dockerfile b/packages/puppeteer-parse/Dockerfile index 771aa0f5b..7faae17bf 100644 --- a/packages/puppeteer-parse/Dockerfile +++ b/packages/puppeteer-parse/Dockerfile @@ -92,6 +92,7 @@ COPY tsconfig.json . COPY .prettierrc . COPY .eslintrc . +COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json COPY /packages/content-handler/package.json ./packages/content-handler/package.json RUN yarn install --pure-lockfile diff --git a/packages/puppeteer-parse/Dockerfile-preview b/packages/puppeteer-parse/Dockerfile-preview deleted file mode 100644 index 7379c0c31..000000000 --- a/packages/puppeteer-parse/Dockerfile-preview +++ /dev/null @@ -1,111 +0,0 @@ -# FROM node:14-slim - -# # Taken from pu - -# # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -# # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer -# # installs, work. -# RUN apt-get update \ -# && apt-get install -y wget gnupg \ -# && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ -# && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ -# && apt-get update \ -# && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ -# --no-install-recommends \ -# && rm -rf /var/lib/apt/lists/* - -# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -# ENV CHROMIUM_PATH "/usr/bin/google-chrome-stable" - -# ------------------------ - -# FROM --platform=linux/arm64 node:14.18 - -# RUN apt-get update \ -# && apt-get install -y chromium \ -# && apt-get install -y ca-certificates \ -# fonts-liberation \ -# libappindicator3-1 \ -# libasound2 \ -# libatk-bridge2.0-0 \ -# libatk1.0-0 \ -# libc6 \ -# libcairo2 \ -# libcups2 \ -# libdbus-1-3 \ -# libexpat1 \ -# libfontconfig1 \ -# libgbm1 \ -# libgcc1 \ -# libglib2.0-0 \ -# libgtk-3-0 \ -# libnspr4 \ -# libnss3 \ -# libpango-1.0-0 \ -# libpangocairo-1.0-0 \ -# libstdc++6 \ -# libx11-6 \ -# libx11-xcb1 \ -# libxcb1 \ -# libxcomposite1 \ -# libxcursor1 \ -# libxdamage1 \ -# libxext6 \ -# libxfixes3 \ -# libxi6 \ -# libxrandr2 \ -# libxrender1 \ -# libxss1 \ -# libxtst6 \ -# lsb-release \ -# wget \ -# xdg-utils - -FROM node:14.18-alpine - -# Installs latest Chromium (92) package. -RUN apk add --no-cache \ - chromium \ - nss \ - freetype \ - harfbuzz \ - ca-certificates \ - ttf-freefont \ - nodejs \ - yarn - -# Add user so we don't need --no-sandbox. -RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ - && mkdir -p /home/pptruser/Downloads /app \ - && chown -R pptruser:pptruser /home/pptruser \ - && chown -R pptruser:pptruser /app - -# Run everything after as non-privileged user. -WORKDIR /app - -ENV CHROMIUM_PATH /usr/bin/chromium-browser -ENV LAUNCH_HEADLESS=true - -COPY package.json . -COPY yarn.lock . -COPY tsconfig.json . -COPY .prettierrc . -COPY .eslintrc . - -COPY /packages/content-handler/package.json ./packages/content-handler/package.json - -RUN yarn install --pure-lockfile - -ADD /packages/puppeteer-parse ./packages/puppeteer-parse -ADD /packages/content-handler ./packages/content-handler -RUN yarn workspace @omnivore/content-handler build - -# After building, fetch the production dependencies -RUN rm -rf /app/packages/puppeteer-parse/node_modules -RUN rm -rf /app/node_modules -RUN yarn install --pure-lockfile --production - -EXPOSE 8080 - -# USER pptruser -ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start_preview"] diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index e158f06d7..d3fadf998 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -118,43 +118,43 @@ const userAgentForUrl = (url) => { // launch Puppeteer const getBrowserPromise = (async () => { - return puppeteer.launch({ - args: chromium.args, - defaultViewport: chromium.defaultViewport, - executablePath: await chromium.executablePath, - headless: chromium.headless, - ignoreHTTPSErrors: true, - }); // return puppeteer.launch({ - // args: [ - // '--allow-running-insecure-content', - // '--autoplay-policy=user-gesture-required', - // '--disable-component-update', - // '--disable-domain-reliability', - // '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', - // '--disable-print-preview', - // '--disable-setuid-sandbox', - // '--disable-site-isolation-trials', - // '--disable-speech-api', - // '--disable-web-security', - // '--disk-cache-size=33554432', - // '--enable-features=SharedArrayBuffer', - // '--hide-scrollbars', - // '--ignore-gpu-blocklist', - // '--in-process-gpu', - // '--mute-audio', - // '--no-default-browser-check', - // '--no-pings', - // '--no-sandbox', - // '--no-zygote', - // '--use-gl=swiftshader', - // '--window-size=1920,1080', - // ].filter((item) => !!item), - // defaultViewport: { height: 1080, width: 1920 }, + // args: chromium.args, + // defaultViewport: chromium.defaultViewport, // executablePath: process.env.CHROMIUM_PATH, - // headless: !!process.env.LAUNCH_HEADLESS, - // timeout: 0, + // headless: chromium.headless, + // ignoreHTTPSErrors: true, // }); + return puppeteer.launch({ + args: [ + '--allow-running-insecure-content', + '--autoplay-policy=user-gesture-required', + '--disable-component-update', + '--disable-domain-reliability', + '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', + '--disable-print-preview', + '--disable-setuid-sandbox', + '--disable-site-isolation-trials', + '--disable-speech-api', + '--disable-web-security', + '--disk-cache-size=33554432', + '--enable-features=SharedArrayBuffer', + '--hide-scrollbars', + '--ignore-gpu-blocklist', + '--in-process-gpu', + '--mute-audio', + '--no-default-browser-check', + '--no-pings', + '--no-sandbox', + '--no-zygote', + '--use-gl=swiftshader', + '--window-size=1920,1080', + ].filter((item) => !!item), + defaultViewport: { height: 1080, width: 1920 }, + executablePath: process.env.CHROMIUM_PATH, + headless: !!process.env.LAUNCH_HEADLESS, + timeout: 120000, // 2 minutes + }); })(); let logRecord, functionStartTime;