From dea4ffae3386e5e373b1a4a5cf160b4aed84d19c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 11 Jan 2023 18:10:22 +0800 Subject: [PATCH] Fixes for dockerizing the import-handler service --- packages/import-handler/Dockerfile | 5 ++--- packages/import-handler/package.json | 9 +++++---- packages/import-handler/src/index.ts | 10 +++++----- packages/import-handler/src/task.ts | 11 ++++++----- packages/import-handler/tsconfig.json | 9 +++++---- yarn.lock | 20 ++++++++++++++++++++ 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/packages/import-handler/Dockerfile b/packages/import-handler/Dockerfile index f4a167620..83a66b834 100644 --- a/packages/import-handler/Dockerfile +++ b/packages/import-handler/Dockerfile @@ -5,12 +5,11 @@ WORKDIR /app ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true RUN apk add g++ make python3 -ENV PORT 9090 +ENV PORT 8080 COPY package.json . COPY yarn.lock . COPY tsconfig.json . -COPY .prettierrc . COPY .eslintrc . COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json @@ -27,6 +26,6 @@ RUN rm -rf /app/packages/import-handler/node_modules RUN rm -rf /app/node_modules RUN yarn install --pure-lockfile --production -EXPOSE 9090 +EXPOSE 8080 ENTRYPOINT ["yarn", "workspace", "@omnivore/import-handler", "start_gcf"] diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index 2413bf556..2ca868a17 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -3,18 +3,17 @@ "version": "1.0.0", "description": "", "main": "build/src/index.js", - "types": "build/src/index.d.ts", "files": [ "build/src" ], - "license": "Apache-2.0", "keywords": [], + "license": "Apache-2.0", "scripts": { "test": "yarn mocha -r ts-node/register --config mocha-config.json", "lint": "eslint src --ext ts,js,tsx,jsx", "compile": "tsc", "build": "tsc", - "start_gcf": "npx functions-framework --port=9090 --target=httpServer", + "start": "functions-framework --target=importHandler", "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" }, "devDependencies": { @@ -38,13 +37,15 @@ "@omnivore/readability": "1.0.0", "@sentry/serverless": "^7.30.0", "@types/express": "^4.17.13", + "axios": "^1.2.2", "csv-parser": "^3.0.0", "dompurify": "^2.4.3", "fs-extra": "^11.1.0", "jsonwebtoken": "^8.5.1", + "linkedom": "^0.14.21", "nodemon": "^2.0.15", "unzip-stream": "^0.3.1", "urlsafe-base64": "^1.0.0", "uuid": "^9.0.0" } -} +} \ No newline at end of file diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index c68e0b395..778c0f705 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -8,13 +8,13 @@ import * as path from 'path' import { importMatterArchive, importMatterHistoryCsv } from './matterHistory' import { Stream } from 'node:stream' import { v4 as uuid } from 'uuid' -import { CONTENT_FETCH_URL, createCloudTask, EMAIL_USER_URL } from './task' +import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' import { promisify } from 'util' import * as jwt from 'jsonwebtoken' import { Readability } from '@omnivore/readability' -import Sentry from '@sentry/serverless' +import * as Sentry from '@sentry/serverless' Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -96,7 +96,7 @@ const createEmailCloudTask = async (userId: string, payload: unknown) => { Cookie: `auth=${authToken}`, } - return createCloudTask(EMAIL_USER_URL, payload, headers) + return createCloudTask(emailUserUrl(), payload, headers) } const sendImportFailedEmail = async (userId: string) => { @@ -157,7 +157,7 @@ const contentHandler = async ( return Promise.resolve() } -export const importHandler: EventFunction = async (event, context) => { +export const gcsEventHandler: EventFunction = async (event, context) => { const data = event as StorageEventData const ctx = context as CloudFunctionsContext @@ -205,7 +205,7 @@ export const importHandler: EventFunction = async (event, context) => { } } -export const httpServer = Sentry.GCPFunction.wrapHttpFunction( +export const importHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { res.send('ok') } diff --git a/packages/import-handler/src/task.ts b/packages/import-handler/src/task.ts index f35c31a2d..16f66b08d 100644 --- a/packages/import-handler/src/task.ts +++ b/packages/import-handler/src/task.ts @@ -3,12 +3,13 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks' const cloudTask = new CloudTasksClient() -export const EMAIL_USER_URL = (() => { - if (!process.env.INTERNAL_SVC_ENDPOINT) { - throw `Environment not configured correctly, no SVC endpoint` +export const emailUserUrl = () => { + const envar = process.env.INTERNAL_SVC_ENDPOINT + if (envar) { + return envar + 'api/user/email' } - return (process.env.INTERNAL_SVC_ENDPOINT ?? '') + 'api/user/email' -})() + throw 'INTERNAL_SVC_ENDPOINT not set' +} export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL diff --git a/packages/import-handler/tsconfig.json b/packages/import-handler/tsconfig.json index 5c4d5c778..8c12cd3a8 100644 --- a/packages/import-handler/tsconfig.json +++ b/packages/import-handler/tsconfig.json @@ -1,10 +1,11 @@ { "extends": "./../../tsconfig.json", - "ts-node": { - "files": true - }, "compilerOptions": { - "outDir": "dist" + "outDir": "build", + "rootDir": ".", + "lib": ["dom"], + // Generate d.ts files + "declaration": true }, "include": ["src", "test"] } diff --git a/yarn.lock b/yarn.lock index 076209800..8a8741388 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10266,6 +10266,15 @@ axios@^1.2.0: form-data "^4.0.0" proxy-from-env "^1.1.0" +axios@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.2.2.tgz#72681724c6e6a43a9fea860fc558127dbe32f9f1" + integrity sha512-bz/J4gS2S3I7mpN/YZfGFTqhXTYzRho8Ay38w2otuuDR322KzFIWm/4W2K6gIwvWaws5n+mnb7D1lN9uD+QH6Q== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + axobject-query@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be" @@ -18550,6 +18559,17 @@ linkedom@^0.14.16: htmlparser2 "^8.0.1" uhyphen "^0.1.0" +linkedom@^0.14.21: + version "0.14.21" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.21.tgz#878e1e5e88028cb1d57bc6262f84484a41a37497" + integrity sha512-V+c0AAFMTVJA2iAhrdd+u44lL0TjL6hBenVB061VQ6BHqTAHtXw1v5F1/CHGKtwg0OHm+hrGbepb9ZSFJ7lJkg== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^8.0.1" + uhyphen "^0.1.0" + linkedom@^0.14.9: version "0.14.9" resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0"