diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index e45c0f6c3..08bc418cb 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -98,6 +98,7 @@ jobs: PG_DB: omnivore_test PG_POOL_MAX: 10 ELASTIC_URL: http://localhost:${{ job.services.elastic.ports[9200] }}/ + REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }} build-docker-images: name: Build docker images runs-on: ubuntu-latest diff --git a/packages/import-handler/Dockerfile-collector b/packages/import-handler/Dockerfile-collector new file mode 100644 index 000000000..fba468482 --- /dev/null +++ b/packages/import-handler/Dockerfile-collector @@ -0,0 +1,31 @@ +FROM node:14.18-alpine + +WORKDIR /app + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN apk add g++ make python3 + +ENV PORT 8080 + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json +COPY /packages/import-handler/package.json ./packages/import-handler/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/import-handler ./packages/import-handler +ADD /packages/readabilityjs ./packages/readabilityjs +RUN yarn workspace @omnivore/import-handler build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/import-handler/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +ENTRYPOINT ["yarn", "workspace", "@omnivore/import-handler", "start:collector"] diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index d1e6f2b34..ec14ef577 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -15,7 +15,7 @@ "build": "tsc && yarn copy-files", "start": "functions-framework --target=importHandler", "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"", - "copy-files": "copyfiles -u 1 src/**/*.lua build/src", + "copy-files": "copyfiles src/luaScripts/*.lua build/", "start:collector": "functions-framework --target=importMetricsCollector" }, "devDependencies": { @@ -49,7 +49,7 @@ "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.21", "nodemon": "^2.0.15", - "redis": "^4.6.6", + "redis": "^4.3.1", "unzip-stream": "^0.3.1", "urlsafe-base64": "^1.0.0", "uuid": "^9.0.0" diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index b90537a38..269ad463c 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -364,15 +364,15 @@ export const importMetricsCollector = Sentry.GCPFunction.wrapHttpFunction( return res.status(401).send({ errorCode: 'UNAUTHENTICATED' }) } - const redisClient = await createRedisClient( - process.env.REDIS_URL, - process.env.REDIS_CERT - ) if (!isUpdateMetricsRequest(req.body)) { console.log('Invalid request body') return res.status(400).send('Bad Request') } + const redisClient = await createRedisClient( + process.env.REDIS_URL, + process.env.REDIS_CERT + ) // update metrics await updateMetrics(redisClient, userId, req.body.taskId, req.body.status) diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 21d4e175a..82798fa89 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -9,135 +9,134 @@ import { stubImportCtx } from '../util' chai.use(chaiString) -describe('Load a simple CSV file', () => { - it('should call the handler for each URL', async () => { - const urls: URL[] = [] - const stream = fs.createReadStream('./test/csv/data/simple.csv') - const stub = await stubImportCtx() - stub.urlHandler = (ctx: ImportContext, url): Promise => { - urls.push(url) - return Promise.resolve() - } +describe('Test csv importer', () => { + let stub: ImportContext - await importCsv(stub, stream) - expect(stub.countFailed).to.equal(0) - expect(stub.countImported).to.equal(2) - expect(urls).to.eql([ - new URL('https://omnivore.app'), - new URL('https://google.com'), - ]) + beforeEach(async () => { + stub = await stubImportCtx() + }) + afterEach(async () => { await stub.redisClient.quit() }) - it('increments the failed count when the URL is invalid', async () => { - const urls: URL[] = [] - const stream = fs.createReadStream('./test/csv/data/simple.csv') - const stub = await stubImportCtx() - stub.urlHandler = (ctx: ImportContext, url): Promise => { - urls.push(url) - return Promise.reject('Failed to import url') - } + describe('Load a simple CSV file', () => { + it('should call the handler for each URL', async () => { + const urls: URL[] = [] + const stream = fs.createReadStream('./test/csv/data/simple.csv') + stub.urlHandler = (ctx: ImportContext, url): Promise => { + urls.push(url) + return Promise.resolve() + } - await importCsv(stub, stream) - expect(stub.countFailed).to.equal(2) - expect(stub.countImported).to.equal(0) + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(2) + expect(urls).to.eql([ + new URL('https://omnivore.app'), + new URL('https://google.com'), + ]) + }) - await stub.redisClient.quit() + it('increments the failed count when the URL is invalid', async () => { + const stream = fs.createReadStream('./test/csv/data/simple.csv') + stub.urlHandler = (ctx: ImportContext, url): Promise => { + return Promise.reject('Failed to import url') + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(2) + expect(stub.countImported).to.equal(0) + }) }) -}) -describe('Load a complex CSV file', () => { - it('should call the handler for each URL, state and labels', async () => { - const results: { - url: URL - state?: ArticleSavingRequestStatus - labels?: string[] - }[] = [] - const stream = fs.createReadStream('./test/csv/data/complex.csv') - const stub = await stubImportCtx() - stub.urlHandler = ( - ctx: ImportContext, - url, - state, - labels - ): Promise => { - results.push({ + describe('Load a complex CSV file', () => { + it('should call the handler for each URL, state and labels', async () => { + const results: { + url: URL + state?: ArticleSavingRequestStatus + labels?: string[] + }[] = [] + const stream = fs.createReadStream('./test/csv/data/complex.csv') + stub.urlHandler = ( + ctx: ImportContext, url, state, - labels, - }) - return Promise.resolve() - } + labels + ): Promise => { + results.push({ + url, + state, + labels, + }) + return Promise.resolve() + } - await importCsv(stub, stream) - expect(stub.countFailed).to.equal(0) - expect(stub.countImported).to.equal(3) - expect(results).to.eql([ - { - url: new URL('https://omnivore.app'), - state: 'ARCHIVED', - labels: ['test'], - }, - { - url: new URL('https://google.com'), - state: 'SUCCEEDED', - labels: ['test', 'development'], - }, - { - url: new URL('https://test.com'), - state: 'SUCCEEDED', - labels: ['test', 'development'], - }, - ]) + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(3) + expect(results).to.eql([ + { + url: new URL('https://omnivore.app'), + state: 'ARCHIVED', + labels: ['test'], + }, + { + url: new URL('https://google.com'), + state: 'SUCCEEDED', + labels: ['test', 'development'], + }, + { + url: new URL('https://test.com'), + state: 'SUCCEEDED', + labels: ['test', 'development'], + }, + ]) + }) + }) - await stub.redisClient.quit() - }) -}) - -describe('A file with no status set', () => { - it('should not try to set status', async () => { - const states: (ArticleSavingRequestStatus | undefined)[] = [] - const stream = fs.createReadStream('./test/csv/data/unset-status.csv') - const stub = stubImportCtx() - stub.urlHandler = ( - ctx: ImportContext, - url, - state?: ArticleSavingRequestStatus - ): Promise => { - states.push(state) - return Promise.resolve() - } - - await importCsv(stub, stream) - expect(stub.countFailed).to.equal(0) - expect(stub.countImported).to.equal(2) - expect(states).to.eql([undefined, ArticleSavingRequestStatus.Archived]) - }) -}) - -describe('A file with some labels', () => { - it('gets the labels, handles empty, and trims extra whitespace', async () => { - const importedLabels: (string[] | undefined)[] = [] - const stream = fs.createReadStream('./test/csv/data/labels.csv') - const stub = stubImportCtx() - stub.urlHandler = ( - ctx: ImportContext, - url, - state?: ArticleSavingRequestStatus, - labels?: string[] - ): Promise => { - importedLabels.push(labels) - return Promise.resolve() - } - - await importCsv(stub, stream) - expect(stub.countFailed).to.equal(0) - expect(stub.countImported).to.equal(3) - expect(importedLabels).to.eql([ - ['Label1', 'Label2', 'Label 3', 'Label 4'], - [], - [], - ]) + describe('A file with no status set', () => { + it('should not try to set status', async () => { + const states: (ArticleSavingRequestStatus | undefined)[] = [] + const stream = fs.createReadStream('./test/csv/data/unset-status.csv') + stub.urlHandler = ( + ctx: ImportContext, + url, + state?: ArticleSavingRequestStatus + ): Promise => { + states.push(state) + return Promise.resolve() + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(2) + expect(states).to.eql([undefined, ArticleSavingRequestStatus.Archived]) + }) + }) + + describe('A file with some labels', () => { + it('gets the labels, handles empty, and trims extra whitespace', async () => { + const importedLabels: (string[] | undefined)[] = [] + const stream = fs.createReadStream('./test/csv/data/labels.csv') + stub.urlHandler = ( + ctx: ImportContext, + url, + state?: ArticleSavingRequestStatus, + labels?: string[] + ): Promise => { + importedLabels.push(labels) + return Promise.resolve() + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(3) + expect(importedLabels).to.eql([ + ['Label1', 'Label2', 'Label 3', 'Label 4'], + [], + [], + ]) + }) }) }) diff --git a/packages/import-handler/test/util.ts b/packages/import-handler/test/util.ts index 5fa9ba785..54a8b5082 100644 --- a/packages/import-handler/test/util.ts +++ b/packages/import-handler/test/util.ts @@ -2,8 +2,9 @@ import { Readability } from '@omnivore/readability' import { ArticleSavingRequestStatus, ImportContext } from '../src' import { createRedisClient } from '../src/redis' -export const stubImportCtx = async () => { - const redisClient = await createRedisClient() +export const stubImportCtx = async (): Promise => { + const redisClient = await createRedisClient(process.env.REDIS_URL) + return { userId: '', countImported: 0, diff --git a/packages/import-handler/tsconfig.json b/packages/import-handler/tsconfig.json index ea8c4d3ef..912ebd323 100644 --- a/packages/import-handler/tsconfig.json +++ b/packages/import-handler/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "./../../tsconfig.json", - "ts-node": { "files": true }, + "ts-node": { "files": true }, "compilerOptions": { "outDir": "build", "rootDir": ".", @@ -8,5 +8,5 @@ // Generate d.ts files "declaration": true }, - "include": ["src", "test"] + "include": ["src/**/*", "test/**/*"] } diff --git a/yarn.lock b/yarn.lock index 5ef541069..339090cfe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5431,11 +5431,6 @@ resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf" integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw== -"@redis/bloom@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" - integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== - "@redis/client@1.3.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77" @@ -5445,25 +5440,11 @@ generic-pool "3.8.2" yallist "4.0.0" -"@redis/client@1.5.7": - version "1.5.7" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.7.tgz#92cc5c98c76f189e37d24f0e1e17e104c6af17d4" - integrity sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw== - dependencies: - cluster-key-slot "1.1.2" - generic-pool "3.9.0" - yallist "4.0.0" - "@redis/graph@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99" integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ== -"@redis/graph@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" - integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== - "@redis/json@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" @@ -5474,21 +5455,11 @@ resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df" integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ== -"@redis/search@1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.2.tgz#6a8f66ba90812d39c2457420f859ce8fbd8f3838" - integrity sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA== - "@redis/time-series@1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4" integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA== -"@redis/time-series@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" - integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== - "@remusao/guess-url-type@^1.1.2": version "1.2.1" resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57" @@ -11841,11 +11812,6 @@ cluster-key-slot@1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== -cluster-key-slot@1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" - integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== - cmd-shim@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -15354,11 +15320,6 @@ generic-pool@3.8.2: resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== -generic-pool@3.9.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" - integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== - gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -24140,18 +24101,6 @@ redis@^4.3.1: "@redis/search" "1.1.0" "@redis/time-series" "1.0.3" -redis@^4.6.6: - version "4.6.6" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.6.tgz#46d4f2d149d1634d6ef53db5747412a0ef7974ec" - integrity sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA== - dependencies: - "@redis/bloom" "1.2.0" - "@redis/client" "1.5.7" - "@redis/graph" "1.1.0" - "@redis/json" "1.0.4" - "@redis/search" "1.1.2" - "@redis/time-series" "1.0.4" - reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"