diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 0086a086e..e45c0f6c3 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -42,6 +42,15 @@ jobs: --health-retries 10 ports: - 9200 + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 6379 steps: - uses: actions/checkout@v2 with: diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 28f6dd958..b90537a38 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -40,7 +40,6 @@ const CONTENT_TYPES = ['text/csv', 'application/zip'] export type UrlHandler = ( ctx: ImportContext, url: URL, - taskId: string, state?: ArticleSavingRequestStatus, labels?: string[] ) => Promise @@ -175,7 +174,6 @@ const handlerForFile = (name: string): importHandlerFunc | undefined => { const urlHandler = async ( ctx: ImportContext, url: URL, - taskId: string, state?: ArticleSavingRequestStatus, labels?: string[] ): Promise => { @@ -185,7 +183,7 @@ const urlHandler = async ( ctx.userId, url, 'csv-importer', - taskId, + ctx.taskId, state, labels && labels.length > 0 ? labels : undefined ) diff --git a/packages/import-handler/src/redis.ts b/packages/import-handler/src/redis.ts index 3804c36ed..481f136b9 100644 --- a/packages/import-handler/src/redis.ts +++ b/packages/import-handler/src/redis.ts @@ -1,9 +1,13 @@ -import fs from 'fs' +import { readFileSync } from 'fs' +import path from 'path' import { createClient } from 'redis' // load lua script export const lua = { - script: fs.readFileSync('./luaScripts/updateMetrics.lua', 'utf8'), + script: readFileSync( + path.resolve(__dirname, 'luaScripts/updateMetrics.lua'), + 'utf8' + ), sha: '', } diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index f045a0c96..21d4e175a 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -13,7 +13,7 @@ describe('Load a simple CSV file', () => { it('should call the handler for each URL', async () => { const urls: URL[] = [] const stream = fs.createReadStream('./test/csv/data/simple.csv') - const stub = stubImportCtx() + const stub = await stubImportCtx() stub.urlHandler = (ctx: ImportContext, url): Promise => { urls.push(url) return Promise.resolve() @@ -26,12 +26,14 @@ describe('Load a simple CSV file', () => { new URL('https://omnivore.app'), new URL('https://google.com'), ]) + + await stub.redisClient.quit() }) it('increments the failed count when the URL is invalid', async () => { const urls: URL[] = [] const stream = fs.createReadStream('./test/csv/data/simple.csv') - const stub = stubImportCtx() + const stub = await stubImportCtx() stub.urlHandler = (ctx: ImportContext, url): Promise => { urls.push(url) return Promise.reject('Failed to import url') @@ -40,6 +42,8 @@ describe('Load a simple CSV file', () => { await importCsv(stub, stream) expect(stub.countFailed).to.equal(2) expect(stub.countImported).to.equal(0) + + await stub.redisClient.quit() }) }) @@ -51,7 +55,7 @@ describe('Load a complex CSV file', () => { labels?: string[] }[] = [] const stream = fs.createReadStream('./test/csv/data/complex.csv') - const stub = stubImportCtx() + const stub = await stubImportCtx() stub.urlHandler = ( ctx: ImportContext, url, @@ -86,6 +90,8 @@ describe('Load a complex CSV file', () => { labels: ['test', 'development'], }, ]) + + await stub.redisClient.quit() }) }) diff --git a/packages/import-handler/test/matter/matter_importer.test.ts b/packages/import-handler/test/matter/matter_importer.test.ts index 90600b16a..94017e634 100644 --- a/packages/import-handler/test/matter/matter_importer.test.ts +++ b/packages/import-handler/test/matter/matter_importer.test.ts @@ -1,15 +1,15 @@ -import 'mocha' +import { Readability } from '@omnivore/readability' import * as chai from 'chai' import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' +import 'mocha' +import { ImportContext } from '../../src' import { importMatterArchive, importMatterHistoryCsv, } from '../../src/matterHistory' import { stubImportCtx } from '../util' -import { ImportContext } from '../../src' -import { Readability } from '@omnivore/readability' chai.use(chaiString) @@ -17,7 +17,7 @@ describe('Load a simple _matter_history file', () => { it('should find the URL of each row', async () => { const urls: URL[] = [] const stream = fs.createReadStream('./test/matter/data/_matter_history.csv') - const stub = stubImportCtx() + const stub = await stubImportCtx() stub.urlHandler = (ctx: ImportContext, url): Promise => { urls.push(url) return Promise.resolve() @@ -29,6 +29,8 @@ describe('Load a simple _matter_history file', () => { expect(urls).to.eql([ new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'), ]) + + await stub.redisClient.quit() }) }) @@ -36,7 +38,7 @@ describe('Load archive file', () => { it('should find the URL of each row', async () => { const urls: URL[] = [] const stream = fs.createReadStream('./test/matter/data/Archive.zip') - const stub = stubImportCtx() + const stub = await stubImportCtx() stub.contentHandler = ( ctx: ImportContext, url: URL, @@ -54,5 +56,7 @@ describe('Load archive file', () => { expect(urls).to.eql([ new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'), ]) + + await stub.redisClient.quit() }) }) diff --git a/packages/import-handler/test/util.ts b/packages/import-handler/test/util.ts index a60564ae2..5fa9ba785 100644 --- a/packages/import-handler/test/util.ts +++ b/packages/import-handler/test/util.ts @@ -1,7 +1,9 @@ import { Readability } from '@omnivore/readability' import { ArticleSavingRequestStatus, ImportContext } from '../src' +import { createRedisClient } from '../src/redis' -export const stubImportCtx = () => { +export const stubImportCtx = async () => { + const redisClient = await createRedisClient() return { userId: '', countImported: 0, @@ -23,5 +25,7 @@ export const stubImportCtx = () => { ): Promise => { return Promise.resolve() }, + redisClient, + taskId: '', } }