Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add dockerfile

Browse Source
This commit is contained in:
Hongbo Wu
2023-05-25 16:14:40 +08:00
parent 8a89a47c16
commit 992a78e173
8 changed files with 159 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/run-tests.yaml vendored
View File

@ -98,6 +98,7 @@ jobs:
PG_DB: omnivore_test
PG_POOL_MAX: 10
ELASTIC_URL: http://localhost:${{ job.services.elastic.ports[9200] }}/
REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}
build-docker-images:
name: Build docker images
runs-on: ubuntu-latest

31
packages/import-handler/Dockerfile-collector Normal file
View File

@ -0,0 +1,31 @@
FROM node:14.18-alpine
WORKDIR /app
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
RUN apk add g++ make python3
ENV PORT 8080
COPY package.json .
COPY yarn.lock .
COPY tsconfig.json .
COPY .eslintrc .
COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
COPY /packages/import-handler/package.json ./packages/import-handler/package.json
RUN yarn install --pure-lockfile
ADD /packages/import-handler ./packages/import-handler
ADD /packages/readabilityjs ./packages/readabilityjs
RUN yarn workspace @omnivore/import-handler build
# After building, fetch the production dependencies
RUN rm -rf /app/packages/import-handler/node_modules
RUN rm -rf /app/node_modules
RUN yarn install --pure-lockfile --production
EXPOSE 8080
ENTRYPOINT ["yarn", "workspace", "@omnivore/import-handler", "start:collector"]

4
packages/import-handler/package.json
View File

@ -15,7 +15,7 @@
"build": "tsc && yarn copy-files",
"start": "functions-framework --target=importHandler",
"dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"",
"copy-files": "copyfiles -u 1 src/**/*.lua build/src",
"copy-files": "copyfiles src/luaScripts/*.lua build/",
"start:collector": "functions-framework --target=importMetricsCollector"
},
"devDependencies": {
@ -49,7 +49,7 @@
"jsonwebtoken": "^8.5.1",
"linkedom": "^0.14.21",
"nodemon": "^2.0.15",
"redis": "^4.6.6",
"redis": "^4.3.1",
"unzip-stream": "^0.3.1",
"urlsafe-base64": "^1.0.0",
"uuid": "^9.0.0"

8
packages/import-handler/src/index.ts
View File

@ -364,15 +364,15 @@ export const importMetricsCollector = Sentry.GCPFunction.wrapHttpFunction(
return res.status(401).send({ errorCode: 'UNAUTHENTICATED' })
}
const redisClient = await createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)
if (!isUpdateMetricsRequest(req.body)) {
console.log('Invalid request body')
return res.status(400).send('Bad Request')
}
const redisClient = await createRedisClient(
process.env.REDIS_URL,
process.env.REDIS_CERT
)
// update metrics
await updateMetrics(redisClient, userId, req.body.taskId, req.body.status)

233
packages/import-handler/test/csv/csv.test.ts
View File

@ -9,135 +9,134 @@ import { stubImportCtx } from '../util'
chai.use(chaiString)
describe('Load a simple CSV file', () => {
it('should call the handler for each URL', async () => {
const urls: URL[] = []
const stream = fs.createReadStream('./test/csv/data/simple.csv')
const stub = await stubImportCtx()
stub.urlHandler = (ctx: ImportContext, url): Promise<void> => {
urls.push(url)
return Promise.resolve()
}
describe('Test csv importer', () => {
let stub: ImportContext
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(2)
expect(urls).to.eql([
new URL('https://omnivore.app'),
new URL('https://google.com'),
])
beforeEach(async () => {
stub = await stubImportCtx()
})
afterEach(async () => {
await stub.redisClient.quit()
})
it('increments the failed count when the URL is invalid', async () => {
const urls: URL[] = []
const stream = fs.createReadStream('./test/csv/data/simple.csv')
const stub = await stubImportCtx()
stub.urlHandler = (ctx: ImportContext, url): Promise<void> => {
urls.push(url)
return Promise.reject('Failed to import url')
}
describe('Load a simple CSV file', () => {
it('should call the handler for each URL', async () => {
const urls: URL[] = []
const stream = fs.createReadStream('./test/csv/data/simple.csv')
stub.urlHandler = (ctx: ImportContext, url): Promise<void> => {
urls.push(url)
return Promise.resolve()
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(2)
expect(stub.countImported).to.equal(0)
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(2)
expect(urls).to.eql([
new URL('https://omnivore.app'),
new URL('https://google.com'),
])
})
await stub.redisClient.quit()
it('increments the failed count when the URL is invalid', async () => {
const stream = fs.createReadStream('./test/csv/data/simple.csv')
stub.urlHandler = (ctx: ImportContext, url): Promise<void> => {
return Promise.reject('Failed to import url')
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(2)
expect(stub.countImported).to.equal(0)
})
})
})
describe('Load a complex CSV file', () => {
it('should call the handler for each URL, state and labels', async () => {
const results: {
url: URL
state?: ArticleSavingRequestStatus
labels?: string[]
}[] = []
const stream = fs.createReadStream('./test/csv/data/complex.csv')
const stub = await stubImportCtx()
stub.urlHandler = (
ctx: ImportContext,
url,
state,
labels
): Promise<void> => {
results.push({
describe('Load a complex CSV file', () => {
it('should call the handler for each URL, state and labels', async () => {
const results: {
url: URL
state?: ArticleSavingRequestStatus
labels?: string[]
}[] = []
const stream = fs.createReadStream('./test/csv/data/complex.csv')
stub.urlHandler = (
ctx: ImportContext,
url,
state,
labels,
})
return Promise.resolve()
}
labels
): Promise<void> => {
results.push({
url,
state,
labels,
})
return Promise.resolve()
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(3)
expect(results).to.eql([
{
url: new URL('https://omnivore.app'),
state: 'ARCHIVED',
labels: ['test'],
},
{
url: new URL('https://google.com'),
state: 'SUCCEEDED',
labels: ['test', 'development'],
},
{
url: new URL('https://test.com'),
state: 'SUCCEEDED',
labels: ['test', 'development'],
},
])
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(3)
expect(results).to.eql([
{
url: new URL('https://omnivore.app'),
state: 'ARCHIVED',
labels: ['test'],
},
{
url: new URL('https://google.com'),
state: 'SUCCEEDED',
labels: ['test', 'development'],
},
{
url: new URL('https://test.com'),
state: 'SUCCEEDED',
labels: ['test', 'development'],
},
])
})
})
await stub.redisClient.quit()
})
})
describe('A file with no status set', () => {
it('should not try to set status', async () => {
const states: (ArticleSavingRequestStatus | undefined)[] = []
const stream = fs.createReadStream('./test/csv/data/unset-status.csv')
const stub = stubImportCtx()
stub.urlHandler = (
ctx: ImportContext,
url,
state?: ArticleSavingRequestStatus
): Promise<void> => {
states.push(state)
return Promise.resolve()
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(2)
expect(states).to.eql([undefined, ArticleSavingRequestStatus.Archived])
})
})
describe('A file with some labels', () => {
it('gets the labels, handles empty, and trims extra whitespace', async () => {
const importedLabels: (string[] | undefined)[] = []
const stream = fs.createReadStream('./test/csv/data/labels.csv')
const stub = stubImportCtx()
stub.urlHandler = (
ctx: ImportContext,
url,
state?: ArticleSavingRequestStatus,
labels?: string[]
): Promise<void> => {
importedLabels.push(labels)
return Promise.resolve()
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(3)
expect(importedLabels).to.eql([
['Label1', 'Label2', 'Label 3', 'Label 4'],
[],
[],
])
describe('A file with no status set', () => {
it('should not try to set status', async () => {
const states: (ArticleSavingRequestStatus | undefined)[] = []
const stream = fs.createReadStream('./test/csv/data/unset-status.csv')
stub.urlHandler = (
ctx: ImportContext,
url,
state?: ArticleSavingRequestStatus
): Promise<void> => {
states.push(state)
return Promise.resolve()
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(2)
expect(states).to.eql([undefined, ArticleSavingRequestStatus.Archived])
})
})
describe('A file with some labels', () => {
it('gets the labels, handles empty, and trims extra whitespace', async () => {
const importedLabels: (string[] | undefined)[] = []
const stream = fs.createReadStream('./test/csv/data/labels.csv')
stub.urlHandler = (
ctx: ImportContext,
url,
state?: ArticleSavingRequestStatus,
labels?: string[]
): Promise<void> => {
importedLabels.push(labels)
return Promise.resolve()
}
await importCsv(stub, stream)
expect(stub.countFailed).to.equal(0)
expect(stub.countImported).to.equal(3)
expect(importedLabels).to.eql([
['Label1', 'Label2', 'Label 3', 'Label 4'],
[],
[],
])
})
})
})

5
packages/import-handler/test/util.ts
View File

@ -2,8 +2,9 @@ import { Readability } from '@omnivore/readability'
import { ArticleSavingRequestStatus, ImportContext } from '../src'
import { createRedisClient } from '../src/redis'
export const stubImportCtx = async () => {
const redisClient = await createRedisClient()
export const stubImportCtx = async (): Promise<ImportContext> => {
const redisClient = await createRedisClient(process.env.REDIS_URL)
return {
userId: '',
countImported: 0,

4
packages/import-handler/tsconfig.json
View File

@ -1,6 +1,6 @@
{
"extends": "./../../tsconfig.json",
"ts-node": { "files": true },
"ts-node": { "files": true },
"compilerOptions": {
"outDir": "build",
"rootDir": ".",
@ -8,5 +8,5 @@
// Generate d.ts files
"declaration": true
},
"include": ["src", "test"]
"include": ["src/**/*", "test/**/*"]
}

51
yarn.lock
View File

@ -5431,11 +5431,6 @@
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.0.2.tgz#42b82ec399a92db05e29fffcdfd9235a5fc15cdf"
integrity sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==
"@redis/bloom@1.2.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==
"@redis/client@1.3.0":
version "1.3.0"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.3.0.tgz#c62ccd707f16370a2dc2f9e158a28b7da049fa77"
@ -5445,25 +5440,11 @@
generic-pool "3.8.2"
yallist "4.0.0"
"@redis/client@1.5.7":
version "1.5.7"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.7.tgz#92cc5c98c76f189e37d24f0e1e17e104c6af17d4"
integrity sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==
dependencies:
cluster-key-slot "1.1.2"
generic-pool "3.9.0"
yallist "4.0.0"
"@redis/graph@1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.0.1.tgz#eabc58ba99cd70d0c907169c02b55497e4ec8a99"
integrity sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==
"@redis/graph@1.1.0":
version "1.1.0"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519"
integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==
"@redis/json@1.0.4":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1"
@ -5474,21 +5455,11 @@
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.0.tgz#7abb18d431f27ceafe6bcb4dd83a3fa67e9ab4df"
integrity sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==
"@redis/search@1.1.2":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.2.tgz#6a8f66ba90812d39c2457420f859ce8fbd8f3838"
integrity sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==
"@redis/time-series@1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.3.tgz#4cfca8e564228c0bddcdf4418cba60c20b224ac4"
integrity sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==
"@redis/time-series@1.0.4":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717"
integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==
"@remusao/guess-url-type@^1.1.2":
version "1.2.1"
resolved "https://registry.yarnpkg.com/@remusao/guess-url-type/-/guess-url-type-1.2.1.tgz#b3e7c32abdf98d0fb4f93cc67cad580b5fe4ba57"
@ -11841,11 +11812,6 @@ cluster-key-slot@1.1.0:
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
cluster-key-slot@1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
cmd-shim@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd"
@ -15354,11 +15320,6 @@ generic-pool@3.8.2:
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
generic-pool@3.9.0:
version "3.9.0"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==
gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@ -24140,18 +24101,6 @@ redis@^4.3.1:
"@redis/search" "1.1.0"
"@redis/time-series" "1.0.3"
redis@^4.6.6:
version "4.6.6"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.6.tgz#46d4f2d149d1634d6ef53db5747412a0ef7974ec"
integrity sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==
dependencies:
"@redis/bloom" "1.2.0"
"@redis/client" "1.5.7"
"@redis/graph" "1.1.0"
"@redis/json" "1.0.4"
"@redis/search" "1.1.2"
"@redis/time-series" "1.0.4"
reflect-metadata@^0.1.13:
version "0.1.13"
resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"