diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 08267ab14..4843ebf70 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -375,6 +375,9 @@ describe('Integrations routers', () => { '1234': { tag: 'test', }, + '1235': { + tag: 'new', + }, }, }, }, diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index ec14ef577..ef7c0fb32 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -42,7 +42,6 @@ "@sentry/serverless": "^7.30.0", "@types/express": "^4.17.13", "axios": "^1.2.2", - "csv-parser": "^3.0.0", "dompurify": "^2.4.3", "fs-extra": "^11.1.0", "glob": "^8.1.0", diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index 70bb8b955..1df145e34 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -8,6 +8,22 @@ import { Stream } from 'stream' import { ImportContext } from '.' import { createMetrics, ImportStatus, updateMetrics } from './metrics' +const parseLabels = (labels: string): string[] => { + try { + // labels follows format: "[""label1"",""label2""]" + return JSON.parse(labels) as string[] + } catch (error) { + console.debug('invalid labels format', labels) + + // labels follows format: "[label1,label2]" + return labels + .slice(1, -1) + .split(',') + .map((l) => l.trim()) + .filter((l) => l !== '') + } +} + export const importCsv = async (ctx: ImportContext, stream: Stream) => { // create metrics in redis await createMetrics(ctx.redisClient, ctx.userId, ctx.taskId, 'csv-importer') @@ -18,15 +34,7 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { try { const url = new URL(row[0]) const state = row.length > 1 && row[1] ? row[1] : undefined - // labels follows format: "[label1,label2]" - const labels = - row.length > 2 - ? (row[2] as string) - .slice(1, -1) - .split(',') - .map((l) => l.trim()) - .filter((l) => l !== '') - : undefined + const labels = row.length > 2 ? parseLabels(row[2]) : undefined // update total counter await updateMetrics( diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 886aea8f5..29f6364e0 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -127,7 +127,12 @@ const createEmailCloudTask = async (userId: string, payload: unknown) => { Cookie: `auth=${authToken}`, } - return createCloudTask(emailUserUrl(), payload, headers) + return createCloudTask( + emailUserUrl(), + payload, + headers, + 'omnivore-email-queue' + ) } const sendImportFailedEmail = async (userId: string) => { diff --git a/packages/import-handler/test/csv/data/complex.csv b/packages/import-handler/test/csv/data/complex.csv index fa4ce8b83..6b8968660 100644 --- a/packages/import-handler/test/csv/data/complex.csv +++ b/packages/import-handler/test/csv/data/complex.csv @@ -1,3 +1,3 @@ -"https://omnivore.app",ARCHIVED,"[test]" -"https://google.com",SUCCEEDED,"[test,development]" +"https://omnivore.app",ARCHIVED,"[""test""]" +"https://google.com",SUCCEEDED,"[""test"",""development""]" https://test.com,SUCCEEDED,"[test, development]" diff --git a/yarn.lock b/yarn.lock index e38aa98fa..2655b1575 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12689,13 +12689,6 @@ csstype@^3.0.2, csstype@^3.0.4: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.8.tgz#d2266a792729fb227cd216fb572f43728e1ad340" integrity sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw== -csv-parser@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-3.0.0.tgz#b88a6256d79e090a97a1b56451f9327b01d710e7" - integrity sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ== - dependencies: - minimist "^1.2.0" - csv-stringify@*, csv-stringify@^6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.4.0.tgz#6d006dca9194700e44f9fbc541bee8bbbd4f459c"