diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 544f97dc6..4b37cfb2f 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -18,7 +18,7 @@ import { v4 as uuidv4 } from 'uuid' import { buildLogger } from '../../utils/logger' const MAX_DAILY_UPLOADS = 4 -const VALID_CONTENT_TYPES = ['text/csv'] +const VALID_CONTENT_TYPES = ['text/csv', 'application/zip'] const logger = buildLogger('app.dispatch') @@ -26,6 +26,8 @@ const extensionForContentType = (contentType: string) => { switch (contentType) { case 'text/csv': return 'csv' + case 'application/zip': + return 'zip' } return '.unknown' } diff --git a/packages/import-handler/Dockerfile b/packages/import-handler/Dockerfile new file mode 100644 index 000000000..f1e4d0dd2 --- /dev/null +++ b/packages/import-handler/Dockerfile @@ -0,0 +1,31 @@ +FROM node:14.18-alpine + +WORKDIR /app + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN apk add g++ make python3 + +ENV PORT 8080 + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json +COPY /packages/import-handler/package.json ./packages/import-handler/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/import-handler ./packages/import-handler +ADD /packages/readabilityjs ./packages/readabilityjs +RUN yarn workspace @omnivore/import-handler build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/import-handler/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +ENTRYPOINT ["yarn", "workspace", "@omnivore/import-handler", "start"] diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index fec6d9d45..2ca868a17 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -3,25 +3,30 @@ "version": "1.0.0", "description": "", "main": "build/src/index.js", - "types": "build/src/index.d.ts", "files": [ "build/src" ], - "license": "Apache-2.0", "keywords": [], + "license": "Apache-2.0", "scripts": { "test": "yarn mocha -r ts-node/register --config mocha-config.json", "lint": "eslint src --ext ts,js,tsx,jsx", "compile": "tsc", "build": "tsc", - "start": "functions-framework --source=build/src/ --target=importHandler", - "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"", - "gcloud-deploy": "gcloud functions deploy importHandler --region=$npm_config_region --runtime nodejs14 --trigger-bucket=$npm_config_bucket --env-vars-file=../gcf-shared/env-$npm_config_env.yaml", - "deploy": "yarn build && yarn gcloud-deploy" + "start": "functions-framework --target=importHandler", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" }, "devDependencies": { - "@types/node": "^14.11.2", + "@types/chai": "^4.3.4", + "@types/chai-string": "^1.4.2", + "@types/dompurify": "^2.4.0", + "@types/fs-extra": "^11.0.1", "@types/jsonwebtoken": "^8.5.0", + "@types/mocha": "^10.0.1", + "@types/node": "^14.11.2", + "@types/unzip-stream": "^0.3.1", + "@types/urlsafe-base64": "^1.0.28", + "@types/uuid": "^9.0.0", "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { @@ -29,9 +34,18 @@ "@google-cloud/functions-framework": "3.1.2", "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^3.0.5", + "@omnivore/readability": "1.0.0", + "@sentry/serverless": "^7.30.0", "@types/express": "^4.17.13", + "axios": "^1.2.2", "csv-parser": "^3.0.0", + "dompurify": "^2.4.3", + "fs-extra": "^11.1.0", "jsonwebtoken": "^8.5.1", - "nodemon": "^2.0.15" + "linkedom": "^0.14.21", + "nodemon": "^2.0.15", + "unzip-stream": "^0.3.1", + "urlsafe-base64": "^1.0.0", + "uuid": "^9.0.0" } -} +} \ No newline at end of file diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index 7f60abadd..c505cc39a 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -5,24 +5,19 @@ import { parse } from '@fast-csv/parse' import { Stream } from 'stream' +import { ImportContext } from '.' -export type UrlHandler = (url: URL) => Promise - -export const importCsv = async ( - stream: Stream, - handler: UrlHandler -): Promise => { +export const importCsv = async (ctx: ImportContext, stream: Stream) => { const parser = parse() stream.pipe(parser) - let count = 0 for await (const row of parser) { try { const url = new URL(row[0]) - await handler(url) + await ctx.urlHandler(ctx, url) + ctx.countImported += 1 } catch (error) { console.log('invalid url', row, error) + ctx.countFailed += 1 } - count++ } - return count } diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 0b37cab30..3333d56df 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -1,41 +1,62 @@ -import { - EventFunction, - CloudFunctionsContext, -} from '@google-cloud/functions-framework/build/src/functions' import { Storage } from '@google-cloud/storage' -import { importCsv, UrlHandler } from './csv' +import { importCsv } from './csv' import * as path from 'path' -import { importMatterHistory } from './matterHistory' +import { importMatterArchive, importMatterHistoryCsv } from './matterHistory' import { Stream } from 'node:stream' import { v4 as uuid } from 'uuid' -import { CONTENT_FETCH_URL, createCloudTask, EMAIL_USER_URL } from './task' +import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' +import axios from 'axios' import { promisify } from 'util' import * as jwt from 'jsonwebtoken' +import { Readability } from '@omnivore/readability' + +import * as Sentry from '@sentry/serverless' + +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) const signToken = promisify(jwt.sign) const storage = new Storage() -interface StorageEventData { - bucket: string +const CONTENT_TYPES = ['text/csv', 'application/zip'] + +export type UrlHandler = (ctx: ImportContext, url: URL) => Promise +export type ContentHandler = ( + ctx: ImportContext, + url: URL, + title: string, + originalContent: string, + parseResult: Readability.ParseResult +) => Promise + +export type ImportContext = { + userId: string + countImported: number + countFailed: number + urlHandler: UrlHandler + contentHandler: ContentHandler +} + +type importHandlerFunc = (ctx: ImportContext, stream: Stream) => Promise + +interface StorageEvent { name: string + bucket: string contentType: string } -type importHandlerFunc = ( - stream: Stream, - handler: UrlHandler -) => Promise +function isStorageEvent(event: any): event is StorageEvent { + return 'name' in event && 'bucket' in event && 'contentType' in event +} -const shouldHandle = (data: StorageEventData, ctx: CloudFunctionsContext) => { - console.log('deciding to handle', ctx, data) - if (ctx.eventType !== 'google.storage.object.finalize') { - return false - } +const shouldHandle = (data: StorageEvent) => { if ( !data.name.startsWith('imports/') || - data.contentType.toLowerCase() != 'text/csv' + CONTENT_TYPES.indexOf(data.contentType.toLocaleLowerCase()) == -1 ) { return false } @@ -69,7 +90,7 @@ const createEmailCloudTask = async (userId: string, payload: unknown) => { Cookie: `auth=${authToken}`, } - return createCloudTask(EMAIL_USER_URL, payload, headers) + return createCloudTask(emailUserUrl(), payload, headers) } const sendImportFailedEmail = async (userId: string) => { @@ -86,14 +107,14 @@ const sendImportCompletedEmail = async ( ) => { return createEmailCloudTask(userId, { subject: 'Your Omnivore import has completed processing', - body: `${urlsEnqueued} URLs have been pcoessed and should be available in your library. ${urlsFailed} URLs failed to be parsed.`, + body: `${urlsEnqueued} URLs have been processed and should be available in your library. ${urlsFailed} URLs failed to be parsed.`, }) } const handlerForFile = (name: string): importHandlerFunc | undefined => { const fileName = path.parse(name).name if (fileName.startsWith('MATTER')) { - return importMatterHistory + return importMatterArchive } else if (fileName.startsWith('URL_LIST')) { return importCsv } @@ -101,18 +122,79 @@ const handlerForFile = (name: string): importHandlerFunc | undefined => { return undefined } -export const importHandler: EventFunction = async (event, context) => { - const data = event as StorageEventData - const ctx = context as CloudFunctionsContext +const urlHandler = async (ctx: ImportContext, url: URL): Promise => { + try { + // Imports are stored in the format imports//-.csv + const result = await importURL(ctx.userId, url, 'csv-importer') + if (result) { + ctx.countImported += 1 + } + } catch (err) { + console.log('error importing url', err) + } +} - if (shouldHandle(data, ctx)) { - console.log('handling csv data', data) +const sendSavePageMutation = async (userId: string, input: unknown) => { + const JWT_SECRET = process.env.JWT_SECRET + const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT - const stream = storage - .bucket(data.bucket) - .file(data.name) - .createReadStream() + if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) { + throw 'Environment not configured correctly' + } + const data = JSON.stringify({ + query: `mutation SavePage ($input: SavePageInput!){ + savePage(input:$input){ + ... on SaveSuccess{ + url + clientRequestId + } + ... on SaveError{ + errorCodes + } + } + }`, + variables: { + input: Object.assign({}, input, { source: 'puppeteer-parse' }), + }, + }) + + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string + const response = await axios.post(`${REST_BACKEND_ENDPOINT}/graphql`, data, { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }) + console.log('save page response: ', response) + + /* eslint-disable @typescript-eslint/no-unsafe-member-access */ + return !!response.data.data.savePage +} + +const contentHandler = async ( + ctx: ImportContext, + url: URL, + title: string, + originalContent: string, + parseResult: Readability.ParseResult +): Promise => { + const requestId = uuid() + const apiResponse = await sendSavePageMutation(ctx.userId, { + url, + clientRequestId: requestId, + title, + originalContent, + parseResult, + }) + if (!apiResponse) { + return Promise.reject() + } + return Promise.resolve() +} + +const handleEvent = async (data: StorageEvent) => { + if (shouldHandle(data)) { const handler = handlerForFile(data.name) if (!handler) { console.log('no handler for file:', data.name) @@ -131,24 +213,54 @@ export const importHandler: EventFunction = async (event, context) => { return } - let countFailed = 0 - let countImported = 0 - await handler(stream, async (url): Promise => { - try { - // Imports are stored in the format imports//-.csv - const result = await importURL(userId, url, 'csv-importer') - console.log('import url result', result) - countImported = countImported + 1 - } catch (err) { - console.log('error importing url', err) - countFailed = countFailed + 1 - } - }) + const stream = storage + .bucket(data.bucket) + .file(data.name) + .createReadStream() - if (countImported <= 1) { - await sendImportFailedEmail(userId) + const ctx = { + userId, + countImported: 0, + countFailed: 0, + urlHandler, + contentHandler, + } + + await handler(ctx, stream) + + if (ctx.countImported > 0) { + await sendImportCompletedEmail(userId, ctx.countImported, ctx.countFailed) } else { - await sendImportCompletedEmail(userId, countImported, countFailed) + await sendImportFailedEmail(userId) } } } + +const getStorageEvent = (pubSubMessage: string): StorageEvent | undefined => { + try { + const str = Buffer.from(pubSubMessage, 'base64').toString().trim() + const obj = JSON.parse(str) as unknown + if (isStorageEvent(obj)) { + return obj + } + } catch (err) { + console.log('error deserializing event: ', { pubSubMessage, err }) + } + return undefined +} + +export const importHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + /* eslint-disable @typescript-eslint/no-unsafe-member-access */ + if ('message' in req.body && 'data' in req.body.message) { + const pubSubMessage = req.body.message.data as string + const obj = getStorageEvent(pubSubMessage) + if (obj) { + await handleEvent(obj) + } + } else { + console.log('no pubsub message') + } + res.send('ok') + } +) diff --git a/packages/import-handler/src/matterHistory.ts b/packages/import-handler/src/matterHistory.ts index 8342735b5..7d339d624 100644 --- a/packages/import-handler/src/matterHistory.ts +++ b/packages/import-handler/src/matterHistory.ts @@ -5,28 +5,234 @@ import { parse } from '@fast-csv/parse' import { Stream } from 'stream' +import unzip from 'unzip-stream' +import fs from 'fs' +import path from 'path' +import * as fsExtra from 'fs-extra' + +import { parseHTML } from 'linkedom' +import { Readability } from '@omnivore/readability' +import createDOMPurify, { SanitizeElementHookEvent } from 'dompurify' + +import { encode } from 'urlsafe-base64' +import crypto from 'crypto' +import { ImportContext } from '.' export type UrlHandler = (url: URL) => Promise -export const importMatterHistory = async ( - stream: Stream, - handler: UrlHandler -): Promise => { +export const importMatterHistoryCsv = async ( + ctx: ImportContext, + stream: Stream +): Promise => { const parser = parse({ headers: true, strictColumnHandling: false, }) stream.pipe(parser) - let count = 0 for await (const row of parser) { try { const url = new URL(row['URL']) - await handler(url) + await ctx.urlHandler(ctx, url) + ctx.countImported += 1 } catch (error) { console.log('invalid url', row, error) + ctx.countFailed += 1 + } + } +} + +const DOM_PURIFY_CONFIG = { + ADD_TAGS: ['iframe'], + ADD_ATTR: ['allow', 'allowfullscreen', 'frameborder', 'scrolling'], + FORBID_ATTR: [ + 'data-ml-dynamic', + 'data-ml-dynamic-type', + 'data-orig-url', + 'data-ml-id', + 'data-ml', + 'data-xid', + 'data-feature', + ], +} + +function domPurifySanitizeHook(node: Element, data: SanitizeElementHookEvent) { + if (data.tagName === 'iframe') { + const urlRegex = /^(https?:)?\/\/www\.youtube(-nocookie)?\.com\/embed\//i + const src = node.getAttribute('src') || '' + const dataSrc = node.getAttribute('data-src') || '' + + if (src && urlRegex.test(src)) { + return + } + + if (dataSrc && urlRegex.test(dataSrc)) { + node.setAttribute('src', dataSrc) + return + } + + node.parentNode?.removeChild(node) + } +} + +function getPurifiedContent(html: string) { + const newWindow = parseHTML('') + const DOMPurify = createDOMPurify(newWindow) + DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) + const clean = DOMPurify.sanitize(html, DOM_PURIFY_CONFIG) + return parseHTML(clean).document +} + +function createImageProxyUrl(url: string, width = 0, height = 0) { + if (process.env.IMAGE_PROXY_URL && process.env.IMAGE_PROXY_SECRET) { + const urlWithOptions = `${url}#${width}x${height}` + const signature = signImageProxyUrl(urlWithOptions) + + return `${process.env.IMAGE_PROXY_URL}/${width}x${height},s${signature}/${url}` + } + return url +} + +function signImageProxyUrl(url: string) { + if (process.env.IMAGE_PROXY_SECRET) { + return encode( + crypto + .createHmac('sha256', process.env.IMAGE_PROXY_SECRET) + .update(url) + .digest() + ) + } + return url +} + +async function getReadabilityResult(url: string, originalContent: string) { + const document = getPurifiedContent(originalContent) + + try { + const article = await new Readability(document, { + createImageProxyUrl, + url, + }).parse() + + if (article) { + return article + } + } catch (error) { + console.log('parsing error for url', url, error) + } + + return null +} + +const unarchive = async (stream: Stream): Promise => { + const archiveDir = `./archive-${Date.now().toString(16)}` + await fsExtra.emptyDir(archiveDir) + + return new Promise((resolve, reject) => { + stream + .pipe(unzip.Extract({ path: archiveDir })) + .on('close', () => { + resolve(archiveDir) + }) + .on('error', reject) + }) +} + +const getMatterHistoryContent = ( + archiveDir: string, + row: Record +) => { + try { + const contentKey = row['File Id'] + const contentPath = path.join(archiveDir, contentKey) + const content = fs.readFileSync(contentPath).toString() + + return content + } catch (err) { + console.log('error getting matter history content: ', { row, err }) + } + return undefined +} + +const getURL = (str: string | undefined) => { + if (!str) { + return undefined + } + + try { + const url = new URL(str) + return url + } catch (err) { + console.log('error parsing url', { str, err }) + } + + return undefined +} + +const handleMatterHistoryRow = async ( + ctx: ImportContext, + archiveDir: string, + row: Record +) => { + const title = row['Title'] + const urlStr = row['URL'] + const url = getURL(urlStr) + + if (!url) { + ctx.countFailed += 1 + return + } + + const originalContent = getMatterHistoryContent(archiveDir, row) + const readabilityResult = originalContent + ? await getReadabilityResult(urlStr, originalContent) + : null + + if (originalContent && readabilityResult) { + await ctx.contentHandler( + ctx, + url, + title, + originalContent, + readabilityResult + ) + } else { + await ctx.urlHandler(ctx, url) + } +} + +export const importMatterArchive = async ( + ctx: ImportContext, + stream: Stream +): Promise => { + const archiveDir = await unarchive(stream) + + try { + const historyFile = path.join(archiveDir, '_matter_history.csv') + + const parser = parse({ + headers: true, + strictColumnHandling: false, + }) + + fs.createReadStream(historyFile).pipe(parser) + + for await (const row of parser) { + try { + await handleMatterHistoryRow(ctx, archiveDir, row) + ctx.countImported += 1 + } catch (error) { + console.log('invalid url', row, error) + ctx.countFailed += 1 + } + } + } catch (err) { + console.log('error handling archive: ', { err }) + } finally { + try { + await fsExtra.rm(archiveDir, { recursive: true, force: true }) + } catch (err) { + console.log('Error removing archive directory', { err }) } - count++ } - return count } diff --git a/packages/import-handler/src/readability.d.ts b/packages/import-handler/src/readability.d.ts new file mode 100644 index 000000000..55bfc74d1 --- /dev/null +++ b/packages/import-handler/src/readability.d.ts @@ -0,0 +1,108 @@ +// Type definitions for non-npm package mozilla-readability 0.2 +// Project: https://github.com/mozilla/readability +// Definitions by: Charles Vandevoorde , Alex Wendland +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped +// TypeScript Version: 2.2 + +declare module '@omnivore/readability' { + class Readability { + constructor(doc: Document, options?: Readability.Options) + + async parse(): Promise + } + + namespace Readability { + interface Options { + /** + * Control whether log messages are sent to the console + */ + debug?: boolean + + /** + * Set a maximum size on the documents that will be processed. This size is + * checked before any parsing operations occur. If the number of elements in + * the document exceeds this threshold then an Error will be thrown. + * + * See implementation details at https://github.com/mozilla/readability/blob/52ab9b5c8916c306a47b2119270dcdabebf9d203/Readability.js#L2019 + */ + maxElemsToParse?: number + + nbTopCandidates?: number + + /** + * Minimum number of characters in the extracted textContent in order to + * consider the article correctly identified. If the threshold is not met then + * the extraction process will automatically run again with different flags. + * + * See implementation details at https://github.com/mozilla/readability/blob/52ab9b5c8916c306a47b2119270dcdabebf9d203/Readability.js#L1208 + * + * Changed from wordThreshold in https://github.com/mozilla/readability/commit/3ff9a166fb27928f222c4c0722e730eda412658a + */ + charThreshold?: number + + /** + * parse() removes the class="" attribute from every element in the given + * subtree, except those that match CLASSES_TO_PRESERVE and + * the classesToPreserve array from the options object. + */ + classesToPreserve?: string[] + + /** + * By default Readability will strip all classes from the HTML elements in the + * processed article. By setting this to `true` the classes will be retained. + * + * This is a blanket alternative to `classesToPreserve`. + * + * Added in https://github.com/mozilla/readability/commit/2982216913af2c66b0690e88606b03116553ad92 + */ + + keepClasses?: boolean + url?: string + + /** + * Function that converts a regular image url into imageproxy url + * @param url string + */ + createImageProxyUrl?: ( + url: string, + width?: number, + height?: number + ) => string + + /** + * By default, Readability will clean all tables from the HTML elements in the + * processed article. But newsletters in emails use tables to display their content. + * By setting this to `true`, these tables will be retained. + */ + keepTables?: boolean + } + + interface ParseResult { + /** Article title */ + title: string + /** Author metadata */ + byline?: string | null + /** Content direction */ + dir?: string | null + /** HTML string of processed article content */ + content: string + /** non-HTML version of `content` */ + textContent: string + /** Length of an article, in characters */ + length: number + /** Article description, or short excerpt from the content */ + excerpt: string + /** Article site name */ + siteName?: string | null + /** Article site icon */ + siteIcon?: string | null + /** Article preview image */ + previewImage?: string | null + /** Article published date */ + publishedDate?: Date | null + language?: string | null + } + } + + export { Readability } +} diff --git a/packages/import-handler/src/task.ts b/packages/import-handler/src/task.ts index f35c31a2d..16f66b08d 100644 --- a/packages/import-handler/src/task.ts +++ b/packages/import-handler/src/task.ts @@ -3,12 +3,13 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks' const cloudTask = new CloudTasksClient() -export const EMAIL_USER_URL = (() => { - if (!process.env.INTERNAL_SVC_ENDPOINT) { - throw `Environment not configured correctly, no SVC endpoint` +export const emailUserUrl = () => { + const envar = process.env.INTERNAL_SVC_ENDPOINT + if (envar) { + return envar + 'api/user/email' } - return (process.env.INTERNAL_SVC_ENDPOINT ?? '') + 'api/user/email' -})() + throw 'INTERNAL_SVC_ENDPOINT not set' +} export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 4ea3ef705..0f695d69e 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -4,6 +4,8 @@ import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' import { importCsv } from '../../src/csv' +import { ImportContext } from '../../src' +import { stubImportCtx } from '../util' chai.use(chaiString) @@ -11,11 +13,15 @@ describe('Load a simple CSV file', () => { it('should call the handler for each URL', async () => { const urls: URL[] = [] const stream = fs.createReadStream('./test/csv/data/simple.csv') - const count = await importCsv(stream, (url): Promise => { + const stub = stubImportCtx() + stub.urlHandler = (ctx: ImportContext, url): Promise => { urls.push(url) return Promise.resolve() - }) - expect(count).to.equal(2) + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(2) expect(urls).to.eql([ new URL('https://omnivore.app'), new URL('https://google.com'), diff --git a/packages/import-handler/test/matter/data/Archive.zip b/packages/import-handler/test/matter/data/Archive.zip new file mode 100644 index 000000000..f63012484 Binary files /dev/null and b/packages/import-handler/test/matter/data/Archive.zip differ diff --git a/packages/import-handler/test/matter/matter_importer.test.ts b/packages/import-handler/test/matter/matter_importer.test.ts index 6a3b24e98..90600b16a 100644 --- a/packages/import-handler/test/matter/matter_importer.test.ts +++ b/packages/import-handler/test/matter/matter_importer.test.ts @@ -3,7 +3,13 @@ import * as chai from 'chai' import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' -import { importMatterHistory } from '../../src/matterHistory' +import { + importMatterArchive, + importMatterHistoryCsv, +} from '../../src/matterHistory' +import { stubImportCtx } from '../util' +import { ImportContext } from '../../src' +import { Readability } from '@omnivore/readability' chai.use(chaiString) @@ -11,11 +17,40 @@ describe('Load a simple _matter_history file', () => { it('should find the URL of each row', async () => { const urls: URL[] = [] const stream = fs.createReadStream('./test/matter/data/_matter_history.csv') - const count = await importMatterHistory(stream, (url): Promise => { + const stub = stubImportCtx() + stub.urlHandler = (ctx: ImportContext, url): Promise => { urls.push(url) return Promise.resolve() - }) - expect(count).to.equal(1) + } + + await importMatterHistoryCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(1) + expect(urls).to.eql([ + new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'), + ]) + }) +}) + +describe('Load archive file', () => { + it('should find the URL of each row', async () => { + const urls: URL[] = [] + const stream = fs.createReadStream('./test/matter/data/Archive.zip') + const stub = stubImportCtx() + stub.contentHandler = ( + ctx: ImportContext, + url: URL, + title: string, + originalContent: string, + parseResult: Readability.ParseResult + ): Promise => { + urls.push(url) + return Promise.resolve() + } + + await importMatterArchive(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(1) expect(urls).to.eql([ new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'), ]) diff --git a/packages/import-handler/test/util.ts b/packages/import-handler/test/util.ts new file mode 100644 index 000000000..58cd9dd10 --- /dev/null +++ b/packages/import-handler/test/util.ts @@ -0,0 +1,22 @@ +import { Readability } from '@omnivore/readability' +import { ImportContext } from '../src' + +export const stubImportCtx = () => { + return { + userId: '', + countImported: 0, + countFailed: 0, + urlHandler: (ctx: ImportContext, url: URL): Promise => { + return Promise.resolve() + }, + contentHandler: ( + ctx: ImportContext, + url: URL, + title: string, + originalContent: string, + parseResult: Readability.ParseResult + ): Promise => { + return Promise.resolve() + }, + } +} diff --git a/packages/import-handler/tsconfig.json b/packages/import-handler/tsconfig.json index f450acf38..ea8c4d3ef 100644 --- a/packages/import-handler/tsconfig.json +++ b/packages/import-handler/tsconfig.json @@ -1,9 +1,12 @@ { - "extends": "@tsconfig/node14/tsconfig.json", + "extends": "./../../tsconfig.json", + "ts-node": { "files": true }, "compilerOptions": { "outDir": "build", "rootDir": ".", - "lib": ["dom"] + "lib": ["dom"], + // Generate d.ts files + "declaration": true }, "include": ["src", "test"] } diff --git a/packages/web/components/templates/ConfirmProfileModal.tsx b/packages/web/components/templates/ConfirmProfileModal.tsx index 72a939947..a61766932 100644 --- a/packages/web/components/templates/ConfirmProfileModal.tsx +++ b/packages/web/components/templates/ConfirmProfileModal.tsx @@ -15,9 +15,8 @@ export function ConfirmProfileModal(): JSX.Element { const [username, setUsername] = useState('') const [debouncedUsername, setDebouncedUsername] = useState('') const [bio, setBio] = useState('') - const [errorMessage, setErrorMessage] = useState( - undefined - ) + const [errorMessage, setErrorMessage] = + useState(undefined) const { isUsernameValid, usernameErrorMessage } = useValidateUsernameQuery({ username: debouncedUsername, @@ -110,8 +109,9 @@ export function ConfirmProfileModal(): JSX.Element { @@ -139,7 +139,12 @@ export function ConfirmProfileModal(): JSX.Element { {isUsernameValid && ( Username is available. diff --git a/packages/web/components/templates/ProfileLayout.tsx b/packages/web/components/templates/ProfileLayout.tsx index 874b30c89..e2d3d6b1a 100644 --- a/packages/web/components/templates/ProfileLayout.tsx +++ b/packages/web/components/templates/ProfileLayout.tsx @@ -12,7 +12,12 @@ export function ProfileLayout(props: ProfileLayoutProps): JSX.Element { {props.children} diff --git a/packages/web/components/templates/auth/EmailForgotPassword.tsx b/packages/web/components/templates/auth/EmailForgotPassword.tsx index b2fca77da..c97a222fc 100644 --- a/packages/web/components/templates/auth/EmailForgotPassword.tsx +++ b/packages/web/components/templates/auth/EmailForgotPassword.tsx @@ -12,9 +12,8 @@ import { parseErrorCodes } from '../../../lib/queryParamParser' export function EmailForgotPassword(): JSX.Element { const router = useRouter() const [email, setEmail] = useState('') - const [errorMessage, setErrorMessage] = useState( - undefined - ) + const [errorMessage, setErrorMessage] = + useState(undefined) useEffect(() => { if (!router.isReady) return @@ -36,7 +35,8 @@ export function EmailForgotPassword(): JSX.Element { width: '70vw', maxWidth: '576px', borderRadius: '8px', - boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px', + border: '1px solid #3D3D3D', + boxShadow: '#B1B1B1 9px 9px 9px -9px', }} > diff --git a/packages/web/components/templates/auth/EmailLogin.tsx b/packages/web/components/templates/auth/EmailLogin.tsx index 6d3533cb9..8175f329f 100644 --- a/packages/web/components/templates/auth/EmailLogin.tsx +++ b/packages/web/components/templates/auth/EmailLogin.tsx @@ -15,9 +15,8 @@ export function EmailLogin(): JSX.Element { const router = useRouter() const [email, setEmail] = useState(undefined) const [password, setPassword] = useState(undefined) - const [errorMessage, setErrorMessage] = useState( - undefined - ) + const [errorMessage, setErrorMessage] = + useState(undefined) useEffect(() => { if (!router.isReady) return @@ -39,7 +38,8 @@ export function EmailLogin(): JSX.Element { width: '70vw', maxWidth: '576px', borderRadius: '8px', - boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px', + border: '1px solid #3D3D3D', + boxShadow: '#B1B1B1 9px 9px 9px -9px', }} > diff --git a/packages/web/components/templates/auth/EmailResetPassword.tsx b/packages/web/components/templates/auth/EmailResetPassword.tsx index eafc284b2..10f378c8b 100644 --- a/packages/web/components/templates/auth/EmailResetPassword.tsx +++ b/packages/web/components/templates/auth/EmailResetPassword.tsx @@ -13,9 +13,8 @@ export function EmailResetPassword(): JSX.Element { const router = useRouter() const [token, setToken] = useState(undefined) const [password, setPassword] = useState('') - const [errorMessage, setErrorMessage] = useState( - undefined - ) + const [errorMessage, setErrorMessage] = + useState(undefined) useEffect(() => { if (!router.isReady) return @@ -45,7 +44,8 @@ export function EmailResetPassword(): JSX.Element { width: '70vw', maxWidth: '576px', borderRadius: '8px', - boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px', + border: '1px solid #3D3D3D', + boxShadow: '#B1B1B1 9px 9px 9px -9px', }} > (undefined) const [fullname, setFullname] = useState(undefined) const [username, setUsername] = useState(undefined) - const [debouncedUsername, setDebouncedUsername] = useState< - string | undefined - >(undefined) - const [errorMessage, setErrorMessage] = useState( - undefined - ) + const [debouncedUsername, setDebouncedUsername] = + useState(undefined) + const [errorMessage, setErrorMessage] = + useState(undefined) useEffect(() => { if (!router.isReady) return @@ -60,7 +58,8 @@ export function EmailSignup(): JSX.Element { width: '70vw', maxWidth: '576px', borderRadius: '8px', - boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px', + border: '1px solid #3D3D3D', + boxShadow: '#B1B1B1 9px 9px 9px -9px', }} > diff --git a/packages/web/pages/invite/[inviteCode].tsx b/packages/web/pages/invite/[inviteCode].tsx index 45e23276f..c95a7365d 100644 --- a/packages/web/pages/invite/[inviteCode].tsx +++ b/packages/web/pages/invite/[inviteCode].tsx @@ -17,9 +17,8 @@ export default function InvitePage(): JSX.Element { const router = useRouter() const { viewerData, viewerDataError, isLoading } = useGetViewerQuery() const { inviteCode } = router.query - const [errorMessage, setErrorMessage] = useState( - undefined - ) + const [errorMessage, setErrorMessage] = + useState(undefined) // Check if the user is logged in and display an error message if they are not useEffect(() => { @@ -77,7 +76,8 @@ export default function InvitePage(): JSX.Element { width: '70vw', maxWidth: '576px', borderRadius: '8px', - boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px', + border: '1px solid #3D3D3D', + boxShadow: '#B1B1B1 9px 9px 9px -9px', }} > diff --git a/packages/web/pages/tools/import/file.tsx b/packages/web/pages/tools/import/file.tsx index d49b551b7..44096d324 100644 --- a/packages/web/pages/tools/import/file.tsx +++ b/packages/web/pages/tools/import/file.tsx @@ -96,7 +96,8 @@ export default function ImportUploader(): JSX.Element { width: '70vw', maxWidth: '576px', borderRadius: '8px', - boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px', + border: '1px solid #3D3D3D', + boxShadow: '#B1B1B1 9px 9px 9px -9px', }} > () + const [file, setFile] = useState() + const [type, setType] = useState() + const [uploadState, setUploadState] = useState('none') + const { acceptedFiles, getRootProps, getInputProps } = useDropzone() + + const onDropAccepted = async (acceptedFiles: File[]) => { + const contentType = 'application/zip' + const file = acceptedFiles.find(() => true) + if (!file) { + setErrorMessage('No file selected.') + return + } + + setUploadState('uploading') + + try { + const result = await uploadImportFileRequestMutation( + UploadImportFileType.MATTER, + contentType + ) + + if (result && result.uploadSignedUrl) { + const uploadRes = await fetch(result.uploadSignedUrl, { + method: 'PUT', + body: file, + headers: { + 'content-type': contentType, + 'content-length': `${file.size}`, + }, + }) + setUploadState('completed') + } else { + setErrorMessage( + 'Unable to create file upload. Please ensure you are logged in.' + ) + setUploadState('none') + } + } catch (error) { + console.log('caught error', error) + if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') { + setErrorMessage('You have exceeded your maximum daily upload limit.') + } + setUploadState('none') + } + } + + return ( + + + + Import Matter Archive + + + Omnivore supports uploading the Archive.zip file generated by + exporting your data from the Matter app. + + + To export your data from Matter, go to My Account, and choose Export + data, this will send you an email with your data in a file + Archive.zip. Upload that file using the uploader on this page. + + + More info + + + Note: Please note you are limited to three import uploads per a + day, and the maximum file size is 10MB. + + + {uploadState == 'completed' ? ( + + Your upload has completed. Please note that it can take some time + for your library to be updated. You will be sent an email when the + process completes. + + ) : ( + <> + + + {({ getRootProps, getInputProps }) => ( +
+ + + {uploadState == 'uploading' ? ( + + ) : ( + <> + + + Click or Drag Archive.zip file here to upload + + + )} + +
+ )} + + + + )} + + {uploadState == 'completed' && ( + + + + )} + + {errorMessage && ( + {errorMessage} + )} + + + + ) +} diff --git a/yarn.lock b/yarn.lock index d469f24e8..8a8741388 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5526,6 +5526,15 @@ "@sentry/utils" "6.19.6" tslib "^1.9.3" +"@sentry/core@7.30.0": + version "7.30.0" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.30.0.tgz#02d2e8747484ba64b6d712e8afe6736250efdc26" + integrity sha512-NeLigkBlpcK63ymM63GoIHurml6V3BUe1Vi+trwm4/qqOTzT7PQhvdJCX+o3+atzRBH+zdb6kd4VWx44Oye3KA== + dependencies: + "@sentry/types" "7.30.0" + "@sentry/utils" "7.30.0" + tslib "^1.9.3" + "@sentry/hub@5.30.0": version "5.30.0" resolved "https://registry.yarnpkg.com/@sentry/hub/-/hub-5.30.0.tgz#2453be9b9cb903404366e198bd30c7ca74cdc100" @@ -5652,6 +5661,19 @@ lru_map "^0.3.3" tslib "^1.9.3" +"@sentry/node@7.30.0": + version "7.30.0" + resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.30.0.tgz#42ef5b29d065f6de6ac5556a56aca20d3b9073e1" + integrity sha512-YYasu6C3I0HBP4N1oc/ed2nunxhGJgtAWaKwq3lo8uk3uF6cB1A8+2e0CpjzU5ejhbaFPUBxHyj4th39Bvku/w== + dependencies: + "@sentry/core" "7.30.0" + "@sentry/types" "7.30.0" + "@sentry/utils" "7.30.0" + cookie "^0.4.1" + https-proxy-agent "^5.0.0" + lru_map "^0.3.3" + tslib "^1.9.3" + "@sentry/node@^5.26.0": version "5.30.0" resolved "https://registry.yarnpkg.com/@sentry/node/-/node-5.30.0.tgz#4ca479e799b1021285d7fe12ac0858951c11cd48" @@ -5693,6 +5715,19 @@ "@types/express" "^4.17.2" tslib "^1.9.3" +"@sentry/serverless@^7.30.0": + version "7.30.0" + resolved "https://registry.yarnpkg.com/@sentry/serverless/-/serverless-7.30.0.tgz#f639c835043de2c3dec10f3667fcf8545a557e24" + integrity sha512-Fs6fuyDR+RJ5yg/m4M9Ids+jVc9FJbE8XBxOGZkpYkjZFLWVETMI3tJt4cF31LOQF/irftr6pKW/blxHAknBrg== + dependencies: + "@sentry/node" "7.30.0" + "@sentry/tracing" "7.30.0" + "@sentry/types" "7.30.0" + "@sentry/utils" "7.30.0" + "@types/aws-lambda" "^8.10.62" + "@types/express" "^4.17.14" + tslib "^1.9.3" + "@sentry/tracing@5.30.0": version "5.30.0" resolved "https://registry.yarnpkg.com/@sentry/tracing/-/tracing-5.30.0.tgz#501d21f00c3f3be7f7635d8710da70d9419d4e1f" @@ -5726,6 +5761,16 @@ "@sentry/utils" "6.19.6" tslib "^1.9.3" +"@sentry/tracing@7.30.0": + version "7.30.0" + resolved "https://registry.yarnpkg.com/@sentry/tracing/-/tracing-7.30.0.tgz#b4576fbaf81ce418f1b7c37e7e5f4f6cf19a3c3b" + integrity sha512-bjGeDeKhpGAmLcWcrXFT/xOfHVwp/j0L1aRHzYHnqgTjVzD0NXcooPu/Nz8vF0paxz+hPD5bJwb8kz/ggJzGWQ== + dependencies: + "@sentry/core" "7.30.0" + "@sentry/types" "7.30.0" + "@sentry/utils" "7.30.0" + tslib "^1.9.3" + "@sentry/tracing@^7.9.0": version "7.9.0" resolved "https://registry.yarnpkg.com/@sentry/tracing/-/tracing-7.9.0.tgz#0cbbf5b61ee76b934d2e4160a0ad3daf0001237b" @@ -5756,6 +5801,11 @@ resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.10.0.tgz#c91d634768336238ac30ed750fa918326c384cbb" integrity sha512-1UBwdbS0xXzANzp63g4eNQly/qKIXp0swP5OTKWoADvKBtL4anroLUA/l8ADMtuwFZYtVANc8WRGxM2+YmaXtg== +"@sentry/types@7.30.0": + version "7.30.0" + resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.30.0.tgz#fc2baeb5b0e1ecc4d52b07b056fcba54449cd9ce" + integrity sha512-l4A86typvt/SfWh5JffpdxNGkg5EEA8m35BzpIcKmCAQZUDmnb4b478r8jdD2uuOjLmPNmZr1tifdRW4NCLuxQ== + "@sentry/types@7.9.0": version "7.9.0" resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.9.0.tgz#8fa952865fda76f7c7c7fc6c84043979a22043ae" @@ -5793,6 +5843,14 @@ "@sentry/types" "7.10.0" tslib "^1.9.3" +"@sentry/utils@7.30.0": + version "7.30.0" + resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.30.0.tgz#1d83145399c65e31f725c1b6ae02f451a990f326" + integrity sha512-tSlBhr5u/LdE2emxIDTDmjmyRr99GnZGIAh5GwRxUgeDQ3VEfNUFlyFodBCbZ6yeYTYd6PWNih5xoHn1+Rf3Sw== + dependencies: + "@sentry/types" "7.30.0" + tslib "^1.9.3" + "@sentry/utils@7.9.0": version "7.9.0" resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.9.0.tgz#497c41efe1b32974208ca68570e42c853b874f77" @@ -7865,6 +7923,11 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.21.tgz#9f35a5643129df132cf3b5c1ec64046ea1af0650" integrity sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg== +"@types/chai@^4.3.4": + version "4.3.4" + resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.4.tgz#e913e8175db8307d78b4e8fa690408ba6b65dee4" + integrity sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw== + "@types/chrome@^0.0.197": version "0.0.197" resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35" @@ -7948,6 +8011,13 @@ dependencies: "@types/trusted-types" "*" +"@types/dompurify@^2.4.0": + version "2.4.0" + resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.4.0.tgz#fd9706392a88e0e0e6d367f3588482d817df0ab9" + integrity sha512-IDBwO5IZhrKvHFUl+clZxgf3hn2b/lU6H1KaBShPkQyGJUQ0xwebezIPSuiyGwfz1UzJWQl4M7BDxtHtCCPlTg== + dependencies: + "@types/trusted-types" "*" + "@types/duplexify@^3.6.0": version "3.6.0" resolved "https://registry.yarnpkg.com/@types/duplexify/-/duplexify-3.6.0.tgz#dfc82b64bd3a2168f5bd26444af165bf0237dcd8" @@ -8009,7 +8079,7 @@ dependencies: "@types/express" "*" -"@types/express@*", "@types/express@^4.16.0", "@types/express@^4.17.13", "@types/express@^4.17.2", "@types/express@^4.17.7": +"@types/express@*", "@types/express@^4.16.0", "@types/express@^4.17.13", "@types/express@^4.17.14", "@types/express@^4.17.2", "@types/express@^4.17.7": version "4.17.15" resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.15.tgz#9290e983ec8b054b65a5abccb610411953d417ff" integrity sha512-Yv0k4bXGOH+8a+7bELd2PqHQsuiANB+A8a4gnQrkRWzrkKlb6KHaVvyXhqs04sVW/OWlbPyYxRgYlIXLfrufMQ== @@ -8058,6 +8128,14 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^11.0.1": + version "11.0.1" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.1.tgz#f542ec47810532a8a252127e6e105f487e0a6ea5" + integrity sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA== + dependencies: + "@types/jsonfile" "*" + "@types/node" "*" + "@types/glob@*": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" @@ -8225,6 +8303,13 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/jsonfile@*": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.1.tgz#ac84e9aefa74a2425a0fb3012bdea44f58970f1b" + integrity sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png== + dependencies: + "@types/node" "*" + "@types/jsonwebtoken@^8.5.0": version "8.5.0" resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-8.5.0.tgz#2531d5e300803aa63279b232c014acf780c981c5" @@ -8313,6 +8398,11 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== +"@types/mocha@^10.0.1": + version "10.0.1" + resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.1.tgz#2f4f65bb08bc368ac39c96da7b2f09140b26851b" + integrity sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q== + "@types/mocha@^8.2.2": version "8.2.3" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" @@ -8653,10 +8743,17 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== +"@types/unzip-stream@^0.3.1": + version "0.3.1" + resolved "https://registry.yarnpkg.com/@types/unzip-stream/-/unzip-stream-0.3.1.tgz#a57b8365be3d7f89960c755c8cc37c83223dc7d0" + integrity sha512-RlE3qaqvu4XaMwxkG/zR1gIunCbqXvNrmZ4BCG7OiQ8QUactFUPxm0TTrOCRJZQfPW3T6XBH7PcHQiiqkdcijw== + dependencies: + "@types/node" "*" + "@types/urlsafe-base64@^1.0.28": version "1.0.28" resolved "https://registry.yarnpkg.com/@types/urlsafe-base64/-/urlsafe-base64-1.0.28.tgz#2cf2098518e98c730a7e00de79a455e0269a3f4d" - integrity sha1-LPIJhRjpjHMKfgDeeaRV4CaaP00= + integrity sha512-TG5dKbqx75FUTXfiARIPvLvMCImVYJbKM+Fvy9HgpxkunHnMHNAn78xpvcZxIbPITyRzf0b2Gl8fnd1Ja3p1eQ== dependencies: "@types/node" "*" @@ -8665,6 +8762,11 @@ resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc" integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== +"@types/uuid@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.0.tgz#53ef263e5239728b56096b0a869595135b7952d2" + integrity sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q== + "@types/voca@^1.4.0": version "1.4.2" resolved "https://registry.yarnpkg.com/@types/voca/-/voca-1.4.2.tgz#4d2ceef3582c2a1f6a1574f71f0897400d910583" @@ -10164,6 +10266,15 @@ axios@^1.2.0: form-data "^4.0.0" proxy-from-env "^1.1.0" +axios@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.2.2.tgz#72681724c6e6a43a9fea860fc558127dbe32f9f1" + integrity sha512-bz/J4gS2S3I7mpN/YZfGFTqhXTYzRho8Ay38w2otuuDR322KzFIWm/4W2K6gIwvWaws5n+mnb7D1lN9uD+QH6Q== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + axobject-query@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be" @@ -10494,6 +10605,14 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== +binary@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79" + integrity sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg== + dependencies: + buffers "~0.1.1" + chainsaw "~0.1.0" + binaryextensions@^2.1.2: version "2.3.0" resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-2.3.0.tgz#1d269cbf7e6243ea886aa41453c3651ccbe13c22" @@ -10793,6 +10912,11 @@ buffer@^6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" +buffers@~0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb" + integrity sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ== + bufrw@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/bufrw/-/bufrw-1.3.0.tgz#28d6cfdaf34300376836310f5c31d57eeb40c8fa" @@ -11089,6 +11213,13 @@ chai@^4.3.6: pathval "^1.1.1" type-detect "^4.0.5" +chainsaw@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98" + integrity sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ== + dependencies: + traverse ">=0.3.0 <0.4" + chalk@^1.0.0, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -13062,6 +13193,11 @@ dompurify@^2.4.1: resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.1.tgz#f9cb1a275fde9af6f2d0a2644ef648dd6847b631" integrity sha512-ewwFzHzrrneRjxzmK6oVz/rZn9VWspGFRDb4/rRtIsM1n36t9AKma/ye8syCpcw+XJ25kOK/hOG7t1j2I2yBqA== +dompurify@^2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.3.tgz#f4133af0e6a50297fc8874e2eaedc13a3c308c03" + integrity sha512-q6QaLcakcRjebxjg8/+NP+h0rPfatOgOzc46Fst9VAA3jF2ApfKBNKMzdP4DYTqtUMXSCd5pRS/8Po/OmoCHZQ== + domutils@^2.0.0, domutils@^2.5.2: version "2.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442" @@ -14783,6 +14919,15 @@ fs-extra@^10.0.0: jsonfile "^6.0.1" universalify "^2.0.0" +fs-extra@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed" + integrity sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -18414,6 +18559,17 @@ linkedom@^0.14.16: htmlparser2 "^8.0.1" uhyphen "^0.1.0" +linkedom@^0.14.21: + version "0.14.21" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.21.tgz#878e1e5e88028cb1d57bc6262f84484a41a37497" + integrity sha512-V+c0AAFMTVJA2iAhrdd+u44lL0TjL6hBenVB061VQ6BHqTAHtXw1v5F1/CHGKtwg0OHm+hrGbepb9ZSFJ7lJkg== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^8.0.1" + uhyphen "^0.1.0" + linkedom@^0.14.9: version "0.14.9" resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0" @@ -25566,6 +25722,11 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== +"traverse@>=0.3.0 <0.4": + version "0.3.9" + resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" + integrity sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ== + tree-kill@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" @@ -26151,6 +26312,14 @@ untildify@^4.0.0: resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== +unzip-stream@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/unzip-stream/-/unzip-stream-0.3.1.tgz#2333b5cd035d29db86fb701ca212cf8517400083" + integrity sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw== + dependencies: + binary "^0.3.0" + mkdirp "^0.5.1" + upath@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"