diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 1161e0d09..de3fc67ba 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -28,7 +28,7 @@ jobs: ports: - 5432 elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 env: discovery.type: single-node http.cors.allow-origin: '*' diff --git a/apple/Omnivore.xcodeproj/project.pbxproj b/apple/Omnivore.xcodeproj/project.pbxproj index 4625c4ea4..34a7c938f 100644 --- a/apple/Omnivore.xcodeproj/project.pbxproj +++ b/apple/Omnivore.xcodeproj/project.pbxproj @@ -1229,7 +1229,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 53; + CURRENT_PROJECT_VERSION = 60; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; INFOPLIST_FILE = InfoPlists/ShareExtensionMac.plist; @@ -1239,7 +1239,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.ShareExtension-Mac"; @@ -1261,7 +1261,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 53; + CURRENT_PROJECT_VERSION = 60; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; INFOPLIST_FILE = InfoPlists/ShareExtensionMac.plist; @@ -1271,7 +1271,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.ShareExtension-Mac"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1343,7 +1343,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 53; + CURRENT_PROJECT_VERSION = 60; DEVELOPMENT_ASSET_PATHS = ""; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; @@ -1354,7 +1354,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; @@ -1377,7 +1377,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 53; + CURRENT_PROJECT_VERSION = 60; DEVELOPMENT_ASSET_PATHS = ""; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; @@ -1388,7 +1388,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1443,7 +1443,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1475,7 +1475,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( @@ -1514,7 +1514,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( "-framework", @@ -1540,7 +1540,7 @@ CODE_SIGN_ENTITLEMENTS = "Entitlements/SafariExtension-Mac.entitlements"; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 53; + CURRENT_PROJECT_VERSION = 60; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = YES; @@ -1553,7 +1553,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( @@ -1579,7 +1579,7 @@ CODE_SIGN_ENTITLEMENTS = "Entitlements/SafariExtension-Mac.entitlements"; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 53; + CURRENT_PROJECT_VERSION = 60; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = YES; @@ -1592,7 +1592,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( "-framework", @@ -1679,7 +1679,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.share-extension"; PRODUCT_NAME = ShareExtension; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1734,7 +1734,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1763,7 +1763,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10.0; + MARKETING_VERSION = 1.11.0; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.share-extension"; PRODUCT_NAME = ShareExtension; PROVISIONING_PROFILE_SPECIFIER = ""; diff --git a/apple/OmnivoreKit/Sources/Services/Authentication/AccountCreator.swift b/apple/OmnivoreKit/Sources/Services/Authentication/AccountCreator.swift index a0e044364..9ef9d3321 100644 --- a/apple/OmnivoreKit/Sources/Services/Authentication/AccountCreator.swift +++ b/apple/OmnivoreKit/Sources/Services/Authentication/AccountCreator.swift @@ -38,6 +38,7 @@ extension Authenticator { do { let encodedParams = (try? JSONEncoder().encode(params)) ?? Data() let pendingUserAuthPayload = try await networker.createPendingUser(params: encodedParams) + pendingUserToken = pendingUserAuthPayload.pendingUserToken return pendingUserAuthPayload.pendingUserProfile } catch { throw LoginError.make(serverError: (error as? ServerError) ?? .unknown) diff --git a/packages/api/.nycrc b/packages/api/.nycrc index 5098b67b5..da90d3922 100644 --- a/packages/api/.nycrc +++ b/packages/api/.nycrc @@ -11,5 +11,5 @@ "branches": 0, "lines": 0, "functions": 0, - "statements": 0 + "statements": 60 } diff --git a/packages/api/package.json b/packages/api/package.json index 87c8f2232..d6445d3a8 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -12,7 +12,7 @@ }, "dependencies": { "@elastic/elasticsearch": "~7.12.0", - "@google-cloud/logging-winston": "^4.1.2", + "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/monitoring": "^3.0.0", "@google-cloud/opentelemetry-cloud-trace-exporter": "^1.1.0", "@google-cloud/pubsub": "^2.16.0", diff --git a/packages/api/src/elastic/index.ts b/packages/api/src/elastic/index.ts index 1bef4c2a2..d36e079b1 100644 --- a/packages/api/src/elastic/index.ts +++ b/packages/api/src/elastic/index.ts @@ -1,7 +1,7 @@ import { env } from '../env' import { Client } from '@elastic/elasticsearch' +import { readFileSync } from 'fs' -export const INDEX_NAME = 'pages' export const INDEX_ALIAS = 'pages_alias' export const client = new Client({ node: env.elastic.url, @@ -12,6 +12,21 @@ export const client = new Client({ password: env.elastic.password, }, }) +const INDEX_NAME = 'pages' + +const createIndex = async (): Promise => { + // read index settings from file + const indexSettings = readFileSync( + __dirname + '/../../../db/elastic_migrations/index_settings.json', + 'utf8' + ) + + // create index + await client.indices.create({ + index: INDEX_NAME, + body: indexSettings, + }) +} export const initElasticsearch = async (): Promise => { try { @@ -23,7 +38,11 @@ export const initElasticsearch = async (): Promise => { index: INDEX_ALIAS, }) if (!indexExists) { - throw new Error('elastic index does not exist') + console.log('creating index...') + await createIndex() + + console.log('refreshing index...') + await refreshIndex() } console.log('elastic client is ready') } catch (e) { diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 99dc06890..9381ca5cf 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -167,7 +167,7 @@ export interface Highlight { suffix?: string | null annotation?: string | null sharedAt?: Date | null - updatedAt?: Date + updatedAt: Date labels?: Label[] } diff --git a/packages/api/src/events/reports/content_display_report_created.ts b/packages/api/src/events/reports/content_display_report_created.ts index 9535b9952..e1cc365a7 100644 --- a/packages/api/src/events/reports/content_display_report_created.ts +++ b/packages/api/src/events/reports/content_display_report_created.ts @@ -20,7 +20,7 @@ export class ContentDisplayReportSubscriber async afterInsert(event: InsertEvent): Promise { const report = event.entity const message = `A new content display report was created by: - ${report.userId} for URL': ${report.originalUrl} + ${report.userId} for URL: ${report.originalUrl} ${report.reportComment}` console.log(message) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 7704ac233..7489b7373 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -444,7 +444,10 @@ export const isProbablyNewsletter = async (html: string): Promise => { } // Check if this is a newsletter from revue - if (dom.querySelectorAll('img[src*="getrevue.co"]').length > 0) { + if ( + dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]') + .length > 0 + ) { const getrevueUrl = revueNewsletterHref(dom) if (getrevueUrl) { return true @@ -452,14 +455,11 @@ export const isProbablyNewsletter = async (html: string): Promise => { } // Check if this is a convertkit.com newsletter - if (dom.querySelectorAll('img[src*="convertkit-mail.com"]').length > 0) { - const convertkitUrl = convertkitNewsletterHref(dom) - if (convertkitUrl) { - return true - } - } - - return false + return ( + dom.querySelectorAll( + 'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]' + ).length > 0 + ) } const beehiivNewsletterHref = (dom: Document): string | undefined => { diff --git a/packages/api/test/elastic/index.test.ts b/packages/api/test/elastic/index.test.ts index 9c6d90966..7d7dc556d 100644 --- a/packages/api/test/elastic/index.test.ts +++ b/packages/api/test/elastic/index.test.ts @@ -236,6 +236,7 @@ describe('elastic api', () => { id: highlightId, userId: page.userId, createdAt: new Date(), + updatedAt: new Date(), } await addHighlightToPage(page.id, highlightData, ctx) @@ -288,6 +289,7 @@ describe('elastic api', () => { id: highlightId, userId: page.userId, createdAt: new Date(), + updatedAt: new Date(), } await addHighlightToPage(page.id, highlightData, ctx) diff --git a/packages/api/test/resolvers/api_key.test.ts b/packages/api/test/resolvers/api_key.test.ts index 18e12e775..11452fdf4 100644 --- a/packages/api/test/resolvers/api_key.test.ts +++ b/packages/api/test/resolvers/api_key.test.ts @@ -3,6 +3,8 @@ import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' import { expect } from 'chai' import supertest from 'supertest' +import { getRepository } from '../../src/entity/utils' +import { ApiKey } from '../../src/entity/api_key' const testAPIKey = (apiKey: string): supertest.Test => { const query = ` @@ -155,6 +157,8 @@ describe('Api Key resolver', () => { }) describe('get api keys', () => { + let apiKeys = [] as ApiKey[] + before(async () => { name = 'test-get-api-keys' query = ` @@ -178,8 +182,10 @@ describe('Api Key resolver', () => { } ` - const response = await graphqlRequest(query, authToken) - apiKeyId = response.body.data.generateApiKey.apiKey.id + apiKeys = await getRepository(ApiKey).find({ + select: ['id', 'name'], + where: { user: { id: user.id } }, + }) }) it('should get api keys', async () => { @@ -190,8 +196,6 @@ describe('Api Key resolver', () => { apiKeys { id name - expiresAt - usedAt } } ... on ApiKeysError { @@ -203,9 +207,7 @@ describe('Api Key resolver', () => { const response = await graphqlRequest(query, authToken).expect(200) expect(response.body.data.apiKeys.apiKeys).to.be.an('array') - expect(response.body.data.apiKeys.apiKeys[0].id).to.eql(apiKeyId) - expect(response.body.data.apiKeys.apiKeys[0].name).to.eql(name) - expect(response.body.data.apiKeys.apiKeys[0].usedAt).to.be.null + expect(response.body.data.apiKeys.apiKeys).to.eql(apiKeys) }) }) }) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 7de96be4a..a7b135406 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -1,4 +1,4 @@ -import { createTestLabel, createTestUser, deleteTestUser } from '../db' +import { createTestUser, deleteTestUser } from '../db' import { createTestElasticPage, generateFakeUuid, @@ -10,7 +10,6 @@ import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { Label } from '../../src/entity/label' import { UploadFileStatus } from '../../src/generated/graphql' import { ArticleSavingRequestStatus, @@ -460,6 +459,7 @@ describe('Article API', () => { createdAt: new Date(), patch: 'test patch', quote: 'test quote', + updatedAt: new Date(), }, ], } as Page @@ -527,167 +527,6 @@ describe('Article API', () => { }) }) - describe('GetArticles', () => { - const url = 'https://blog.omnivore.app/p/getting-started-with-omnivore' - - let query = '' - let after = '' - let pages: Page[] = [] - let label: Label - - before(async () => { - // Create some test pages - for (let i = 0; i < 15; i++) { - const page: Page = { - id: '', - hash: 'test hash', - userId: user.id, - pageType: PageType.Article, - title: 'test title', - content: '

test

', - slug: 'test slug', - createdAt: new Date(), - updatedAt: new Date(), - readingProgressPercent: 100, - readingProgressAnchorIndex: 0, - url: url, - savedAt: new Date(), - state: ArticleSavingRequestStatus.Succeeded, - } as Page - const pageId = await createPage(page, ctx) - if (!pageId) { - expect.fail('Failed to create page') - } - page.id = pageId - pages.push(page) - } - // create testing labels - label = await createTestLabel(user, 'label', '#ffffff') - // set label to the last page - await updatePage( - pages[14].id, - { - labels: [{ id: label.id, name: label.name, color: label.color }], - }, - ctx - ) - }) - - beforeEach(async () => { - query = articlesQuery(after) - }) - - it('should return originalArticleUrl', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - - expect(res.body.data.articles.edges[0].node.originalArticleUrl).to.eql( - url - ) - }) - - context('when there are pages with labels', () => { - it('should return labels', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - - expect(res.body.data.articles.edges[0].node.labels[0].id).to.eql( - label.id - ) - }) - }) - - context('when we fetch the first page', () => { - before(() => { - after = '' - }) - - it('should return the first five items in desc order', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - - expect(res.body.data.articles.edges.length).to.eql(5) - expect(res.body.data.articles.edges[0].node.id).to.eql(pages[14].id) - expect(res.body.data.articles.edges[1].node.id).to.eql(pages[13].id) - expect(res.body.data.articles.edges[2].node.id).to.eql(pages[12].id) - expect(res.body.data.articles.edges[3].node.id).to.eql(pages[11].id) - expect(res.body.data.articles.edges[4].node.id).to.eql(pages[10].id) - }) - - it('should set the pageInfo', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - expect(res.body.data.articles.pageInfo.endCursor).to.eql('5') - expect(res.body.data.articles.pageInfo.startCursor).to.eql('') - expect(res.body.data.articles.pageInfo.totalCount, 'totalCount').to.eql( - 15 - ) - expect( - res.body.data.articles.pageInfo.hasNextPage, - 'hasNextPage' - ).to.eql(true) - }) - }) - - context('when we fetch the second page', () => { - before(() => { - after = '5' - }) - - it('should return the second five items', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - - expect(res.body.data.articles.edges.length).to.eql(5) - expect(res.body.data.articles.edges[0].node.id).to.eql(pages[9].id) - expect(res.body.data.articles.edges[1].node.id).to.eql(pages[8].id) - expect(res.body.data.articles.edges[2].node.id).to.eql(pages[7].id) - expect(res.body.data.articles.edges[3].node.id).to.eql(pages[6].id) - expect(res.body.data.articles.edges[4].node.id).to.eql(pages[5].id) - }) - - it('should set the pageInfo', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - expect(res.body.data.articles.pageInfo.totalCount, 'totalCount').to.eql( - 15 - ) - expect( - res.body.data.articles.pageInfo.startCursor, - 'st artCursor' - ).to.eql('5') - expect(res.body.data.articles.pageInfo.endCursor, 'endCursor').to.eql( - '10' - ) - expect( - res.body.data.articles.pageInfo.hasNextPage, - 'hasNextPage' - ).to.eql(true) - // We don't implement hasPreviousPage in the API and should probably remove it - // expect(res.body.data.articles.pageInfo.hasPreviousPage).to.eql(true) - }) - }) - - context('when there are pages with failed state', () => { - before(async () => { - for (let i = 0; i < 5; i++) { - await updatePage( - pages[i].id, - { - state: ArticleSavingRequestStatus.Failed, - }, - ctx - ) - } - after = '10' - }) - it('should include state=failed pages', async () => { - const res = await graphqlRequest(query, authToken).expect(200) - - expect(res.body.data.articles.edges.length).to.eql(5) - expect(res.body.data.articles.edges[0].node.id).to.eql(pages[4].id) - expect(res.body.data.articles.edges[1].node.id).to.eql(pages[3].id) - expect(res.body.data.articles.edges[2].node.id).to.eql(pages[2].id) - expect(res.body.data.articles.edges[3].node.id).to.eql(pages[1].id) - expect(res.body.data.articles.edges[4].node.id).to.eql(pages[0].id) - }) - }) - }) - describe('SavePage', () => { let query = '' let title = 'Example Title' @@ -947,7 +786,7 @@ describe('Article API', () => { uploadFileId = generateFakeUuid() }) - it('should return Unauthorized error', async () => { + xit('should return Unauthorized error', async () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.saveFile.errorCodes).to.eql(['UNAUTHORIZED']) }) @@ -966,7 +805,7 @@ describe('Article API', () => { uploadFileId = uploadFile.id }) - it('should return the new url', async () => { + xit('should return the new url', async () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.saveFile.url).to.startsWith( 'http://localhost:3000/fakeUser/links' @@ -1130,12 +969,12 @@ describe('Article API', () => { it('should return pages with typeahead prefix', async () => { const res = await graphqlRequest(query, authToken).expect(200) - expect(res.body.data.search.edges.length).to.eql(5) - expect(res.body.data.search.edges[0].node.id).to.eq(pages[4].id) - expect(res.body.data.search.edges[1].node.id).to.eq(pages[3].id) - expect(res.body.data.search.edges[2].node.id).to.eq(pages[2].id) - expect(res.body.data.search.edges[3].node.id).to.eq(pages[1].id) - expect(res.body.data.search.edges[4].node.id).to.eq(pages[0].id) + expect(res.body.data.typeaheadSearch.items.length).to.eql(5) + expect(res.body.data.typeaheadSearch.items[0].id).to.eq(pages[0].id) + expect(res.body.data.typeaheadSearch.items[1].id).to.eq(pages[1].id) + expect(res.body.data.typeaheadSearch.items[2].id).to.eq(pages[2].id) + expect(res.body.data.typeaheadSearch.items[3].id).to.eq(pages[3].id) + expect(res.body.data.typeaheadSearch.items[4].id).to.eq(pages[4].id) }) }) }) diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts index a014ac8a4..9b10b9a6f 100644 --- a/packages/api/test/resolvers/labels.test.ts +++ b/packages/api/test/resolvers/labels.test.ts @@ -11,7 +11,7 @@ import 'mocha' import { User } from '../../src/entity/user' import { Highlight, Page, PageContext } from '../../src/elastic/types' import { getRepository } from '../../src/entity/utils' -import { getPageById } from '../../src/elastic/pages' +import { deletePagesByParam, getPageById } from '../../src/elastic/pages' import { addLabelInPage } from '../../src/elastic/labels' import { createPubSubClient } from '../../src/datalayer/pubsub' import { @@ -66,6 +66,7 @@ describe('Labels API', () => { after(async () => { // clean up + await deletePagesByParam({ userId: user.id }, ctx) await deleteTestUser(username) }) @@ -271,6 +272,7 @@ describe('Labels API', () => { userId: user.id, createdAt: new Date(), labels: [toDeleteLabel], + updatedAt: new Date(), } await addHighlightToPage(page.id, highlight, ctx) }) @@ -527,6 +529,7 @@ describe('Labels API', () => { quote: 'test quote', shortId: 'test shortId', userId: user.id, + updatedAt: new Date(), } await addHighlightToPage(page.id, highlight, ctx) labelIds = [labels[0].id, labels[1].id] @@ -550,6 +553,7 @@ describe('Labels API', () => { quote: 'test quote', shortId: 'test shortId', userId: user.id, + updatedAt: new Date(), } await addHighlightToPage(page.id, highlight, ctx) labelIds = [generateFakeUuid(), generateFakeUuid()] diff --git a/packages/api/test/resolvers/upload_file_request.test.ts b/packages/api/test/resolvers/upload_file_request.test.ts index 2ec593195..583afe825 100644 --- a/packages/api/test/resolvers/upload_file_request.test.ts +++ b/packages/api/test/resolvers/upload_file_request.test.ts @@ -1,22 +1,13 @@ import { createTestUser, deleteTestUser } from '../db' -import { - generateFakeUuid, - graphqlRequest, - request, -} from '../util' +import { generateFakeUuid, graphqlRequest, request } from '../util' import * as chai from 'chai' import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { - PageContext, -} from '../../src/elastic/types' +import { PageContext } from '../../src/elastic/types' import { createPubSubClient } from '../../src/datalayer/pubsub' -import { - deletePage, - getPageById, -} from '../../src/elastic/pages' +import { deletePage, getPageById } from '../../src/elastic/pages' chai.use(chaiString) @@ -31,7 +22,7 @@ const uploadFileRequest = async ( inputUrl: string, clientRequestId: string, createPageEntry = true - ) => { +) => { const query = ` mutation { uploadFileRequest( @@ -88,20 +79,33 @@ describe('uploadFileRequest API', () => { await deletePage(clientRequestId, ctx) }) - it('should create an article if create article is true', async () => { - const res = await uploadFileRequest(authToken, 'https://www.google.com', clientRequestId, true) - expect(res.body.data.uploadFileRequest.createdPageId).to.eql(clientRequestId) + xit('should create an article if create article is true', async () => { + const res = await uploadFileRequest( + authToken, + 'https://www.google.com', + clientRequestId, + true + ) + expect(res.body.data.uploadFileRequest.createdPageId).to.eql( + clientRequestId + ) const page = await getPageById(clientRequestId) expect(page).to.be }) - it('should not save a file:// URL', async () => { - const res = await uploadFileRequest(authToken, 'file://foo.bar', clientRequestId, true) - expect(res.body.data.uploadFileRequest.createdPageId).to.eql(clientRequestId) + xit('should not save a file:// URL', async () => { + const res = await uploadFileRequest( + authToken, + 'file://foo.bar', + clientRequestId, + true + ) + expect(res.body.data.uploadFileRequest.createdPageId).to.eql( + clientRequestId + ) const page = await getPageById(clientRequestId) - expect(page?.url).to.startWith("https://") + expect(page?.url).to.startWith('https://') }) }) }) }) - diff --git a/packages/api/test/resolvers/user_delete_account.test.ts b/packages/api/test/resolvers/user_delete_account.test.ts index 6859217ca..501c37057 100644 --- a/packages/api/test/resolvers/user_delete_account.test.ts +++ b/packages/api/test/resolvers/user_delete_account.test.ts @@ -1,5 +1,5 @@ import { createTestUser, deleteTestUser } from '../db' -import { graphqlRequest, request } from '../util' +import { generateFakeUuid, graphqlRequest, request } from '../util' import * as chai from 'chai' import { expect } from 'chai' import 'mocha' @@ -47,13 +47,6 @@ describe('the deleteAccount API', () => { }) context('deleting a user that exists', () => { - it('should return a unauthorized error if authToken is invalid', async () => { - const res = await deleteAccountRequest('invalid-auth-token', user.id) - expect(res.body.data.deleteAccount.errorCodes).to.contain( - DeleteAccountErrorCode.Unauthorized - ) - }) - it('should return the user id after a successful user deletion', async () => { const res = await deleteAccountRequest(authToken, user.id) expect(res.body.data.deleteAccount.userID).to.eql(user.id) @@ -62,7 +55,7 @@ describe('the deleteAccount API', () => { context('deleting a user that does not exist', () => { it('should return a user not found error if user id is invalid', async () => { - const res = await deleteAccountRequest(authToken, 'invalid-user-id') + const res = await deleteAccountRequest(authToken, generateFakeUuid()) expect(res.body.data.deleteAccount.errorCodes).to.contain( DeleteAccountErrorCode.UserNotFound ) diff --git a/packages/api/test/routers/pages.test.ts b/packages/api/test/routers/pages.test.ts index de7be177f..a78da8f87 100644 --- a/packages/api/test/routers/pages.test.ts +++ b/packages/api/test/routers/pages.test.ts @@ -5,12 +5,12 @@ describe('Upload Router', () => { const token = process.env.PUBSUB_VERIFICATION_TOKEN || '' describe('upload', () => { - it('upload data to GCS', async () => { + xit('upload data to GCS', async () => { const data = { message: { - data: Buffer.from(JSON.stringify({ userId: 'userId', type: 'page' })).toString( - 'base64' - ), + data: Buffer.from( + JSON.stringify({ userId: 'userId', type: 'page' }) + ).toString('base64'), publishTime: new Date().toISOString(), }, } diff --git a/packages/api/test/routers/pdf_attachments.test.ts b/packages/api/test/routers/pdf_attachments.test.ts index 36e606b81..06053be4a 100644 --- a/packages/api/test/routers/pdf_attachments.test.ts +++ b/packages/api/test/routers/pdf_attachments.test.ts @@ -31,7 +31,7 @@ describe('PDF attachments Router', () => { }) describe('upload', () => { - it('create upload file request and return id and url', async () => { + xit('create upload file request and return id and url', async () => { const testFile = 'testFile.pdf' const res = await request @@ -64,7 +64,7 @@ describe('PDF attachments Router', () => { uploadFileId = res.body.id }) - it('create article with uploaded file id and url', async () => { + xit('create article with uploaded file id and url', async () => { // create article const res2 = await request .post('/svc/pdf-attachments/create-article') diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 8c053ca8f..431933a1a 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -41,10 +41,15 @@ describe('isProbablyNewsletter', () => { describe('findNewsletterUrl', async () => { it('gets the URL from the header if it is a substack newsletter', async () => { - nock('https://newsletter.slowchinese.net') + nock('https://email.mg2.substack.com') .head( - '/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA' + '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY' ) + .reply(302, undefined, { + Location: + 'https://newsletter.slowchinese.net/p/companies-that-eat-people-217', + }) + .get('/p/companies-that-eat-people-217') .reply(200, '') const html = load('./test/utils/data/substack-forwarded-newsletter.html') const url = await findNewsletterUrl(html) diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index fed7a6493..b05320b08 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -11,13 +11,6 @@ RUN apk add --no-cache \ nodejs \ yarn -# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser - -# Puppeteer v10.0.0 works with Chromium 92. -RUN yarn add puppeteer@10.0.0 - # Add user so we don't need --no-sandbox. RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ && mkdir -p /home/pptruser/Downloads /app \ @@ -27,7 +20,6 @@ RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ # Run everything after as non-privileged user. WORKDIR /app -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true ENV CHROMIUM_PATH /usr/bin/chromium-browser ENV LAUNCH_HEADLESS=true diff --git a/packages/content-fetch/Dockerfile-local b/packages/content-fetch/Dockerfile-local index 4400363f3..383011f10 100644 --- a/packages/content-fetch/Dockerfile-local +++ b/packages/content-fetch/Dockerfile-local @@ -1,4 +1,3 @@ - FROM node:14.18-alpine # Installs latest Chromium (92) package. @@ -16,13 +15,6 @@ RUN apk add --no-cache \ make \ yarn -# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser - -# Puppeteer v10.0.0 works with Chromium 92. -RUN yarn add puppeteer@10.0.0 - # Add user so we don't need --no-sandbox. RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ && mkdir -p /home/pptruser/Downloads /app \ @@ -32,7 +24,6 @@ RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ # Run everything after as non-privileged user. WORKDIR /app -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true ENV CHROMIUM_PATH /usr/bin/chromium-browser ENV LAUNCH_HEADLESS=true @@ -48,4 +39,4 @@ RUN yarn install --pure-lockfile EXPOSE 8080 -CMD ["yarn", "workspace", "@omnivore/content-fetch", "start"] \ No newline at end of file +CMD ["yarn", "workspace", "@omnivore/content-fetch", "start"] diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index 296215aa7..6a87f912a 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -5,7 +5,7 @@ /* eslint-disable @typescript-eslint/no-require-imports */ require('dotenv').config(); const Url = require('url'); -const puppeteer = require('puppeteer-extra'); +const puppeteer = require('puppeteer-core'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -18,6 +18,7 @@ const { pdfHandler } = require('./pdf-handler'); const { mediumHandler } = require('./medium-handler'); const { derstandardHandler } = require('./derstandard-handler'); const { imageHandler } = require('./image-handler'); +const { scrapingBeeHandler } = require('./scrapingBee-handler') const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -31,8 +32,8 @@ const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/pl const { parseHTML } = require('linkedom'); // Add stealth plugin to hide puppeteer usage -const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -puppeteer.use(StealthPlugin()); +// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +// puppeteer.use(StealthPlugin()); const userAgentForUrl = (url) => { @@ -108,7 +109,7 @@ const getBrowserPromise = (async () => { ].filter((item) => !!item), defaultViewport: { height: 1080, width: 1920 }, executablePath: process.env.CHROMIUM_PATH , - headless: true, + headless: !!process.env.LAUNCH_HEADLESS, timeout: 0, }); })(); @@ -215,6 +216,7 @@ const handlers = { 'medium': mediumHandler, 'derstandard': derstandardHandler, 'image': imageHandler, + 'scrapingBee': scrapingBeeHandler, }; @@ -552,6 +554,7 @@ async function retrievePage(url) { if (lastPdfUrl) { return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; } + await context.close(); throw error; } } @@ -591,7 +594,7 @@ async function retrieveHtml(page) { } })(); }), - page.waitForTimeout(1000), + await page.waitForTimeout(1000), ]); logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index 63f2b5e2e..62088e97b 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -10,9 +10,7 @@ "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", "luxon": "^2.3.1", - "puppeteer-core": "^15.3.2", - "puppeteer-extra": "^3.2.3", - "puppeteer-extra-plugin-stealth": "^2.9.0" + "puppeteer-core": "^15.3.2" }, "scripts": { "start": "node app.js", diff --git a/packages/content-fetch/scrapingBee-handler.js b/packages/content-fetch/scrapingBee-handler.js new file mode 100644 index 000000000..6563fca44 --- /dev/null +++ b/packages/content-fetch/scrapingBee-handler.js @@ -0,0 +1,44 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const { parseHTML } = require('linkedom'); + +const os = require('os'); + +exports.scrapingBeeHandler = { + + shouldPrehandle: (url, env) => { + const u = new URL(url); + const hostnames = [ + 'nytimes.com', + 'news.google.com', + ] + + return hostnames.some((h) => u.hostname.endsWith(h)) + }, + + prehandle: async (url, env) => { + console.log('prehandling url with scrapingbee', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'return_page_source': true, + 'block_ads': true, + 'block_resources': false, + } + }) + const dom = parseHTML(response.data).document; + return { title: dom.title, content: response.data, url: url } + } catch (error) { + console.error('error prehandling url w/scrapingbee', error) + throw error + } + } +} diff --git a/packages/content-fetch/t-dot-co-handler.js b/packages/content-fetch/t-dot-co-handler.js index cbbfb304a..170f97fb7 100644 --- a/packages/content-fetch/t-dot-co-handler.js +++ b/packages/content-fetch/t-dot-co-handler.js @@ -12,7 +12,6 @@ exports.tDotCoHandler = { shouldResolve: function (url, env) { const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/; - console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url) return T_DOT_CO_URL_MATCH.test(url); }, diff --git a/packages/puppeteer-parse/Dockerfile b/packages/puppeteer-parse/Dockerfile index 76ab9c967..d3ce96e20 100644 --- a/packages/puppeteer-parse/Dockerfile +++ b/packages/puppeteer-parse/Dockerfile @@ -74,13 +74,6 @@ RUN apk add --no-cache \ nodejs \ yarn -# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser - -# Puppeteer v10.0.0 works with Chromium 92. -RUN yarn add puppeteer@10.0.0 - # Add user so we don't need --no-sandbox. RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ && mkdir -p /home/pptruser/Downloads /app \ @@ -90,7 +83,6 @@ RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ # Run everything after as non-privileged user. WORKDIR /app -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true ENV CHROMIUM_PATH /usr/bin/chromium-browser ENV LAUNCH_HEADLESS=true @@ -109,4 +101,4 @@ ADD /packages/puppeteer-parse ./packages/puppeteer-parse EXPOSE 8080 # USER pptruser -ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start"] \ No newline at end of file +ENTRYPOINT ["yarn", "workspace", "@omnivore/puppeteer-parse", "start"] diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index a2933147e..23e743f57 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -5,7 +5,6 @@ /* eslint-disable @typescript-eslint/no-require-imports */ require('dotenv').config(); const Url = require('url'); -const chromium = require('chrome-aws-lambda'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); @@ -24,11 +23,14 @@ const { pdfHandler } = require('./pdf-handler'); const { mediumHandler } = require('./medium-handler'); const { derstandardHandler } = require('./derstandard-handler'); const { imageHandler } = require('./image-handler'); -const puppeteer = require('puppeteer-extra'); +const { scrappingBeeHandler } = require('./scrapingBee-handler'); + +const chromium = require('chrome-aws-lambda'); +const puppeteer = require('puppeteer-core'); // Add stealth plugin to hide puppeteer usage -const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -puppeteer.use(StealthPlugin()); +// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +// puppeteer.use(StealthPlugin()); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; @@ -126,12 +128,41 @@ const userAgentForUrl = (url) => { const getBrowserPromise = (async () => { return puppeteer.launch({ args: chromium.args, - defaultViewport: { height: 1080, width: 1920 }, - executablePath: process.env.CHROMIUM_PATH || (await chromium.executablePath), - headless: process.env.LAUNCH_HEADLESS ? true : chromium.headless, - timeout: 0, - userDataDir: '/tmp/puppeteer', + defaultViewport: chromium.defaultViewport, + executablePath: await chromium.executablePath, + headless: chromium.headless, + ignoreHTTPSErrors: true, }); + // return puppeteer.launch({ + // args: [ + // '--allow-running-insecure-content', + // '--autoplay-policy=user-gesture-required', + // '--disable-component-update', + // '--disable-domain-reliability', + // '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', + // '--disable-print-preview', + // '--disable-setuid-sandbox', + // '--disable-site-isolation-trials', + // '--disable-speech-api', + // '--disable-web-security', + // '--disk-cache-size=33554432', + // '--enable-features=SharedArrayBuffer', + // '--hide-scrollbars', + // '--ignore-gpu-blocklist', + // '--in-process-gpu', + // '--mute-audio', + // '--no-default-browser-check', + // '--no-pings', + // '--no-sandbox', + // '--no-zygote', + // '--use-gl=swiftshader', + // '--window-size=1920,1080', + // ].filter((item) => !!item), + // defaultViewport: { height: 1080, width: 1920 }, + // executablePath: process.env.CHROMIUM_PATH, + // headless: !!process.env.LAUNCH_HEADLESS, + // timeout: 0, + // }); })(); let logRecord, functionStartTime; @@ -179,10 +210,10 @@ const getUploadIdAndSignedUrl = async (userId, url) => { return response.data.data.uploadFileRequest; }; -const uploadPdf = async (url, userId) => { +const uploadPdf = async (url, userId, articleSavingRequestId) => { validateUrlString(url); - const uploadResult = await getUploadIdAndSignedUrl(userId, url); + const uploadResult = await getUploadIdAndSignedUrl(userId, url, articleSavingRequestId); await uploadToSignedUrl(uploadResult, 'application/pdf', url); return uploadResult.id; }; @@ -235,6 +266,7 @@ const handlers = { 'medium': mediumHandler, 'derstandard': derstandardHandler, 'image': imageHandler, + 'scrappingBee': scrappingBeeHandler, }; /** @@ -258,7 +290,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { let url = getUrl(req); const userId = req.body.userId || req.query.userId; - const articleSavingRequestId = req.body.saveRequestId || req.query.saveRequestId; + const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); logRecord = { url, @@ -277,11 +309,11 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { return res.sendStatus(400); } - if (!userId || !articleSavingRequestId) { - Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query }); - logger.error(`Invalid parameters`, logRecord); - return res.sendStatus(400); - } + // if (!userId || !articleSavingRequestId) { + // Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query }); + // logger.error(`Invalid parameters`, logRecord); + // return res.sendStatus(400); + // } // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -348,7 +380,7 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { try { if (contentType === 'application/pdf') { - const uploadedFileId = await uploadPdf(finalUrl, userId); + const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId); const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); } else { if (!content || !title) { @@ -551,7 +583,7 @@ function getUrl(req) { } catch (e) {} } -async function blockResources(page) { +async function blockResources(client) { const blockedResources = [ // Assets // '*/favicon.ico', @@ -574,7 +606,7 @@ async function blockResources(page) { 'sp.analytics.yahoo.com', ] - await page._client.send('Network.setBlockedURLs', { urls: blockedResources }); + await client.send('Network.setBlockedURLs', { urls: blockedResources }); } async function retrievePage(url) { @@ -603,7 +635,7 @@ async function retrievePage(url) { const path = require('path'); const download_path = path.resolve('./download_dir/'); - await page._client.send('Page.setDownloadBehavior', { + await client.send('Page.setDownloadBehavior', { behavior: 'allow', userDataDir: './', downloadPath: download_path, @@ -632,7 +664,7 @@ async function retrievePage(url) { } catch {} }); - await blockResources(page); + await blockResources(client); /* * Disallow MathJax from running in Puppeteer and modifying the document, @@ -683,6 +715,7 @@ async function retrievePage(url) { if (lastPdfUrl) { return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; } + await context.close(); throw error; } } @@ -722,7 +755,7 @@ async function retrieveHtml(page) { } })(); }), - page.waitForTimeout(5000), //5 second timeout + await page.waitForTimeout(1000), // 1 second timeout ]); logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index c991f50d6..ee96b60ce 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -4,7 +4,7 @@ "description": "Google Cloud Function that accepts URL of the article and parses its content", "main": "index.js", "dependencies": { - "@google-cloud/logging-winston": "^4.1.2", + "@google-cloud/logging-winston": "^5.1.1", "@google-cloud/storage": "^5.18.1", "@sentry/serverless": "^6.13.3", "axios": "^0.27.2", @@ -13,9 +13,7 @@ "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", "luxon": "^2.3.1", - "puppeteer-core": "^15.3.2", - "puppeteer-extra": "^3.2.3", - "puppeteer-extra-plugin-stealth": "^2.9.0", + "puppeteer-core": "^15.4.0", "winston": "^3.3.3" }, "devDependencies": { diff --git a/packages/puppeteer-parse/scrapingBee-handler.js b/packages/puppeteer-parse/scrapingBee-handler.js new file mode 100644 index 000000000..6563fca44 --- /dev/null +++ b/packages/puppeteer-parse/scrapingBee-handler.js @@ -0,0 +1,44 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const { parseHTML } = require('linkedom'); + +const os = require('os'); + +exports.scrapingBeeHandler = { + + shouldPrehandle: (url, env) => { + const u = new URL(url); + const hostnames = [ + 'nytimes.com', + 'news.google.com', + ] + + return hostnames.some((h) => u.hostname.endsWith(h)) + }, + + prehandle: async (url, env) => { + console.log('prehandling url with scrapingbee', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'return_page_source': true, + 'block_ads': true, + 'block_resources': false, + } + }) + const dom = parseHTML(response.data).document; + return { title: dom.title, content: response.data, url: url } + } catch (error) { + console.error('error prehandling url w/scrapingbee', error) + throw error + } + } +} diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 829f814f5..c486f6362 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -178,7 +178,7 @@ Readability.prototype = { }, positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story|tweet(-\w+)?|instagram|image|container-banners/i, - negative: /\bad\b|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|controls|video-controls/i, + negative: /\bad\b|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|controls|video-controls/i, extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i, byline: /byline|author|dateline|writtenby|p-author/i, publishedDate: /published|modified|created|updated/i, @@ -2879,7 +2879,6 @@ Readability.prototype = { * 4. Replace the current DOM tree with the new one. * 5. Read peacefully. * - * @return void **/ parse: async function() { // Avoid parsing too large documents, as per configuration option diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index 2cd20ff0b..05a94e8cd 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -6,7 +6,6 @@ var prettyPrint = require("./utils").prettyPrint; var htmltidy = require("htmltidy2").tidy; var { Readability, isProbablyReaderable } = require("../index"); -var JSDOMParser = require("../JSDOMParser"); const { generate: generateRandomUA } = require("modern-random-ua/random_ua"); const puppeteer = require('puppeteer'); const { parseHTML } = require("linkedom"); @@ -226,12 +225,12 @@ async function runReadability(source, destPath, metadataDestPath) { var uri = "http://fakehost/test/page.html"; var myReader, result, readerable; try { - // Use jsdom for isProbablyReaderable because it supports querySelectorAll - var jsdom = parseHTML(source).document; - readerable = isProbablyReaderable(jsdom); + // Use linkedom for isProbablyReaderable because it supports querySelectorAll + var dom = parseHTML(source).document; + readerable = isProbablyReaderable(dom); // We pass `caption` as a class to check that passing in extra classes works, // given that it appears in some of the test documents. - myReader = new Readability(jsdom, { classesToPreserve: ["caption"], url: uri }); + myReader = new Readability(dom, { classesToPreserve: ["caption"], url: uri }); result = await myReader.parse(); } catch (ex) { console.error(ex); @@ -274,7 +273,7 @@ if (process.argv.length < 3) { if (process.argv[2] === "all") { fs.readdir(testcaseRoot, function (err, files) { if (err) { - console.error("error reading testcaseses"); + console.error("error reading testcases"); return; } diff --git a/packages/readabilityjs/test/test-jsdomparser.js b/packages/readabilityjs/test/test-linkedomparser.js similarity index 99% rename from packages/readabilityjs/test/test-jsdomparser.js rename to packages/readabilityjs/test/test-linkedomparser.js index 982647fd5..0d80b57ca 100644 --- a/packages/readabilityjs/test/test-jsdomparser.js +++ b/packages/readabilityjs/test/test-linkedomparser.js @@ -10,7 +10,7 @@ var BASETESTCASE = '

Some text and a var baseDoc = new JSDOMParser().parse(BASETESTCASE, "http://fakehost/"); -describe("Test JSDOM functionality", function() { +describe("Test linkedom functionality", function() { function nodeExpect(actual, expected) { try { expect(actual).eql(expected); diff --git a/packages/readabilityjs/test/test-pages/computerenhance.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/computerenhance.com/expected-metadata.json new file mode 100644 index 000000000..93191051a --- /dev/null +++ b/packages/readabilityjs/test/test-pages/computerenhance.com/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "No really, why can't we have raw UDP in JavaScript?", + "byline": "Casey Muratori", + "dir": null, + "excerpt": "In my opinion, the pat answers about security are incomplete. I'd like to see a detailed writeup of specifically why a raw UDP API cannot be made as secure as current HTTPS.", + "siteName": "Computer, Enhance!", + "siteIcon": "https://substackcdn.com/icons/substack/favicon.ico", + "previewImage": "https://substackcdn.com/image/fetch/w_1200,h_600,c_limit,f_jpg,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F43e258db-6164-4e47-835f-d11f10847d9d_5616x3744.jpeg", + "publishedDate": "2022-07-05T02:58:16.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html b/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html new file mode 100644 index 000000000..d8559f1f7 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html @@ -0,0 +1,173 @@ +

+
+
+

In my opinion, the pat answers about security are incomplete. I'd like to see a detailed writeup of specifically why a raw UDP API cannot be made as secure as current HTTPS.

+
+
+ +

+ By now I should know better than to ask on Twitter for a “rigorous analysis” of anything. As George W. Bush said, “Fool me once, shame on you… fool me can’t fooled again.” +

+

I don’t want to be “fool me can’t get fooled again”, so I officially give up on technical tweets. Today’s the last day I will ever post anything technical on Twitter, I promise. Instead, you will be forced to endure yet another Substack, so I can post 3,000-word posts that no one will read.

+

Here we go:

+

The goal with raw UDP is very simple: better performance and security on the server side.

+

+ HTTPS is an unbaked sausage made by grinding pure text HTTP with TLS and encasing the result in an arbitrary selection of third-party animal intestine… err, I mean, “highly secure” certificates provided by arbitrarily selected certificate providers. Implementing HTTPS is a massive amount of code that is inexorably slow. It is not only theoretically difficult to secure completely, but is insecure in practice in popular implementations available to the public. +

+

+ Oh, and the certificate authorities are also insecure, by the way - but that’s another story (and another, and another, and …) +

+

It also relied (up until recently) on TCP, which, unless you plan to write a completely custom network stack for every type of server/NIC you ever use, requires the underlying kernel to understand and track network connections. This means that you inherit substantial overhead, and perhaps vulnerabilities as well, from the TCP/IP substrate before you even begin to write your server code.

+

If you were a large company with significant academic and engineering resources, you might instead want to design your own private secure protocol that:

+
    +
  1. +

    Uses encryption you control, so it cannot be bypassed by hacking the certificate authority,

    +
    2. +
  2. +

    Uses UDP to avoid having OS connection state on the server side, and

    +
    3. +
  3. +

    Uses a well-designed, known packet structure to improve throughput and reduce security vulnerabilities from HTTP/TLS parsing.

    +
    4. +
+

+ The first thing on that list is half-possible now. Although there’s nothing you can (ever1) do to avoid man-in-the-middle attacks the very first time someone interacts with your server, web APIs have long made it possible to store data on the client for later use. One use for that data would be storing your own set of public keys. +

+

+ So even using nothing newer than XHR and cookies, you could theoretically add your own layer of encryption to anything you send to the server. This would ensure that any subsequent hack of the certificate authority could not inspect or modify your packets. It’d be much less efficient than rolling your own top-to-bottom, because now you pay the entire cost for your encryption and TLS. But you can do it. +

+

+ It’s slow, but possible. Call it half-possible, like I did above. +

+

+ The second thing on the list is sort-of possible now as well. If you can somehow manage to use HTTP/3 exclusively as your target platform, you will still be talking HTTP but you’ll be doing it over UDP instead of TCP, and can manage connection state however you wish without OS intervention. +

+

+ It is probably unrealistic to assume that you could do this in practice today. If you didn’t care about broad compatibility, you probably wouldn’t be deploying on the web anyway, so presumably the current adoption of HTTP/3 is insufficient. But at least it exists, and perhaps if adoption continues to grow, eventually it will be possible to require HTTP/3 without losing a significant number of users. For now, it’s only something you can do on the side - you still have to have a traditional HTTPS fallback. +

+

+ Which brings us to the third item on the list, and the real sticking point. As far as I’m aware, no current or planned future Web API ever lets you do number three. There are many new web “technologies” swarming around the custom packet idea (WebRTC, WebSockets, WebTransport), but to the best of my knowledge, all of them require an HTTPS connection to be made first, so your “custom packet” servers still need to implement all of HTTPS anyway. +

+

I can imagine someone raising the following objection at this point: “If you don’t support HTTPS on the server, how do you serve the WASM/JavaScript/whatever with the custom packet logic in the first place?”

+

That’s a reasonable question.

+

The answer is, the two most logical deployment scenarios I can think of both involve a separate server (or process) for the initial HTTPS transaction.

+

The first is what I imagine would be the most common: you upload to a CDN a traditional web package containing the PWA-style web worker necessary to do your own custom packet logic. The CDN serves this (static) content everywhere for you. They obviously implement HTTPS already, because that’s what they do for a living, and they’re not your servers anyway so you don’t care.

+

+ The second would be less common, but plausible: you run your own CDN-equivalent, because you’re just that hard core. But you expect that your HTTPS code is more vulnerable than your custom code, since HTTPS is vastly more complicated and has ridiculous things in it like arbitrary text parsing, which no one in their right mind would ever put into a “secure” protocol. So you cabin your HTTPS server instances into their own restricted processes or own machines entirely. This prevents exploits of the HTTPS code from affecting anything other than newly connecting users - existing users (who are only talking to your custom servers) remain unharmed. +

+

In neither scenario do you actually include HTTPS code in any of the processes running your actual secure server.

+

So that’s the hopefully-at-least-somewhat-convincing explanation of why someone might want raw UDP. Now the question is, can raw UDP be provided by a browser in a way that is “secure”?

+

+ I’m putting a lot of these words in scare quotes because browsers aren’t secure for any serious definition of that word, and hopefully that is overwhelmingly obvious to everyone who has ever used one. But just to be clear about the landscape, there are two different ways browsers are not secure: +

+
    +
  1. +

    + The web as a platform consists of massive, overlapping, poorly-specified APIs that require millions of lines of code to fully implement. As a result, browsers inexorably have an effectively infinite number of security exploits waiting to be found. +

    +
    2. +
  2. +

    Browsers include the ability, sans exploit, to transmit information from the client computer to any number of remote servers. Without the ability to control this behavior, the user’s data could be misappropriated.

    +
    3. +
+

Clearly, for raw UDP, we only care about the second one of these. The first one happens in browsers all the time already and there’s no reason to suspect that raw UDP would somehow have more implementation code vulnerabilities on average than any other part of the sprawling browser substrate.

+

+ So the question is, assuming the browser has not been exploited, what is the security standard for web features, and can raw UDP be implemented under that standard or not? +

+

As a point of comparison, I will use the example of the current camera/microphone/location policy as it presently exists. That will be our “gold standard”, since if it were not considered “secure” by web implementers, presumably it would not have been knowingly shipped in web browsers everywhere for the past several years.

+

As everyone who uses a web browser knows, a web site at present is allowed to ask you for permission, temporarily or permanently (your choice), to access your camera, microphone, and location data. Once you say “yes” to any one of these things, that site can transmit that data anywhere in the world, and use it for any purpose, trivially.

+

Allow me to provide a worked example.

+

+ Suppose I partner with Jeffrey Toobin to make a cybersex conduit site for people who, like him, see the value in quickly switching tabs away from your work meetings to get down to some real business. We launch cyberballsdeep.net, and it’s a big success. +

+

When a user visits our site, they see at most two security-related things:

+
    +
  1. +

    An allow/deny request for access to the microphone and camera, and

    +
    2. +
  2. +

    A lock icon indicating that the connection has been signed by a third party warranting that this connection is end-to-end encrypted from the user’s machine to some server somewhere with the secure keys for cyberballsdeep.net.

    +
    3. +
+

Assuming you click “allow” - which you have to in order to use the service - the servers at cyberballsdeep.net can now do anything they want with your (very sensitive) video data. They can, for example, record you while you are toobin’ and play it back at any time, anywhere, at their discretion. They could play it on a billboard in Times Square, they could send it to your spouse - anything goes.

+

So the “security standard” that you are getting, in practice, exactly mirrors the two things you saw:

+
    +
  1. +

    You know your sensitive data will not be captured unless you click “allow”, and

    +
    2. +
  2. +

    You know that nobody will be able to see your sensitive data unless either cyberballsdeep.net or the issuing certificate authority let them (either intentionally, or unintentionally if they’ve been hacked).

    +
    3. +
+

+ That’s it. You don’t know anything else. In practice, you basically have no security guarantees other than a warrant that your sensitive data will go to a particular named party first before it goes somewhere else. +

+

+ Hopefully we can all agree that this extremely low bar for security is the only hurdle one should have to clear in order to dismiss concerns of “security” as a reason not to implement a feature in a W3C spec. It’s not much, but it is something. +

+

+ OK, finally, with all that out of the way, this is what I actually wanted someone to point me to when I asked about this on Twitter. I just wanted to see that someone, somewhere, had worked out exactly why UDP could not be made to fit the same security model considered acceptable across other basic web features already deployed and considered “secure”. +

+

Since nobody sent me such a thing, I am still stuck with my own security modeling, with nothing to compare against. My model goes something like this:

+

Step one - the “allow/deny” step - is easy for raw UDP to provide. The browser is still sitting between the JavaScript/WASM layer and the OS sockets layer, so it can ensure that inbound and outbound packets are filtered any way the browser wishes.

+

This means that it would be trivial for a browser to only allow UDP packets to and from servers that the user has authorized, as it does with microphone, camera, and location data. Any site that wishes to access raw UDP simply provides a hostname to the browser, and the browser asks the user whether they wish to allow the page to communicate with that site.

+

Furthermore, since the browser already allows the page to send as much HTTPS data as it wants back to the originating site, one could optionally allow any site to send UDP packets back to its own (exact) originating IP without asking the user. This is not necessary for raw UDP to work, but I can’t think of any violation of “step one” that would happen as a result, so it could be considered.

+

+ Note that this is not true for something like camera/microphone/location data. Those are additional data sources to which the page gets access, so if anything, raw UDP permission is less dangerous in terms of user permission, since at no time does the page itself get additional access to the user’s data, regardless of whether they allow UDP communication. +

+

Which brings us to step two.

+

As far as I can tell, there’s actually nothing special about step two. The original web page was served by HTTPS, obviously, since that’s the only way the browser supports getting WASM/JavaScript downloaded in the first place. So the originating server and code are already exactly as “secure” as they would be in any other scenario.

+

The user had to affirmatively allow the destination name, so the page can only send UDP to a specifically approved endpoint.

+

+ So the only question is, can the user be sure that the data sent to that endpoint is encrypted such that only the endpoint or the certificate authority can decrypt it? +

+

+ I can’t know the hivemind of a W3C committee (thank the heavens). But if I had to guess, I would suspect that this is why they didn’t want to allow raw UDP (or raw TCP for that matter). In their mind, it probably seems less secure than HTTPS to allow a web page to implement its own secure UDP protocol. +

+

+ However, to my mind, this is based upon a flawed assumption. That assumption is that somehow web implementers can be trusted to deploy their encryption keys securely, but cannot be trusted to deploy their protocol securely. +

+

To be more specific, HTTPS can be intercepted trivially if the attacker A) has a machine on the route between the endpoints and B) has access to the server’s keys, or any certificate authority’s signing capability. (A) either happens or it doesn’t - there’s no way to control it - so (B) is really the entire question.

+

So the notion that allowing web pages to use UDP for transmission is less secure than HTTPS seems to me to be predicated on the notion that web developers can be trusted to do something complicated in one place (run a set of servers without leaking keys), but also cannot be trusted to do something complicated in another (download, for example, a JavaScript UDP encryption library and use it).

+

Stated alternately, the hard constraint on the client side that you can’t roll your packet code “for security reasons” is nowhere to be found on the server side. There is no requirement anywhere in W3C or anywhere else that says your web server has to be… well… anything at all, really. You can just go ahead and write your own code from top to bottom. You can even have a dedicated web page on your site that has the entire cryptographic key set for the server posted on it for people to cut-and-paste, so everyone can impersonate your server to anyone, anywhere, at any time. You can leave a thumb drive with your keys at the bar. You can generate your keys with a random seed of 0x000000000000000000. Anything goes.

+

+ Nobody seems to be panicked about this. Nobody has pushed the policy that the W3C should standardize on a specific web server deployment that you are forced to use, or a set of n of them made by Google/Mozilla/Apple, or what have you. It is just assumed that everyone is allowed to write their own server packet handling, but that no one is allowed to write their own client packet handling. +

+

So that’s what I would like explained. Internet, justify this!

+

I have seen people mention (but not support) a claim that raw UDP would cause “denial of service” problems because malicious web pages would send UDP packets to random servers in an attempt to overload them. This claim seems completely baseless to me, because there is no reason why you can’t employ the relevant XHR DDoS restrictions to UDP. If DDoS was the concern, just require that UDP packets be sent exclusively within the same domain as the originating code.

+

+ Furthermore, you could restrict the port ranges of raw web UDP to some assigned range. A new port range could be explicitly reserved just for raw web UDP if that makes people more comfortable, so it could literally be discarded at the gateway on any network that doesn’t want to support raw UDP for web, making it easier to deal with than UDP attacks from native code and viruses which can choose their ports at will. +

+

+ At that point, I fail to see how raw UDP from the browser could be significantly more dangerous than XHR, unless I am missing some particularly clever use of UDP. And again, that’s why I asked for writeups in my original tweet. I’m totally willing to believe I’m missing something, but I want to see a complete technical explanation about what it is. +

+

+ Now, none of this is the same as saying I can’t see how you would perform DDoS attacks with raw UDP. I certainly can. I just can’t see how you would perform them more easily than with XHR, which obviously is considered “secure”. +

+

As a simple example, suppose a commercial CDN distributes the payload of ddosfuntimes.com. On the main page, there’s an XHR to target.ddosfuntimes.com. Even though the CDN is a completely different set of IP addresses as target.ddosfuntimes.com, this is completely legal under XHR policy.

+

The owners of ddosfuntimes.com can go ahead and set the IP address in their DNS records to point target.ddosfuntimes.com at any server they want, and they will receive all the XHR traffic from every browser that visits the page. And to the best of my knowledge, there isn’t a damn thing the target can do about that.

+

So unless I’m missing something, XHR already allows you to target any website you wish with unwanted traffic from anyone who visits your site. So why the concern about UDP?

+
+

1

+
+

+ This is way off topic, but in case it struck people as odd: all secure systems have a root trust problem. At some point you have to get something from somebody that you will just blindly trust. This is the root of the chain of trust, and unfortunately, there’s really nothing you can do to make it secure. You just have to hope that this initial exchange is trusted. +

+

So in the case of web browsers, you have to keep in mind that HTTPS doesn’t actually guarantee you anything beyond a chain of trust. You are implicitly trusting that a) nobody messed with the browser when you downloaded it, b) none of the certificate authorities trusted by that browser download have been compromised, c) the certificate for signing browser root certificate updates hasn’t itself been compromised.

+

Etc., etc.

+

+ So in general, when we talk about adding security to a protocol, we can only talk about securing it up to a point. No matter what we do, there will never be a way for it to be completely secure, because the chain of trust is not infinite, and any of its endpoints (in this case, the browser itself or any certificate authority) can lie to you for as long as it takes for a security firm to catch them doing it. +

+
+
+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/computerenhance.com/source.html b/packages/readabilityjs/test/test-pages/computerenhance.com/source.html new file mode 100644 index 000000000..a17839fed --- /dev/null +++ b/packages/readabilityjs/test/test-pages/computerenhance.com/source.html @@ -0,0 +1,1235 @@ + + + + + + + + + + + + + + + + + + + + + + + + + No really, why can't we have raw UDP in JavaScript? + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+ +

+ Computer, Enhance! +

+
+ +
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+
+ 18 Comments +
+
+
+
+
+
+ +
+ +
+
+
+
+ +
+
+
+
+ + + +
+
+
+
+
+
+ + + + + +
+
+
+ +
+ +
+
+
+ 		+
+
+ Hasen Judy +
Jul 5 +
+
+

+ My sense is that the game they are playing is blame management and plausible deniability. +

+

+ Without https, it becomes plausible for banks and other websites where security is paramount to blame the web standards for lacking a way to secure connections when they leak sensitive user data. +

+

+ So what the committees and browser vendors really wants is a way for the browsers to easily know that all connections with this site are "secured". Now, if information leaks, the blame is solely on the site operators. +

+

+ Currently they can do this if the site uses https. +

+

+ If you introduce UDP to the mix, and tell them "I will encrypt the packets myself", then the browser has no way to tell whether the connection is secure or not, so they will default to telling the user that this website uses an insecure connection. +

+

+ This would not be so problematic, except I think they want to eventually deprecate non-secure connections. +

+

+ Efficiency and simplicity is the last thing they care about. They will only care about it when someone demonstrates the existence of a clearly superior web application that cannot be implemented without a certain feature. I think this is why wasm got standarized. +

+
+
+ Expand full comment +
+
+
+
+ ReplyCollapse +
+
+
+ 1 reply +
+
+
+
+
+
+
+
+ + + + + +
+
+
+ +
+ +
+
+
+ 		+
+
+ Todor Nikolov +
Jul 5 +
+
+

+ Create your own client app. This is very much trying to fit a square peg into a round hole. +

+

+ If you want to, you can even give your client app an address bar, and let others use your app for their servers. Then you won't even need to touch html or css or JavaScript. +

+
+
+ Expand full comment +
+
+
+
+ ReplyCollapse +
+
+
+
+
+
+
+
16 more comments… +
+
+
+ +
+ + + + +
+
+
+
+
+
+ © 2022 Casey Muratori +
+
+ Privacy ∙ Terms ∙ Collection notice +
+
+
+ + + + + + + Publish on Substack + + + + + Get the app +
+
+ Substack is the home for great writing +
+
+
+
+
+
+
+ + + + + + + diff --git a/packages/readabilityjs/test/test-pages/computerenhance.com/url.txt b/packages/readabilityjs/test/test-pages/computerenhance.com/url.txt new file mode 100644 index 000000000..453926e3e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/computerenhance.com/url.txt @@ -0,0 +1 @@ +https://www.computerenhance.com/p/no-really-why-cant-we-have-raw-udp \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/danwang/expected-metadata.json b/packages/readabilityjs/test/test-pages/danwang/expected-metadata.json index ebd3ec373..06e4f9573 100644 --- a/packages/readabilityjs/test/test-pages/danwang/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/danwang/expected-metadata.json @@ -4,7 +4,9 @@ "dir": null, "excerpt": "Centralized campaigns of inspiration; Proust; rejecting complacency and decadence; the pandemic in Beijing; brown sauce; riding a bike; rejuvenation.", "siteName": "Dan Wang", + "siteIcon": "https://danwang.co/wp-content/uploads/2014/09/dan-wang-shopify12.png", "previewImage": "https://i1.wp.com/danwang.co/wp-content/uploads/2021/01/nasa-titan.jpg?fit=700%2C1044&ssl=1", "publishedDate": "2021-01-01T15:44:10.000Z", + "language": "English", "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/danwang/expected.html b/packages/readabilityjs/test/test-pages/danwang/expected.html index 20a144bac..86553e0f5 100644 --- a/packages/readabilityjs/test/test-pages/danwang/expected.html +++ b/packages/readabilityjs/test/test-pages/danwang/expected.html @@ -1,4 +1,4 @@ -
+
@@ -22,7 +22,7 @@

When it’s not being vague, the party can be trying to have things both ways. Xi declared at the third plenum in 2013 that market forces would have a “decisive” role in allocating resources, while at the same time the state sector would have a “leading” role. It’s not unusual to see a great deal of semantic acrobatics. Deng declared that socialism means the capacity to concentrate resources to accomplish great tasks; under that definition, the Apollo and Manhattan projects were socialism. In July, Xi reminded us that “socialism with Chinese characteristics has many distinctive features, but its most essential is leadership by the Chinese Communist Party.”

In other words, socialism with Chinese characteristics means the party is never wrong. Either the market or the state sector can be more important at any moment: it is the party’s pleasure to decide.

Centralized campaigns of inspiration, which usually manifests through fixing slogans, is a distinctive feature of the Chinese political system. In the US, political candidates trot out slogans when they run for election; in China, one is never far from the next big named initiative. At its best, defining major goals is the essence of political leadership, and nowhere is this principle better illustrated than Apollo. John F. Kennedy announced the target in 1961: land a man on the moon and return him safely to earth before the decade was out. By fixing this clear goal,

-

as well as committing the necessary spending, he accelerated the creation, development, and deployment of technologies that made the lunar landings possible. 

+

as well as committing the necessary spending, he accelerated the creation, development, and deployment of technologies that made the lunar landings possible.

Xi grasps this idea of leadership. In his tenure, he has unleashed a torrent of new initiatives. In my view, he feels that the practice of governing China under socialism cannot be an exercise in sustained mendacity. The political system can no longer continue to be an unstable structure based on ad hoc compromises; instead it must have a clear organizational structure, with the party at the top. And the ruling party needs to have the political consciousness of an effective governing force.

Consider two of his most important initiatives: the campaign against corruption and the move toward law-based governance. Xi has decided that corruption is not a mystery to be endured, but a problem to be solved. A few years past the peak of the crackdown, it’s fair to say that the campaign hasn’t solely been effective in removing his adversaries, but has also been broad enough to restore some degree of public confidence in government. A few commentators contend that removal of opportunities for graft have prompted talented people to leave government. But the flip side of that coin has been the improvement in morale among the civil servants who found corruption among colleagues to be intolerable, and can finally see themselves doing public work well. 

And for years, Xi has emphasized following clear rules of written procedure, under the rubric of “law-based governance.”

@@ -40,11 +40,11 @@

Given the importance of the slogan, it’s worthwhile to try to come to terms with the fondness and reverence his generation has for the party’s early days. Many of the people tormented by the party center, including Deng and Xi’s father, have ended up being fiercely loyal to the party.

That shows not just that human nature is complex, but also that the revolutionary heritage of the party instills pride. The CCP started out as a combat party constantly at the mercy of forces grander than itself, achieving its goals after an unusually long struggle that repeatedly brought it to the brink of death. Daniel Koss reminds us that the longer that revolutionary parties have to struggle before consolidating power, the more stronger their ideological commitments and the greater their governance durability tend to be.

Xi is keen to reflect upon the regime’s history. He has decided that the party must believe in itself, and that it is correct to do so: “If our Party members and officials are firm in their ideals and convictions and maintain high morale in their activities and initiatives, and if our people are high-spirited and determined, then we will surely create many miracles.”

-

Furthermore, he has stated: “The prospects are bright but the challenges are severe. All comrades must aim high and look far, be alert to dangers even in times of calm, have the courage to pursue reform and break new ground, and never become hardened to change.” 

+

Furthermore, he has stated: “The prospects are bright but the challenges are severe. All comrades must aim high and look far, be alert to dangers even in times of calm, have the courage to pursue reform and break new ground, and never become hardened to change.”

Thus I’ve arrived at the idea that a commitment to centralized campaigns of inspiration, represented by the tendency to fix clear goals, is the booster stage required to leave the gravitational pull of decadence and complacency. Ross Douthat laments that “a consistent ineffectuality in American governance is just the way things are.”

And he references Jacques Barzun, who defines a decadent society as one that is “peculiarly restless, for it sees no clear lines of advance.” As a society turns developed, its main problems become social: an organizational sclerosis, which no technology is sophisticated enough to solve. No great effort is required to identify the comprehensive paralysis in the US. And that is the political and social current that Xi is trying to reverse in China.

One way to do that is to continue to pursue GDP growth, which has mostly become an unfashionable idea today in the west. Xi reminded the state in July that “economic work must be our core task, if we succeed in that, then the rest of our tasks become easy.”

-

Barry Naughton has noted that “China’s system of incentives for local bureaucrats to encourage growth is extremely unusual, and seems only to exist in China. It is a blunt and powerful instrument.” 

+

Barry Naughton has noted that “China’s system of incentives for local bureaucrats to encourage growth is extremely unusual, and seems only to exist in China. It is a blunt and powerful instrument.”

This emphasis on growth makes it less likely for China to develop into American complacency or decadence. There are other types of paralysis that it stands a good chance of avoiding. With its emphasis on the real economy, it is trying to avoid the fate of Hong Kong, where local elites have reorganized the productive forces completely around sustaining high property prices and managing mainland liquidity flows. With its emphasis on economic growth, it cannot be like Taiwan, whose single bright corporate beacon is surrounded by a mass of firms undergoing genteel decline. With its emphasis on manufacturing, it cannot be like the UK, which is so successful in the sounding-clever industries—television, journalism, finance, and universities—while seeing a falling share of R&D intensity and a global loss of standing among its largest firms.

Douthat’s book does not deal seriously with China, only with a fantasy of a universally-surveilled society under the rubric of a social credit system. If he did engage more seriously, he might pick up what Frank Pieke has termed “neo-socialism,” which is the attempt to harness market liberalization to strengthen state capacity and a more Leninist party.

In return, the state provides purpose and direction, as well as inspiring the rest of society with a transformative mission. It helps, of course, that Xi is a genuine believer in socialism, which to him is both an instrument as well as an end. He’s leveraging that belief to reject decadence and assert agency to point out new lines of advance.

@@ -54,8 +54,7 @@

That was quite a lot of theory. Where does it fall apart?

Xi has said: “If we turn a blind eye to challenges, or even dodge or disguise them; if we fear to advance in the face of challenges and sit by and watch the unfolding calamity; then they will grow beyond our control and cause irreparable damage.”

Instead of heeding this warning, authorities in Wuhan suppressed reporting of a spread of a novel virus. At a time when they should have imposed restrictions, they congregated thousands around a gigantic potluck. That has indeed unfolded into a calamity.

-

Xi has said: “Some officials are perfunctory in their work, shirking responsibility when troubles come and dodging thorny problems. They like to report every trifle to their superiors for approval or directives. In doing so, they appear to be abiding by the rules but are actually avoiding responsibilities. Some make ill-considered or purely arbitrary decisions. They place themselves above the party organization and allow no dissenting voices.”

-

 But as economic growth slows down, the country is doubling down on centralized government. Over the last several years, the state is taking more of a leading role in the economy, which means a larger role for bureaucrats.

+

Xi has said: “Some officials are perfunctory in their work, shirking responsibility when troubles come and dodging thorny problems. They like to report every trifle to their superiors for approval or directives. In doing so, they appear to be abiding by the rules but are actually avoiding responsibilities. Some make ill-considered or purely arbitrary decisions. They place themselves above the party organization and allow no dissenting voices.”

 

But as economic growth slows down, the country is doubling down on centralized government. Over the last several years, the state is taking more of a leading role in the economy, which means a larger role for bureaucrats.

Xi has said: “Self-criticism needs to be specific about our problems and needs to touch underlying questions… We must be gratified when told of our errors; we must not shy away from our shortcomings. We must accommodate different opinions and sharp criticism.”

When medical professionals spoke up about a strange new virus circulating in Wuhan, police gave them reprimands. More and more often, the state is simply arresting critics. Even though the government has every reason to be confident about the effectiveness of its virus containment, it has issued a jail sentence to a citizen journalist under the catch-all charge of “picking quarrels and provoking trouble.” For all the emphasis on seeking truth from facts, the state still maintains this practice of shooting the messenger or jailing its critics.

On its own terms, the party center’s instruction is unevenly followed. And there are plenty of reasons to doubt the sustainability of Chinese growth that exist beyond the party’s capacity for self-reform. The following have all received extensive treatment: demographics will be a clear and serious drag in only a few years; an uncomfortable buildup of debt is now accompanied by growing investor discomfort with strategic defaults; the environment is bearing greater stresses; and based on the state’s aggression abroad and the operation of detention camps for minority groups at home, the rest of the world has become much less friendly towards China. One can add more items here, I want to consider the problems with centralized campaigns of inspiration.

@@ -137,7 +136,7 @@

In the early months of the pandemic, I picked up a new skill: riding a bike. I’ve always been mortified to admit that I never properly knew how. With the encouragement of kind and patient friends, I’ve enjoyed cycling so much that it has become the primary way I get around Beijing. The city is good for cyclists, with its wide bicycle paths and flat roads. (Given the behavior of most drivers though, Beijing requires taking seriously the principle of safety first.) My favorite activity has become to cycle to the Forbidden City and back home, a nice hour-long ride that I would do after lunch. I’m still enjoying the feeling of gliding down a road on my own propulsion, which gives me a sense of slight unreality. That’s been good for thinking: I wrote significant chunks of this letter while riding down Beijing’s second and fourth ring roads.

This year marks my seventh of not drinking. I expect that I’m in the best shape of my life, given that, regular bike rides, occasional badminton sessions, and working out with my personal trainer three times a week. Still, I’m exhausted. That doesn’t mean it’s time to slow down. There are too many interesting things left to do.

- +

Titan, a planet-sized moon of Saturn, has a thick atmosphere and liquid oceans. It and Europa—one of the moons of Jupiter, which might have warm liquid oceans—offer the best chances of discovering extraterrestrial life in our solar system. Credit: JPL @@ -158,10 +157,121 @@

+
+
+
    +
  1. +

    see Anne-Marie Brady: Marketing Dictatorship: Propaganda and Thought Work in Contemporary China↩ +

    +
    2. + +
  2. +

    中国特色社会主义有很多特点和特征，但最本质的特征是坚持中国共产党领导。http://www.qstheory.cn/dukan/qs/2020-07/15/c_1126234524.htm↩ +

    +
    3. + +
  3. +

    For more, see Charles Fishman’s excellent One Giant Leap, which showed how NASA had to invent a thousand and one technologies to reach the moon ↩ +

    +
    4. + +
  4. +

    Sometimes translated as “rule of law”: 依法治国↩ +

    +
    5. + +
  5. +

    + http://www.xinhuanet.com/english/2020-06/07/c_139120424.htm↩ +

    +
    6. + +
  6. +

    see Dan Grover on the UI changes that Chinese apps made: http://dangrover.com/blog/2020/04/05/covid-in-ui.html↩ +

    +
    7. + +
  7. +

    That’s a broad and unfair generalization, I know. This Economist leader offers a more nuanced view: https://www.economist.com/briefing/2020/08/15/xi-jinping-is-trying-to-remake-the-chinese-economy↩ +

    +
    8. + +
  8. +

    This is my translation of 不忘初心、牢记使命. There are variations on the third line, I included one I’ve seen: 永远奋斗 ↩ +

    +
    9. + +
  9. +

    see this excellent discussion between Frederick Teiwes and Joseph Torigian https://omny.fm/shows/the-little-red-podcast/xi-dada-and-daddy-power-the-party-and-the-presiden↩ +

    +
    10. + +
  10. +

    From Dialectical Materialism Is the Worldview and Methodology of Chinese Communists, 广大党员、干部理想信念坚定、干事创业精气神足，人民群众精神振奋、发愤图强，就可以创造出很多人间奇迹 http://www.qstheory.cn/dukan/qs/2018-12/31/c_1123923896.htm↩ +

    +
    11. + +
  11. +

    Report to the 19th party congress: http://www.xinhuanet.com/english/download/Xi_Jinping’s_report_at_19th_CPC_National_Congress.pdf↩ +

    +
    12. + +
  12. +

    see The Decadent Society↩ +

    +
    13. + +
  13. +

    经济工作是中心工作，党的领导当然要在中心工作中得到充分体现，抓住了中心工作这个牛鼻子，其他工作就可以更好展开。http://www.qstheory.cn/dukan/qs/2020-07/15/c_1126234524.htm↩ +

    +
    14. + +
  14. +

    see Frank Pieke’s Knowing China↩ +

    +
    15. + +
  15. +

    see Dialectical Materialism Is the Worldview and Methodology of Chinese Communists 如果对矛盾熟视无睹，甚至回避、掩饰矛盾，在矛盾面前畏缩不前，坐看矛盾恶性转化，那就会积重难返，最后势必造成无法弥补的损失。 http://www.qstheory.cn/dukan/qs/2018-12/31/c_1123923896.htm↩ +

    +
    16. + +
  16. +

    from the speech at the Third Plenary Session of the 19th Central Commission for Discipline Inspection↩ +

    +
    17. + +
  17. +

    from Goals of the Aspiration and Mission Education Campaign, May 31 2019↩ +

    +
    18. + +
  18. +

    http://www.chinafilm.gov.cn/chinafilm/contents/141/2533.shtml↩ +

    +
    19. + +
  19. +

    Wang Hongsheng, a boss at Jinghai, admits to fretting about interruptions to chick supplies, even wondering if President Donald Trump might curb American exports. https://www.economist.com/china/2020/10/31/high-tech-chickens-are-a-case-study-of-why-self-reliance-is-so-hard↩ +

    +
    20. + +
  20. +

    see this WSJ story https://www.wsj.com/articles/the-u-s-vs-china-the-high-cost-of-the-technology-cold-war-11603397438 and Doug Fuller’s claim on Tokyo Electron https://www.jhuapl.edu/assessing-us-china-technology-connections/publications↩ +

    +
    21. + +
  21. +

    This is admittedly a bit of my own fanciful translation of 必须看到，实体经济是基础，各种制造业不能丢，作为14亿人口的大国，粮食和实体产业要以自己为主，这一条绝对不能丢 http://www.qstheory.cn/dukan/qs/2020-10/31/c_1126680390.htm↩ +

    +
    22. + +
+
- \ No newline at end of file + \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/garymarcus/expected.html b/packages/readabilityjs/test/test-pages/garymarcus/expected.html index f2b7d8024..a18902025 100644 --- a/packages/readabilityjs/test/test-pages/garymarcus/expected.html +++ b/packages/readabilityjs/test/test-pages/garymarcus/expected.html @@ -46,6 +46,10 @@

Epilogue:

Last word to philosopher poet Jag Bhalla

+
+

1

+

To be triply sure I asked Aguera y Arcas if I could have access to LaMDA; so far Google has been unwilling to let pesky academics like me have a look see. I’ll report back if that changes.

+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json b/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json index 6af058d1a..ffec318f4 100644 --- a/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json @@ -4,7 +4,9 @@ "dir": null, "excerpt": "The “Weak Garden of Eden” model for the origin and dispersal of modern humans (Harpendinget al., 1993) posits that modern humans spread into separate …", "siteName": null, + "siteIcon": "https://sdfestaticassets-eu-west-1.sciencedirectassets.com/shared-assets/13/images/favSD.ico", "previewImage": "https://ars.els-cdn.com/content/image/1-s2.0-S0047248420X00121-cov150h.gif", "publishedDate": null, + "language": "English", "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/sciencedirect/expected.html b/packages/readabilityjs/test/test-pages/sciencedirect/expected.html index a83f5cc56..17228ea7c 100644 --- a/packages/readabilityjs/test/test-pages/sciencedirect/expected.html +++ b/packages/readabilityjs/test/test-pages/sciencedirect/expected.html @@ -1,27 +1,53 @@ -
-
+
+
-

Elsevier logo

+

Elsevier logo

-

Elsevier +

Elsevier

-

Journal of Human Evolution +

Journal of Human Evolution

Abstract

-

The “Weak Garden of Eden” model for the origin and dispersal of modern humans (Harpendinget al., 1993) posits that modern humans spread into separate regions from a restricted source, around 100 ka (thousand years ago), then passed through population bottlenecks. Around 50 ka, dramatic growth occurred within dispersed populations that were genetically isolated from each other. Population growth began earliest in Africa and later in Eurasia and is hypothesized to have been caused by the invention and spread of a more efficient Later Stone Age/Upper Paleolithic technology, which developed in equatorial Africa.

-

Climatic and geological evidence suggest an alternative hypothesis for Late Pleistocene population bottlenecks and releases. The last glacial period was preceded by one thousand years of the coldest temperatures of the Later Pleistocene (∼71–70 ka), apparently caused by the eruption of Toba, Sumatra. Toba was the largest known explosive eruption of the Quaternary. Toba's volcanic winter could have decimated most modern human populations, especially outside of isolated tropical refugia. Release from the bottleneck could have occurred either at the end of this hypercold phase, or 10,000 years later, at the transition from cold oxygen isotope stage 4 to warmer stage 3. The largest populations surviving through the bottleneck should have been found in the largest tropical refugia, and thus in equatorial Africa. High genetic diversity in modern Africans may thus reflect a less severe bottleneck rather than earlier population growth.

+

The “Weak Garden of Eden” model for the origin and dispersal of modern humans (Harpendinget al., 1993) posits that modern humans spread into separate regions from a restricted source, around 100  +   + ka (thousand years ago), then passed through population bottlenecks. Around 50  +   + ka, dramatic growth occurred within dispersed populations that were genetically isolated from each other. Population growth began earliest in Africa and later in Eurasia and is hypothesized to have been caused by the invention and spread of a more efficient Later Stone Age/Upper Paleolithic technology, which developed in equatorial Africa. +

+

Climatic and geological evidence suggest an alternative hypothesis for Late Pleistocene population bottlenecks and releases. The last glacial period was preceded by one thousand years of the coldest temperatures of the Later Pleistocene (∼71–70  +   + ka), apparently caused by the eruption of Toba, Sumatra. Toba was the largest known explosive eruption of the Quaternary. Toba's volcanic winter could have decimated most modern human populations, especially outside of isolated tropical refugia. Release from the bottleneck could have occurred either at the end of this hypercold phase, or 10,000 years later, at the transition from cold oxygen isotope stage 4 to warmer stage 3. The largest populations surviving through the bottleneck should have been found in the largest tropical refugia, and thus in equatorial Africa. High genetic diversity in modern Africans may thus reflect a less severe bottleneck rather than earlier population growth. +

Volcanic winter may have reduced populations to levels low enough for founder effects, genetic drift and local adaptations to produce rapid population differentiation. If Toba caused the bottlenecks, then modern human races may have differentiated abruptly, only 70 thousand years ago.

+
+
+
+ ☆ +
+
+

P. Mellars

+
+
+
+
+ f1 +
+
+

E-mail: Ambrose@uiuc.edu

+
+
+
-
\ No newline at end of file +
diff --git a/packages/readabilityjs/test/test-readability.js b/packages/readabilityjs/test/test-readability.js index 6db89071e..ac0d3dc8a 100644 --- a/packages/readabilityjs/test/test-readability.js +++ b/packages/readabilityjs/test/test-readability.js @@ -326,8 +326,8 @@ describe("Test pages", function() { describe(testPage.dir, function() { var uri = "http://fakehost/test/page.html"; - runTestsWithItems("jsdom", function(source) { - var doc =parseHTML(source).document; + runTestsWithItems("linkedom", function(source) { + var doc = parseHTML(source).document; removeCommentNodesRecursively(doc); return doc; }, testPage.source, testPage.expectedContent, testPage.expectedMetadata, uri); diff --git a/packages/web/components/elements/ModalPrimitives.tsx b/packages/web/components/elements/ModalPrimitives.tsx index a8e6db4d2..876104de3 100644 --- a/packages/web/components/elements/ModalPrimitives.tsx +++ b/packages/web/components/elements/ModalPrimitives.tsx @@ -97,7 +97,6 @@ export const ModalButtonBar = (props: ModalButtonBarProps) => { }} > + + + + Don't have an account? {' '} + + Sign up + + + + Forgot your password? {' '} + + Click here + + + + + ) +} diff --git a/packages/web/components/templates/EmailResetPassword.tsx b/packages/web/components/templates/EmailResetPassword.tsx new file mode 100644 index 000000000..4d4b256db --- /dev/null +++ b/packages/web/components/templates/EmailResetPassword.tsx @@ -0,0 +1,94 @@ +import { SpanBox, VStack } from '../elements/LayoutPrimitives' +import { Button } from '../elements/Button' +import { StyledText } from '../elements/StyledText' +import { useEffect, useState } from 'react' +import { FormInput } from '../elements/FormElements' +import { TermAndConditionsFooter } from './LoginForm' +import { fetchEndpoint } from '../../lib/appConfig' +import { logoutMutation } from '../../lib/networking/mutations/logoutMutation' +import { styled } from '@stitches/react' +import { useRouter } from 'next/router' +import { formatMessage } from '../../locales/en/messages' +import { parseErrorCodes } from '../../lib/queryParamParser' + +const StyledTextSpan = styled('span', StyledText) + +const BorderedFormInput = styled(FormInput, { + height: '40px', + paddingLeft: '6px', + borderRadius: '6px', + background: 'white', + border: `1px solid 1px solid rgba(0, 0, 0, 0.06)`, +}) + +const FormLabel = styled('label', { + fontSize: '16px', + color: '$omnivoreGray', +}) + +export function EmailResetPassword(): JSX.Element { + const router = useRouter() + const [email, setEmail] = useState(undefined) + const [errorMessage, setErrorMessage] = useState(undefined) + + useEffect(() => { + if (!router.isReady) return + const errorCode = parseErrorCodes(router.query) + const errorMsg = errorCode + ? formatMessage({ id: `error.${errorCode}` }) + : undefined + setErrorMessage(errorMsg) + }, [router.isReady, router.query]) + + return ( +
+ + Reset your password + + + Email + { e.preventDefault(); setEmail(e.target.value); }} + /> + + + + {errorMessage && ( + {errorMessage} + )} + + + +
+ ) +} diff --git a/packages/web/components/templates/EmailSignup.tsx b/packages/web/components/templates/EmailSignup.tsx new file mode 100644 index 000000000..8c12b5b30 --- /dev/null +++ b/packages/web/components/templates/EmailSignup.tsx @@ -0,0 +1,184 @@ +import { HStack, SpanBox, VStack } from '../elements/LayoutPrimitives' +import { Button } from '../elements/Button' +import { StyledText } from '../elements/StyledText' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { FormInput } from '../elements/FormElements' +import { TermAndConditionsFooter } from './LoginForm' +import { fetchEndpoint } from '../../lib/appConfig' +import { useValidateUsernameQuery } from '../../lib/networking/queries/useValidateUsernameQuery' +import { logoutMutation } from '../../lib/networking/mutations/logoutMutation' +import { styled } from '@stitches/react' +import { useRouter } from 'next/router' +import { formatMessage } from '../../locales/en/messages' +import { parseErrorCodes } from '../../lib/queryParamParser' +import Link from 'next/link' + +const StyledTextSpan = styled('span', StyledText) + +const BorderedFormInput = styled(FormInput, { + height: '40px', + paddingLeft: '6px', + borderRadius: '6px', + background: 'white', + border: `1px solid 1px solid rgba(0, 0, 0, 0.06)`, +}) + +const FormLabel = styled('label', { + fontSize: '16px', + color: '$omnivoreGray', +}) + +export function EmailSignup(): JSX.Element { + const router = useRouter() + const [email, setEmail] = useState(undefined) + const [password, setPassword] = useState(undefined) + const [fullname, setFullname] = useState(undefined) + const [username, setUsername] = useState(undefined) + const [debouncedUsername, setDebouncedUsername] = useState(undefined) + const [errorMessage, setErrorMessage] = useState(undefined) + + useEffect(() => { + if (!router.isReady) return + const errorCode = parseErrorCodes(router.query) + const errorMsg = errorCode + ? formatMessage({ id: `error.${errorCode}` }) + : undefined + setErrorMessage(errorMsg) + }, [router.isReady, router.query]) + + const { isUsernameValid, usernameErrorMessage } = useValidateUsernameQuery({ + username: debouncedUsername ?? '', + }) + + const handleUsernameChange = useCallback( + (event: React.ChangeEvent): void => { + setUsername(event.target.value) + setTimeout(() => { + setDebouncedUsername(event.target.value) + }, 400) + }, + [] + ) + + return ( +
+ + Sign Up + + + Email + { e.preventDefault(); setEmail(e.target.value); }} + /> + + + + Password + setPassword(e.target.value)} + /> + + + + Full Name + setFullname(e.target.value)} + /> + + + + Username + + + {username && username.length > 0 && usernameErrorMessage && ( + + {usernameErrorMessage} + + )} + {isUsernameValid && ( + + Username is available. + + )} + + + {errorMessage && ( + {errorMessage} + )} + + + + + + + + Already have an account? {' '} + + Login instead + + + + +
+ ) +} diff --git a/packages/web/components/templates/LoginForm.tsx b/packages/web/components/templates/LoginForm.tsx index 48996afcb..c48ef75f2 100644 --- a/packages/web/components/templates/LoginForm.tsx +++ b/packages/web/components/templates/LoginForm.tsx @@ -9,6 +9,8 @@ import { } from '../../lib/appConfig' import AppleLogin from 'react-apple-login' +const StyledTextSpan = styled('span', StyledText) + export type LoginFormProps = { errorMessage?: string } @@ -104,22 +106,39 @@ export function LoginForm(props: LoginFormProps): JSX.Element { /> )} - +{/* + + + Use your email address to{' '} + + + Login + + {' '} + or{' '} + + + Signup + + {' '} + with your email address. + + */} + + ) } -function LoginFormHeader() { - - - return ( - <> - - - ) -} - function GoogleAuthButton() { return ( @@ -148,8 +167,6 @@ function GoogleAuthButton() { } export function TermAndConditionsFooter(): JSX.Element { - const StyledTextSpan = styled('span', StyledText) - return ( - {props.children} @@ -37,7 +30,7 @@ export function ProfileLayout(props: ProfileLayoutProps): JSX.Element { alignment="center" distribution="between" css={{ - mt: '1px', + mt: '18px', ml: '18px', mr: '0', '@smDown': { diff --git a/packages/web/components/templates/article/HighlightNoteModal.tsx b/packages/web/components/templates/article/HighlightNoteModal.tsx index cece1ecb5..38283f7cb 100644 --- a/packages/web/components/templates/article/HighlightNoteModal.tsx +++ b/packages/web/components/templates/article/HighlightNoteModal.tsx @@ -82,26 +82,34 @@ export function HighlightNoteModal( event.preventDefault() }} > - - - - - +
{ + event.preventDefault() + saveNoteChanges() + props.onOpenChange(false) + }} + > + + + + + +
) diff --git a/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx b/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx index ec73a6477..5d9610987 100644 --- a/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx +++ b/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx @@ -1,7 +1,6 @@ import { Box, HStack, VStack } from './../../elements/LayoutPrimitives' import type { LibraryItem, - LibraryItemsData, LibraryItemsQueryInput, } from '../../../lib/networking/queries/useGetLibraryItemsQuery' import { useGetLibraryItemsQuery } from '../../../lib/networking/queries/useGetLibraryItemsQuery' @@ -39,15 +38,18 @@ import { Label } from '../../../lib/networking/fragments/labelFragment' import { EmptyLibrary } from './EmptyLibrary' import TopBarProgress from 'react-topbar-progress-indicator' import { - State, PageType, + State, } from '../../../lib/networking/fragments/articleFragment' -import { useRegisterActions, createAction, useKBar, Action } from 'kbar' +import { Action, createAction, useKBar, useRegisterActions } from 'kbar' import { EditTitleModal } from './EditTitleModal' import { useGetUserPreferences } from '../../../lib/networking/queries/useGetUserPreferences' -import { searchQuery } from '../../../lib/networking/queries/search' import debounce from 'lodash/debounce' -import { SearchItem, TypeaheadSearchItemsData, typeaheadSearchQuery } from '../../../lib/networking/queries/typeaheadSearch' +import { + SearchItem, + TypeaheadSearchItemsData, + typeaheadSearchQuery, +} from '../../../lib/networking/queries/typeaheadSearch' export type LayoutType = 'LIST_LAYOUT' | 'GRID_LAYOUT' @@ -67,21 +69,24 @@ const SAVED_SEARCHES: Record = { const fetchSearchResults = async (query: string, cb: any) => { if (!query.startsWith('#')) return - const res = await typeaheadSearchQuery({ limit: 10, searchQuery: query.substring(1)}) - cb(res); -}; + const res = await typeaheadSearchQuery({ + limit: 10, + searchQuery: query.substring(1), + }) + cb(res) +} const debouncedFetchSearchResults = debounce((query, cb) => { - fetchSearchResults(query, cb); -}, 300); + fetchSearchResults(query, cb) +}, 300) export function HomeFeedContainer(): JSX.Element { useGetUserPreferences() const { viewerData } = useGetViewerQuery() const router = useRouter() - const { queryValue } = useKBar((state) => ({queryValue: state.searchQuery})); - const [searchResults, setSearchResults] = useState([]); + const { queryValue } = useKBar((state) => ({ queryValue: state.searchQuery })) + const [searchResults, setSearchResults] = useState([]) const defaultQuery = { limit: 10, @@ -118,16 +123,17 @@ export function HomeFeedContainer(): JSX.Element { ) const { itemsPages, size, setSize, isValidating, performActionOnItem } = - useGetLibraryItemsQuery(queryInputs) + useGetLibraryItemsQuery(queryInputs) useEffect(() => { if (queryValue.startsWith('#')) { - debouncedFetchSearchResults(queryValue, (data: TypeaheadSearchItemsData) => { - setSearchResults(data?.typeaheadSearch.items || []) - }) - } - else setSearchResults([]) - + debouncedFetchSearchResults( + queryValue, + (data: TypeaheadSearchItemsData) => { + setSearchResults(data?.typeaheadSearch.items || []) + } + ) + } else setSearchResults([]) }, [queryValue]) useEffect(() => { @@ -217,9 +223,7 @@ export function HomeFeedContainer(): JSX.Element { const target = document.getElementById(id) if (target) { try { - if ( - !isVisible(target) - ) { + if (!isVisible(target)) { target.scrollIntoView({ block: 'center', behavior: isSmouth ? 'smooth' : 'auto', @@ -449,19 +453,19 @@ export function HomeFeedContainer(): JSX.Element { }) ) - const ARCHIVE_ACTION = !activeItem?.node.isArchived ? - createAction({ - section: 'Library', - name: 'Archive selected item', - shortcut: ['e'], - perform: () => handleCardAction('archive', activeItem), - }) : - createAction({ - section: 'Library', - name: 'UnArchive selected item', - shortcut: ['e'], - perform: () => handleCardAction('unarchive', activeItem), - }) + const ARCHIVE_ACTION = !activeItem?.node.isArchived + ? createAction({ + section: 'Library', + name: 'Archive selected item', + shortcut: ['e'], + perform: () => handleCardAction('archive', activeItem), + }) + : createAction({ + section: 'Library', + name: 'UnArchive selected item', + shortcut: ['e'], + perform: () => handleCardAction('unarchive', activeItem), + }) const ACTIVE_ACTIONS = [ ARCHIVE_ACTION, @@ -506,21 +510,27 @@ export function HomeFeedContainer(): JSX.Element { // }), ] - useRegisterActions(searchResults.map(link => ({ - id: link.id, - section: 'Search Results', - name: link.title, - keywords: '#' + link.title, - perform: () => { - const username = viewerData?.me?.profile.username - if (username) { - setActiveCardId(link.id) - router.push(`/${username}/${link.slug}`) - } - }, - })), [searchResults]) + useRegisterActions( + searchResults.map((link) => ({ + id: link.id, + section: 'Search Results', + name: link.title, + keywords: '#' + link.title + ' #' + link.siteName, + perform: () => { + const username = viewerData?.me?.profile.username + if (username) { + setActiveCardId(link.id) + router.push(`/${username}/${link.slug}`) + } + }, + })), + [searchResults] + ) - useRegisterActions(activeCardId ? [...ACTIVE_ACTIONS, ...UNACTIVE_ACTIONS] : UNACTIVE_ACTIONS, [activeCardId, activeItem]); + useRegisterActions( + activeCardId ? [...ACTIVE_ACTIONS, ...UNACTIVE_ACTIONS] : UNACTIVE_ACTIONS, + [activeCardId, activeItem] + ) useFetchMore(handleFetchMore) return ( @@ -600,7 +610,10 @@ type HomeFeedContentProps = { function HomeFeedGrid(props: HomeFeedContentProps): JSX.Element { const { viewerData } = useGetViewerQuery() - const [layout, setLayout] = usePersistedState({ key: 'libraryLayout', initialValue: 'GRID_LAYOUT' }) + const [layout, setLayout] = usePersistedState({ + key: 'libraryLayout', + initialValue: 'GRID_LAYOUT', + }) const [showRemoveLinkConfirmation, setShowRemoveLinkConfirmation] = useState(false) const [linkToRemove, setLinkToRemove] = useState() @@ -861,7 +874,9 @@ function HomeFeedGrid(props: HomeFeedContentProps): JSX.Element { )} {props.showEditTitleModal && ( props.actionHandler('update-item', item)} + updateItem={(item: LibraryItem) => + props.actionHandler('update-item', item) + } onOpenChange={() => props.setShowEditTitleModal(false)} item={linkToEdit as LibraryItem} /> diff --git a/packages/web/components/templates/landing/LandingSectionsContainer.tsx b/packages/web/components/templates/landing/LandingSectionsContainer.tsx index 681d84267..541faf932 100644 --- a/packages/web/components/templates/landing/LandingSectionsContainer.tsx +++ b/packages/web/components/templates/landing/LandingSectionsContainer.tsx @@ -86,6 +86,7 @@ const callToActionStyles = { border: '1px solid #D8D7D5', boxShadow: '0px 7px 8px rgba(32, 31, 29, 0.03), 0px 18px 24px rgba(32, 31, 29, 0.03)', padding: 40, + marginTop: 64, height: 330, '@mdDown': { display: 'none', diff --git a/packages/web/lib/keyboardShortcuts/navigationShortcuts.ts b/packages/web/lib/keyboardShortcuts/navigationShortcuts.ts index a00fcbcba..0bc1c841f 100644 --- a/packages/web/lib/keyboardShortcuts/navigationShortcuts.ts +++ b/packages/web/lib/keyboardShortcuts/navigationShortcuts.ts @@ -199,13 +199,13 @@ export function highlightBarKeyboardCommands( { shortcutKeys: ['shift', 'h'], actionDescription: 'Highlight selected text', - shortcutKeyDescription: 'shift + h', + shortcutKeyDescription: 'h', callback: () => actionHandler('createHighlight'), }, { shortcutKeys: ['shift', 'c'], actionDescription: 'Annotate selected text', - shortcutKeyDescription: 'shift + c', + shortcutKeyDescription: 'n', callback: () => setTimeout(() => actionHandler('openNoteModal'), 0), }, // { diff --git a/packages/web/next.config.js b/packages/web/next.config.js index e637ae2c8..610dff904 100644 --- a/packages/web/next.config.js +++ b/packages/web/next.config.js @@ -8,7 +8,6 @@ const moduleExports = { ], }, rewrites: () => [ - { source: '/about', destination: '/static/landing/about.html' }, { source: '/api/graphql', destination: `https://api-${process.env.NEXT_PUBLIC_APP_ENV}.omnivore.app/api/graphql`, diff --git a/packages/web/pages/[username]/[slug]/index.tsx b/packages/web/pages/[username]/[slug]/index.tsx index a4cbe75d6..a2c9d36c5 100644 --- a/packages/web/pages/[username]/[slug]/index.tsx +++ b/packages/web/pages/[username]/[slug]/index.tsx @@ -121,6 +121,23 @@ export default function Home(): JSX.Element { } }, [article, cache, mutate, router, readerSettings]) + useEffect(() => { + const archive = () => { + actionHandler('archive') + } + const openOriginalArticle = () => { + actionHandler('openOriginalArticle') + } + + document.addEventListener('archive', archive) + document.addEventListener('openOriginalArticle', openOriginalArticle) + + return () => { + document.removeEventListener('archive', archive) + document.removeEventListener('openOriginalArticle', openOriginalArticle) + } + }, [actionHandler]) + useKeyboardShortcuts( articleKeyboardCommands(router, async (action) => { actionHandler(action) @@ -144,7 +161,9 @@ export default function Home(): JSX.Element { section: 'Article', name: 'Open original article', shortcut: ['o'], - perform: () => actionHandler('openOriginalArticle') + perform: () => { + document.dispatchEvent(new Event('openOriginalArticle')); + } }, { id: 'back_home', @@ -158,7 +177,9 @@ export default function Home(): JSX.Element { section: 'Article', name: 'Archive current item', shortcut: ['e'], - perform: () => actionHandler('archive'), + perform: () => { + document.dispatchEvent(new Event('archive')); + } }, { id: 'highlight', diff --git a/packages/web/pages/landing.tsx b/packages/web/pages/about.tsx similarity index 100% rename from packages/web/pages/landing.tsx rename to packages/web/pages/about.tsx diff --git a/packages/web/pages/email-login.tsx b/packages/web/pages/email-login.tsx index c402990ea..6011118f0 100644 --- a/packages/web/pages/email-login.tsx +++ b/packages/web/pages/email-login.tsx @@ -1,48 +1,56 @@ -import { PrimaryLayout } from '../components/templates/PrimaryLayout' -import { useEffect, useState } from 'react' -import { useRouter } from 'next/router' -import { StyledText } from '../components/elements/StyledText' -import { fetchEndpoint } from '../lib/appConfig' -import { parseErrorCodes } from '../lib/queryParamParser' -import { formatMessage } from '../locales/en/messages' - -export default function EmailLogin(): JSX.Element { - const [errorMessage, setErrorMessage] = useState( - undefined - ) - const [message, setMessage] = useState(undefined) - const router = useRouter() - - useEffect(() => { - if (!router.isReady) return - const errorCode = parseErrorCodes(router.query) - const errorMsg = errorCode - ? formatMessage({ id: `error.${errorCode}` }) - : undefined - setErrorMessage(errorMsg) - - const message = router.query.message - ? formatMessage({ id: `login.${router.query.message}` }) - : undefined - setMessage(message) - }, [router.isReady, router.query]) +import { PageMetaData } from '../components/patterns/PageMetaData' +import { ProfileLayout } from '../components/templates/ProfileLayout' +import { EmailLogin } from '../components/templates/EmailLogin' +export default function EmailLoginPage(): JSX.Element { return ( - - {message && {message}} -

Email Login

-
-
- - -
-
- - -
- {errorMessage && {errorMessage}} - -
- + <> + + + + +
+ ) } + +// export default function EmailLogin(): JSX.Element { +// const [errorMessage, setErrorMessage] = useState( +// undefined +// ) +// const [message, setMessage] = useState(undefined) +// const router = useRouter() + +// useEffect(() => { +// if (!router.isReady) return +// const errorCode = parseErrorCodes(router.query) +// const errorMsg = errorCode +// ? formatMessage({ id: `error.${errorCode}` }) +// : undefined +// setErrorMessage(errorMsg) + +// const message = router.query.message +// ? formatMessage({ id: `login.${router.query.message}` }) +// : undefined +// setMessage(message) +// }, [router.isReady, router.query]) + +// return ( +// +// {message && {message}} +//

Email Login

+//
+//
+// +// +//
+//
+// +// +//
+// {errorMessage && {errorMessage}} +// +//
+// +// ) +// } diff --git a/packages/web/pages/email-registration.tsx b/packages/web/pages/email-registration.tsx deleted file mode 100644 index 468569262..000000000 --- a/packages/web/pages/email-registration.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import { PrimaryLayout } from '../components/templates/PrimaryLayout' -import { useEffect, useState } from 'react' -import { useRouter } from 'next/router' -import { StyledText } from '../components/elements/StyledText' -import { fetchEndpoint } from '../lib/appConfig' -import { parseErrorCodes } from '../lib/queryParamParser' -import { formatMessage } from '../locales/en/messages' - -export default function EmailRegistration(): JSX.Element { - const [errorMessage, setErrorMessage] = useState( - undefined - ) - const router = useRouter() - - useEffect(() => { - if (!router.isReady) return - const errorCode = parseErrorCodes(router.query) - const message = errorCode - ? formatMessage({ id: `error.${errorCode}` }) - : undefined - setErrorMessage(message) - }, [router.isReady, router.query]) - - return ( - -

Email Registration

-
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
- {errorMessage && {errorMessage}} - -
- - ) -} diff --git a/packages/web/pages/email-reset-password.tsx b/packages/web/pages/email-reset-password.tsx new file mode 100644 index 000000000..88b0e4b8c --- /dev/null +++ b/packages/web/pages/email-reset-password.tsx @@ -0,0 +1,15 @@ +import { PageMetaData } from '../components/patterns/PageMetaData' +import { ProfileLayout } from '../components/templates/ProfileLayout' +import { EmailResetPassword } from '../components/templates/EmailResetPassword' + +export default function EmailRegistrationPage(): JSX.Element { + return ( + <> + + + + +
+ + ) +} diff --git a/packages/web/pages/email-signup.tsx b/packages/web/pages/email-signup.tsx new file mode 100644 index 000000000..5fa84d37b --- /dev/null +++ b/packages/web/pages/email-signup.tsx @@ -0,0 +1,15 @@ +import { PageMetaData } from '../components/patterns/PageMetaData' +import { ProfileLayout } from '../components/templates/ProfileLayout' +import { EmailSignup } from '../components/templates/EmailSignup' + +export default function EmailRegistrationPage(): JSX.Element { + return ( + <> + + + + +
+ + ) +} diff --git a/packages/web/public/static/landing/about.html b/packages/web/public/static/landing/about.html deleted file mode 100644 index 850d2def7..000000000 --- a/packages/web/public/static/landing/about.html +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - About Omnivore

Everything you read. Safe, organized, and easy to share.

Get started
Illustration

This is Omnivore

Collect and share the best of the web

Simply reader-friendly.

With a single click using Omnivore’s extension or mobile app, save any link you come across on the Internet. Your links are saved forever, so you will never lose anything. We also strip out the unnecessary content to give you a reader-friendly view of your saved pages.

Save articles
View your library

Make it your own.

Curate your own personal collection of saved links and annotate these links with your thoughts. Omnivore lets you highlight any text you find inspiring and attach personal notes to these highlights.

A better way to share.

With Omnivore’s highlight and notes feature, you can share specific snippets from a link with your friends. Our share feature is integrated with top social media sites so sharing with your friends is just one click away.

Share your notes
Customize your profile

Discover new content.

Follow friends and people you admire and see what they are reading. With our highlight and notes feature, you can read through the eyes of others and see what others have highlighted and commented.

Get started with Omnivore ->
\ No newline at end of file diff --git a/packages/web/public/static/landing/landing-1.png b/packages/web/public/static/landing/landing-1.png index f199bf765..bd6c11c0b 100644 Binary files a/packages/web/public/static/landing/landing-1.png and b/packages/web/public/static/landing/landing-1.png differ diff --git a/packages/web/public/static/landing/landing-1@2x.png b/packages/web/public/static/landing/landing-1@2x.png index 299ab5d2e..0a59e4b23 100644 Binary files a/packages/web/public/static/landing/landing-1@2x.png and b/packages/web/public/static/landing/landing-1@2x.png differ diff --git a/packages/web/public/static/landing/landing-2.png b/packages/web/public/static/landing/landing-2.png index b602ca09b..17dc66bc5 100644 Binary files a/packages/web/public/static/landing/landing-2.png and b/packages/web/public/static/landing/landing-2.png differ diff --git a/packages/web/public/static/landing/landing-2@2x.png b/packages/web/public/static/landing/landing-2@2x.png index 2247052f0..c4fe75fb8 100644 Binary files a/packages/web/public/static/landing/landing-2@2x.png and b/packages/web/public/static/landing/landing-2@2x.png differ diff --git a/yarn.lock b/yarn.lock index 96f5c31b5..96ef94df4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2350,21 +2350,6 @@ dependencies: tslib "^2.1.0" -"@google-cloud/common@^3.4.1": - version "3.7.1" - resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.7.1.tgz#e6a4b512ea0c72435b853831565bfba6a8dff2ac" - integrity sha512-BJfcV5BShbunYcn5HniebXLVp2Y6fpuesNegyar5CG8H2AKYHlKxnVID+FSwy92WAW4N2lpGdvxRsmiAn8Fc3w== - dependencies: - "@google-cloud/projectify" "^2.0.0" - "@google-cloud/promisify" "^2.0.0" - arrify "^2.0.1" - duplexify "^4.1.1" - ent "^2.2.0" - extend "^3.0.2" - google-auth-library "^7.0.2" - retry-request "^4.2.2" - teeny-request "^7.0.0" - "@google-cloud/common@^3.8.1": version "3.9.0" resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.9.0.tgz#d93e62d13e66edacfad1cd25b20fdbbc11d9f6dd" @@ -2380,6 +2365,21 @@ retry-request "^4.2.2" teeny-request "^7.0.0" +"@google-cloud/common@^4.0.0": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-4.0.2.tgz#2b34ad6213c42de081c3d633ab26381cbf17a421" + integrity sha512-LgUoPQq1CNzMAtqnIJLetj7hlzZUS+CG9mBuZthqek6+eGu3PsH2IuEbbtLSUgPZNgOgi/fWGWpbNPP7wgjF0A== + dependencies: + "@google-cloud/projectify" "^3.0.0" + "@google-cloud/promisify" "^3.0.0" + arrify "^2.0.1" + duplexify "^4.1.1" + ent "^2.2.0" + extend "^3.0.2" + google-auth-library "^8.0.2" + retry-request "^5.0.0" + teeny-request "^8.0.0" + "@google-cloud/firestore@^4.5.0": version "4.15.1" resolved "https://registry.yarnpkg.com/@google-cloud/firestore/-/firestore-4.15.1.tgz#ed764fc76823ce120e68fe8c27ef1edd0650cd93" @@ -2403,32 +2403,32 @@ read-pkg-up "^7.0.1" semver "^7.3.5" -"@google-cloud/logging-winston@^4.1.2": - version "4.2.2" - resolved "https://registry.yarnpkg.com/@google-cloud/logging-winston/-/logging-winston-4.2.2.tgz#372b5c17a3ab00a9628c5ae4924f564829c0b3bb" - integrity sha512-KUg1VIrUjPyK1+mWNrKjcjCBoMONZSK0qYbGb41WYnhPAECKvjeWDXumO/Zwr66qa7ktvQDnFKRk7ASVmUPWrg== +"@google-cloud/logging-winston@^5.1.1": + version "5.1.1" + resolved "https://registry.yarnpkg.com/@google-cloud/logging-winston/-/logging-winston-5.1.1.tgz#8e25de69d2ce8c3e89cb62b4a50db50900c3d867" + integrity sha512-KvZjQgFOHGUS1xLgGIcbQmRrniX1MLr9jevz3LUvt/c2uZJYw+DeYedRnr8AzFy9askpoKbEDkDBSTnATXEqsg== dependencies: - "@google-cloud/logging" "^9.6.9" - google-auth-library "^7.0.0" + "@google-cloud/logging" "^10.0.1" + google-auth-library "^8.0.2" lodash.mapvalues "^4.6.0" winston-transport "^4.3.0" -"@google-cloud/logging@^9.6.9": - version "9.7.0" - resolved "https://registry.yarnpkg.com/@google-cloud/logging/-/logging-9.7.0.tgz#30dde6c1fbcaf8275bfd88250032b31d58023218" - integrity sha512-/3tZ+gefcDtcDu93kcJ/FpdEmvyUtvM0HGBOHyTBehqIpLFWnF6x7VPQL92dZE3arBdIuZh79J41/LIVeHNvqg== +"@google-cloud/logging@^10.0.1": + version "10.1.1" + resolved "https://registry.yarnpkg.com/@google-cloud/logging/-/logging-10.1.1.tgz#c3e9dff453ab85147bc5e017a836cc29e127ac19" + integrity sha512-PPtlrE+fWhJssDMcgclfzjWlRypMQxHRqPRZAFvexHcYytpAJ+ppBmczi/30LtC7fyrj9K8wvifNPI+1DSUJ4Q== dependencies: - "@google-cloud/common" "^3.4.1" - "@google-cloud/paginator" "^3.0.0" - "@google-cloud/projectify" "^2.0.0" - "@google-cloud/promisify" "^2.0.0" + "@google-cloud/common" "^4.0.0" + "@google-cloud/paginator" "^4.0.0" + "@google-cloud/projectify" "^3.0.0" + "@google-cloud/promisify" "^3.0.0" arrify "^2.0.1" dot-prop "^6.0.0" eventid "^2.0.0" extend "^3.0.2" gcp-metadata "^4.0.0" - google-auth-library "^7.0.0" - google-gax "^2.24.1" + google-auth-library "^8.0.2" + google-gax "^3.0.1" on-finished "^2.3.0" pumpify "^2.0.1" stream-events "^1.0.5" @@ -2465,6 +2465,14 @@ arrify "^2.0.0" extend "^3.0.2" +"@google-cloud/paginator@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@google-cloud/paginator/-/paginator-4.0.0.tgz#9c3e01544717aecb9a922b4269ff298f30a0f1bb" + integrity sha512-wNmCZl+2G2DmgT/VlF+AROf80SoaC/CwS8trwmjNaq26VRNK8yPbU5F/Vy+R9oDAGKWQU2k8+Op5H4kFJVXFaQ== + dependencies: + arrify "^2.0.0" + extend "^3.0.2" + "@google-cloud/precise-date@^2.0.0": version "2.0.3" resolved "https://registry.yarnpkg.com/@google-cloud/precise-date/-/precise-date-2.0.3.tgz#14f6f28ce35dabf3882e7aeab1c9d51bd473faed" @@ -2475,11 +2483,21 @@ resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-2.1.0.tgz#3df145c932e244cdeb87a30d93adce615bc69e6d" integrity sha512-qbpidP/fOvQNz3nyabaVnZqcED1NNzf7qfeOlgtAZd9knTwY+KtsGRkYpiQzcATABy4gnGP2lousM3S0nuWVzA== +"@google-cloud/projectify@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-3.0.0.tgz#302b25f55f674854dce65c2532d98919b118a408" + integrity sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA== + "@google-cloud/promisify@^2.0.0": version "2.0.3" resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.3.tgz#f934b5cdc939e3c7039ff62b9caaf59a9d89e3a8" integrity sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw== +"@google-cloud/promisify@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-3.0.0.tgz#5cd6941fc30c4acac18051706aa5af96069bd3e3" + integrity sha512-91ArYvRgXWb73YvEOBMmOcJc0bDRs5yiVHnqkwoG0f3nm7nZuipllz6e7BvFESBvjkDTBC0zMD8QxedUwNLc1A== + "@google-cloud/pubsub@^2.16.0", "@google-cloud/pubsub@^2.16.3", "@google-cloud/pubsub@^2.18.4": version "2.19.0" resolved "https://registry.yarnpkg.com/@google-cloud/pubsub/-/pubsub-2.19.0.tgz#45541e66db9fbe9faa4f00e89a44f41954c6fc86" @@ -4617,14 +4635,6 @@ dependencies: "@babel/runtime" "^7.13.10" -"@radix-ui/react-arrow@0.1.3": - version "0.1.3" - resolved "https://registry.yarnpkg.com/@radix-ui/react-arrow/-/react-arrow-0.1.3.tgz#17f86eab216c48aff17b13b811569a9bbabaa44d" - integrity sha512-9x1gRYdlUD5OUwY7L+M+4FY/YltDSsrNSj8QXGPbxZxL5ghWXB/4lhyIGccCwk/e8ggfmQYv9SRNmn3LavPo3A== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-primitive" "0.1.3" - "@radix-ui/react-arrow@0.1.4": version "0.1.4" resolved "https://registry.yarnpkg.com/@radix-ui/react-arrow/-/react-arrow-0.1.4.tgz#a871448a418cd3507d83840fdd47558cb961672b" @@ -4706,19 +4716,6 @@ aria-hidden "^1.1.1" react-remove-scroll "^2.4.0" -"@radix-ui/react-dismissable-layer@0.1.3": - version "0.1.3" - resolved "https://registry.yarnpkg.com/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-0.1.3.tgz#d427c7520c3799d2b957e40e7d67045d96120356" - integrity sha512-3veE7M8K13Qb+6+tC3DHWmWV9VMuuRoZvRLdrvz7biSraK/qkGBN4LbKZDaTdw2D2HS7RNpSd/sF8pFd3TaAgA== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/primitive" "0.1.0" - "@radix-ui/react-context" "0.1.1" - "@radix-ui/react-primitive" "0.1.3" - "@radix-ui/react-use-body-pointer-events" "0.1.0" - "@radix-ui/react-use-callback-ref" "0.1.0" - "@radix-ui/react-use-escape-keydown" "0.1.0" - "@radix-ui/react-dismissable-layer@0.1.5": version "0.1.5" resolved "https://registry.yarnpkg.com/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-0.1.5.tgz#9379032351e79028d472733a5cc8ba4a0ea43314" @@ -4753,16 +4750,6 @@ dependencies: "@babel/runtime" "^7.13.10" -"@radix-ui/react-focus-scope@0.1.3": - version "0.1.3" - resolved "https://registry.yarnpkg.com/@radix-ui/react-focus-scope/-/react-focus-scope-0.1.3.tgz#b1cc825b6190001d731417ed90d192d13b41bce1" - integrity sha512-bKi+lw14SriQqYWMBe13b/wvxSqYMC+3FylMUEwOKA6JrBoldpkhX5XffGDdpDRTTpjbncdH3H7d1PL5Bs7Ikg== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-compose-refs" "0.1.0" - "@radix-ui/react-primitive" "0.1.3" - "@radix-ui/react-use-callback-ref" "0.1.0" - "@radix-ui/react-focus-scope@0.1.4": version "0.1.4" resolved "https://registry.yarnpkg.com/@radix-ui/react-focus-scope/-/react-focus-scope-0.1.4.tgz#c830724e212d42ffaaa81aee49533213d09b47df" @@ -4773,14 +4760,6 @@ "@radix-ui/react-primitive" "0.1.4" "@radix-ui/react-use-callback-ref" "0.1.0" -"@radix-ui/react-id@0.1.4": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@radix-ui/react-id/-/react-id-0.1.4.tgz#4cd6126e6ac8a43ebe6d52948a068b797cc9ad71" - integrity sha512-/hq5m/D0ZfJWOS7TLF+G0l08KDRs87LBE46JkAvgKkg1fW4jkucx9At9D9vauIPSbdNmww5kXEp566hMlA8eXA== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-use-layout-effect" "0.1.0" - "@radix-ui/react-id@0.1.5", "@radix-ui/react-id@^0.1.1": version "0.1.5" resolved "https://registry.yarnpkg.com/@radix-ui/react-id/-/react-id-0.1.5.tgz#010d311bedd5a2884c1e9bb6aaaa4e6cc1d1d3b8" @@ -4825,41 +4804,26 @@ react-remove-scroll "^2.4.0" "@radix-ui/react-popover@^0.1.1": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@radix-ui/react-popover/-/react-popover-0.1.4.tgz#f7302ae75f94007edc6294815bc031ec46fa5b4e" - integrity sha512-5oaBFkGCGfomXO5HTrhORixDoeAjl3XeSLbVSC9G1Yq//lfaTC5sJYSKSf4mnQulzM9XYchtyjHoBa0O1OBM5Q== + version "0.1.6" + resolved "https://registry.yarnpkg.com/@radix-ui/react-popover/-/react-popover-0.1.6.tgz#788e969239d9c55239678e615ab591b6b7ba5cdc" + integrity sha512-zQzgUqW4RQDb0ItAL1xNW4K4olUrkfV3jeEPs9rG+nsDQurO+W9TT+YZ9H1mmgAJqlthyv1sBRZGdBm4YjtD6Q== dependencies: "@babel/runtime" "^7.13.10" "@radix-ui/primitive" "0.1.0" "@radix-ui/react-compose-refs" "0.1.0" "@radix-ui/react-context" "0.1.1" - "@radix-ui/react-dismissable-layer" "0.1.3" + "@radix-ui/react-dismissable-layer" "0.1.5" "@radix-ui/react-focus-guards" "0.1.0" - "@radix-ui/react-focus-scope" "0.1.3" - "@radix-ui/react-id" "0.1.4" - "@radix-ui/react-popper" "0.1.3" - "@radix-ui/react-portal" "0.1.3" - "@radix-ui/react-presence" "0.1.1" - "@radix-ui/react-primitive" "0.1.3" + "@radix-ui/react-focus-scope" "0.1.4" + "@radix-ui/react-id" "0.1.5" + "@radix-ui/react-popper" "0.1.4" + "@radix-ui/react-portal" "0.1.4" + "@radix-ui/react-presence" "0.1.2" + "@radix-ui/react-primitive" "0.1.4" "@radix-ui/react-use-controllable-state" "0.1.0" aria-hidden "^1.1.1" react-remove-scroll "^2.4.0" -"@radix-ui/react-popper@0.1.3": - version "0.1.3" - resolved "https://registry.yarnpkg.com/@radix-ui/react-popper/-/react-popper-0.1.3.tgz#a93bdd72845566007e5f3868caddd62318bb781e" - integrity sha512-2OV2YaJv7iTZexJY3HJ7B6Fs1A/3JXd3fRGU4JY0guACfGMD1C/jSgds505MKQOTiHE/quI6j3/q8yfzFjJR9g== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/popper" "0.1.0" - "@radix-ui/react-arrow" "0.1.3" - "@radix-ui/react-compose-refs" "0.1.0" - "@radix-ui/react-context" "0.1.1" - "@radix-ui/react-primitive" "0.1.3" - "@radix-ui/react-use-rect" "0.1.1" - "@radix-ui/react-use-size" "0.1.0" - "@radix-ui/rect" "0.1.1" - "@radix-ui/react-popper@0.1.4": version "0.1.4" resolved "https://registry.yarnpkg.com/@radix-ui/react-popper/-/react-popper-0.1.4.tgz#dfc055dcd7dfae6a2eff7a70d333141d15a5d029" @@ -4875,15 +4839,6 @@ "@radix-ui/react-use-size" "0.1.1" "@radix-ui/rect" "0.1.1" -"@radix-ui/react-portal@0.1.3": - version "0.1.3" - resolved "https://registry.yarnpkg.com/@radix-ui/react-portal/-/react-portal-0.1.3.tgz#56826e789b3d4e37983f6d23666e3f1b1b9ee358" - integrity sha512-DrV+sPYLs0HhmX5/b7yRT6nLM9Nl6FtQe2KUG+46kiCOKQ+0XzNMO5hmeQtyq0mRf/qlC02rFu6OMsWpIqVsJg== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-primitive" "0.1.3" - "@radix-ui/react-use-layout-effect" "0.1.0" - "@radix-ui/react-portal@0.1.4": version "0.1.4" resolved "https://registry.yarnpkg.com/@radix-ui/react-portal/-/react-portal-0.1.4.tgz#17bdce3d7f1a9a0b35cb5e935ab8bc562441a7d2" @@ -4893,15 +4848,6 @@ "@radix-ui/react-primitive" "0.1.4" "@radix-ui/react-use-layout-effect" "0.1.0" -"@radix-ui/react-presence@0.1.1": - version "0.1.1" - resolved "https://registry.yarnpkg.com/@radix-ui/react-presence/-/react-presence-0.1.1.tgz#2088dec6f4f8042f83dd2d6bf9e8ef09dadbbc15" - integrity sha512-LsL+NcWDpFUAYCmXeH02o4pgqcSLpwxP84UIjCtpIKrsPe2vLuhcp79KC/jZJeXz+of2lUpMAxpM+eCpxFZtlg== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-compose-refs" "0.1.0" - "@radix-ui/react-use-layout-effect" "0.1.0" - "@radix-ui/react-presence@0.1.2": version "0.1.2" resolved "https://registry.yarnpkg.com/@radix-ui/react-presence/-/react-presence-0.1.2.tgz#9f11cce3df73cf65bc348e8b76d891f0d54c1fe3" @@ -4979,14 +4925,6 @@ "@radix-ui/react-use-rect" "0.1.1" "@radix-ui/react-visually-hidden" "0.1.4" -"@radix-ui/react-use-body-pointer-events@0.1.0": - version "0.1.0" - resolved "https://registry.yarnpkg.com/@radix-ui/react-use-body-pointer-events/-/react-use-body-pointer-events-0.1.0.tgz#29b211464493f8ca5149ce34b96b95abbc97d741" - integrity sha512-svPyoHCcwOq/vpWNEvdH/yD91vN9p8BtiozNQbjVmJRxQ/vS12zqk70AxTGWe+2ZKHq2sggpEQNTv1JHyVFlnQ== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-use-layout-effect" "0.1.0" - "@radix-ui/react-use-body-pointer-events@0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@radix-ui/react-use-body-pointer-events/-/react-use-body-pointer-events-0.1.1.tgz#63e7fd81ca7ffd30841deb584cd2b7f460df2597" @@ -5047,13 +4985,6 @@ "@babel/runtime" "^7.13.10" "@radix-ui/rect" "0.1.1" -"@radix-ui/react-use-size@0.1.0": - version "0.1.0" - resolved "https://registry.yarnpkg.com/@radix-ui/react-use-size/-/react-use-size-0.1.0.tgz#dc49295d646f5d3f570943dbb88bd94fc7db7daf" - integrity sha512-TcZAsR+BYI46w/RbaSFCRACl+Jh6mDqhu6GS2r0iuJpIVrj8atff7qtTjmMmfGtEDNEjhl7DxN3pr1nTS/oruQ== - dependencies: - "@babel/runtime" "^7.13.10" - "@radix-ui/react-use-size@0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@radix-ui/react-use-size/-/react-use-size-0.1.1.tgz#f6b75272a5d41c3089ca78c8a2e48e5f204ef90f" @@ -5111,16 +5042,16 @@ any-observable "^0.3.0" "@segment/analytics-next@^1.33.5": - version "1.38.0" - resolved "https://registry.yarnpkg.com/@segment/analytics-next/-/analytics-next-1.38.0.tgz#efbe0e1f8a3c6bd1522efad8d5ba70ed8c0af4e4" - integrity sha512-904AYNi6wekvAVcaVGQhcH5Jeif98JvN5YEmsuiDuw+j6oiWijdJN6krY4QLh703zPIJvZVf5j+JSh13mZNaiA== + version "1.40.0" + resolved "https://registry.yarnpkg.com/@segment/analytics-next/-/analytics-next-1.40.0.tgz#703d2389340e5511b4080ea003668cd0bcb32024" + integrity sha512-Kg5C55BCB+/jRcFTWU2vEZdUb7nPaKutkAHobSQRld/RQWX24RcMzx20jtspZ8m0FCXQrg0Mm8Khpk2sNvh3Hg== dependencies: "@lukeed/uuid" "^2.0.0" "@segment/analytics.js-video-plugins" "^0.2.1" "@segment/facade" "^3.4.9" "@segment/tsub" "^0.1.12" dset "^3.1.2" - js-cookie "^2.2.1" + js-cookie "3.0.1" node-fetch "^2.6.7" spark-md5 "^3.0.1" tslib "^2.4.0" @@ -7569,13 +7500,6 @@ resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.12.tgz#6b2c510a7ad7039e98e7b8d3d6598f4359e5c080" integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw== -"@types/debug@^4.1.0": - version "4.1.7" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" - integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== - dependencies: - "@types/ms" "*" - "@types/diff-match-patch@^1.0.32": version "1.0.32" resolved "https://registry.yarnpkg.com/@types/diff-match-patch/-/diff-match-patch-1.0.32.tgz#d9c3b8c914aa8229485351db4865328337a3d09f" @@ -7877,11 +7801,6 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw== -"@types/ms@*": - version "0.7.31" - resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" - integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== - "@types/nanoid@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/nanoid/-/nanoid-3.0.0.tgz#c757b20f343f3a1dd76e80a9a431b6290fc20f35" @@ -7987,13 +7906,6 @@ resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.4.tgz#fcf7205c25dff795ee79af1e30da2c9790808f11" integrity sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ== -"@types/puppeteer@*": - version "5.4.6" - resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-5.4.6.tgz#afc438e41dcbc27ca1ba0235ea464a372db2b21c" - integrity sha512-98Kghehs7+/GD9b56qryhqdqVCXUTbetTv3PlvDnmFRTHQH0j9DIp1f7rkAW3BAj4U3yoeSEQnKgdW8bDq0Y0Q== - dependencies: - "@types/node" "*" - "@types/qs@*", "@types/qs@^6.9.5": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" @@ -10832,17 +10744,6 @@ cliui@^7.0.2: strip-ansi "^6.0.0" wrap-ansi "^7.0.0" -clone-deep@^0.2.4: - version "0.2.4" - resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-0.2.4.tgz#4e73dd09e9fb971cc38670c5dced9c1896481cc6" - integrity sha1-TnPdCen7lxzDhnDF3O2cGJZIHMY= - dependencies: - for-own "^0.1.3" - is-plain-object "^2.0.1" - kind-of "^3.0.2" - lazy-cache "^1.0.3" - shallow-clone "^0.1.2" - clone-deep@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" @@ -11727,9 +11628,9 @@ cyclist@^1.0.1: integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= cypress@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-10.1.0.tgz#6514a26c721822a02bc194e9a7f72c3142aea174" - integrity sha512-aQ4JVZVib4Xd9FZW8IRZfKelUvqF4y5A+oUbNvn8TlsBmEfIg3m5Xd6Mt6PVU/jHiVJ9Psl905B3ZPnrDcmyuQ== + version "10.3.0" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-10.3.0.tgz#fae8d32f0822fcfb938e79c7c31ef344794336ae" + integrity sha512-txkQWKzvBVnWdCuKs5Xc08gjpO89W2Dom2wpZgT9zWZT5jXxqPIxqP/NC1YArtkpmp3fN5HW8aDjYBizHLUFvg== dependencies: "@cypress/request" "^2.88.10" "@cypress/xvfb" "^1.2.4" @@ -13802,23 +13703,11 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0, follow-redirects@^1.14.4, fol resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== -for-in@^0.1.3: - version "0.1.8" - resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1" - integrity sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE= - for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= -for-own@^0.1.3: - version "0.1.5" - resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" - integrity sha1-UmXGgaTylNq78XyVCbZ2OqhFEM4= - dependencies: - for-in "^1.0.1" - for-own@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" @@ -13986,15 +13875,6 @@ fs-extra@^0.30.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" -fs-extra@^10.0.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" - integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -14497,7 +14377,7 @@ goober@^2.1.1: resolved "https://registry.yarnpkg.com/goober/-/goober-2.1.8.tgz#e592c04d093cb38f77b38cfcb012b7811c85765e" integrity sha512-S0C85gCzcfFCMSdjD/CxyQMt1rbf2qEg6hmDzxk2FfD7+7Ogk55m8ZFUMtqNaZM4VVX/qaU9AzSORG+Gf4ZpAQ== -google-auth-library@^7.0.0, google-auth-library@^7.0.2, google-auth-library@^7.6.1, google-auth-library@^7.9.2: +google-auth-library@^7.0.0, google-auth-library@^7.6.1, google-auth-library@^7.9.2: version "7.14.1" resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.14.1.tgz#e3483034162f24cc71b95c8a55a210008826213c" integrity sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA== @@ -15729,7 +15609,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.0.2, is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -16041,7 +15921,7 @@ is-plain-object@5.0.0, is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== -is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: +is-plain-object@^2.0.3, is-plain-object@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== @@ -16894,10 +16774,10 @@ js-beautify@^1.13.0: glob "^7.1.3" nopt "^5.0.0" -js-cookie@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-2.2.1.tgz#69e106dc5d5806894562902aa5baec3744e9b2b8" - integrity sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ== +js-cookie@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-3.0.1.tgz#9e39b4c6c2f56563708d7d31f6f5f21873a92414" + integrity sha512-+0rgsUXZu4ncpPxRL+lNEptWMOWl9etvPHc/koSRp6MPwpRYAhmk0dUG00J4bxVV3r9uUzfo24wW0knS07SKSw== js-string-escape@^1.0.1: version "1.0.1" @@ -17232,13 +17112,6 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" -kind-of@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-2.0.1.tgz#018ec7a4ce7e3a86cb9141be519d24c8faa981b5" - integrity sha1-AY7HpM5+OobLkUG+UZ0kyPqpgbU= - dependencies: - is-buffer "^1.0.2" - kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -17346,16 +17219,6 @@ lazy-ass@^1.6.0: resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= -lazy-cache@^0.2.3: - version "0.2.7" - resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-0.2.7.tgz#7feddf2dcb6edb77d11ef1d117ab5ffdf0ab1b65" - integrity sha1-f+3fLctu23fRHvHRF6tf/fCrG2U= - -lazy-cache@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e" - integrity sha1-odePw6UEdMuAhF07O24dpJpEbo4= - lazy-universal-dotenv@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/lazy-universal-dotenv/-/lazy-universal-dotenv-3.0.1.tgz#a6c8938414bca426ab8c9463940da451a911db38" @@ -18243,15 +18106,6 @@ meow@^8.0.0: type-fest "^0.18.0" yargs-parser "^20.2.3" -merge-deep@^3.0.1: - version "3.0.3" - resolved "https://registry.yarnpkg.com/merge-deep/-/merge-deep-3.0.3.tgz#1a2b2ae926da8b2ae93a0ac15d90cd1922766003" - integrity sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA== - dependencies: - arr-union "^3.1.0" - clone-deep "^0.2.4" - kind-of "^3.0.2" - merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -18566,14 +18420,6 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mixin-object@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/mixin-object/-/mixin-object-2.0.1.tgz#4fb949441dab182540f1fe035ba60e1947a5e57e" - integrity sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4= - dependencies: - for-in "^0.1.3" - is-extendable "^0.1.1" - mkdirp-classic@^0.5.2: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -20975,52 +20821,23 @@ puppeteer-core@^15.3.2: unbzip2-stream "1.4.3" ws "8.8.0" -puppeteer-extra-plugin-stealth@^2.9.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.10.0.tgz#583512dcef66af79e5736b02ed6856fd4171b80a" - integrity sha512-Bpotsjr9cqjzn7On/hL9xzzpxLFWza0mRCwhzNEGPdORbjkXDGu7dg+6mgVg6EtSI9lOr8M1bUJBFvJhVpFJzA== +puppeteer-core@^15.4.0: + version "15.4.0" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-15.4.0.tgz#37536c973ea8920181effde47c22c67c36d1db21" + integrity sha512-nUu0aqeOsYnKJwKlHNNCU5cqVsJ+p1EPDzNRITcEV3n1Mz06Ev2DNsb7CTtGd6Sx2rjoseD6zZzEU7XZWocYwQ== dependencies: - debug "^4.1.1" - puppeteer-extra-plugin "^3.2.0" - puppeteer-extra-plugin-user-preferences "^2.3.1" - -puppeteer-extra-plugin-user-data-dir@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.3.1.tgz#1a8777b6383cf212de361497e94616ab29712a38" - integrity sha512-yhaYMaNFdfQ1LbA94ZElW1zU8rh+MFmO+GZA0gtQ8BXc+UZ6aRrWS9flIZvlXDzk+ZsXhCbTEohEwZ8lEDLRVA== - dependencies: - debug "^4.1.1" - fs-extra "^10.0.0" - puppeteer-extra-plugin "^3.2.0" - -puppeteer-extra-plugin-user-preferences@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.3.1.tgz#20faacd9e4cc00a52e8261604309e897aa569fa5" - integrity sha512-t/FyGQj2aqtHOROqL02z+k2kNQe0cjT0Hd9pG5FJ7x0JXx1722PhOuK7FeJLQMJ+BLl2YvCUgaWSC8Zohjts5A== - dependencies: - debug "^4.1.1" - deepmerge "^4.2.2" - puppeteer-extra-plugin "^3.2.0" - puppeteer-extra-plugin-user-data-dir "^2.3.1" - -puppeteer-extra-plugin@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.0.tgz#f964e2a714d0f9c7a00b557c780ac28c6affd5e9" - integrity sha512-wbiw12USE3b+maMk/IMaroYsz7rusVI9G+ml6pCFCnFFh91Z9BAEiVzhCpOHuquVXEiCCsDTWhDUgvdNxQHOyw== - dependencies: - "@types/debug" "^4.1.0" - debug "^4.1.1" - merge-deep "^3.0.1" - -puppeteer-extra@^3.2.3: - version "3.2.3" - resolved "https://registry.yarnpkg.com/puppeteer-extra/-/puppeteer-extra-3.2.3.tgz#1b24ae12ab7c7660f81922c1065beb5887cc189e" - integrity sha512-CnSN9yIedbAbS8WmRybaDHJLf6goRk+VYM/kbH6i/+EMadCaAeh2O+1/mFUMN2LbkbDNAp2Vd/UwrTVCHjTxyg== - dependencies: - "@types/debug" "^4.1.0" - "@types/puppeteer" "*" - debug "^4.1.1" - deepmerge "^4.2.2" + cross-fetch "3.1.5" + debug "4.3.4" + devtools-protocol "0.0.1011705" + extract-zip "2.0.1" + https-proxy-agent "5.0.1" + pkg-dir "4.2.0" + progress "2.0.3" + proxy-from-env "1.1.0" + rimraf "3.0.2" + tar-fs "2.1.1" + unbzip2-stream "1.4.3" + ws "8.8.0" puppeteer@^10.1.0: version "10.4.0" @@ -22424,16 +22241,6 @@ sha.js@^2.4.0, sha.js@^2.4.11, sha.js@^2.4.8: inherits "^2.0.1" safe-buffer "^5.0.1" -shallow-clone@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-0.1.2.tgz#5909e874ba77106d73ac414cfec1ffca87d97060" - integrity sha1-WQnodLp3EG1zrEFM/sH/yofZcGA= - dependencies: - is-extendable "^0.1.1" - kind-of "^2.0.1" - lazy-cache "^0.2.3" - mixin-object "^2.0.1" - shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -23487,6 +23294,17 @@ teeny-request@^7.0.0: stream-events "^1.0.5" uuid "^8.0.0" +teeny-request@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-8.0.0.tgz#9614410ba70114fd28ba7bf5077dce3e2f02adf7" + integrity sha512-6KEYxXI4lQPSDkXzXpPmJPNmo7oqduFFbhOEHf8sfsLbXyCsb+umUjBtMGAKhaSToD8JNCtQutTRefu29K64JA== + dependencies: + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.0" + node-fetch "^2.6.1" + stream-events "^1.0.5" + uuid "^8.0.0" + telejson@^5.3.2, telejson@^5.3.3: version "5.3.3" resolved "https://registry.yarnpkg.com/telejson/-/telejson-5.3.3.tgz#fa8ca84543e336576d8734123876a9f02bf41d2e"