From 9056318667c0431921fe898f60a720815c8e1117 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 14 Feb 2023 16:30:19 +0800 Subject: [PATCH 01/48] Change type field in integration table from enum to text type --- packages/api/src/entity/integration.ts | 8 ++----- .../api/src/resolvers/integrations/index.ts | 21 +++++++++++++++---- packages/api/src/routers/svc/integrations.ts | 4 ++-- packages/api/src/utils/createTask.ts | 3 +-- .../api/test/resolvers/integrations.test.ts | 17 +++++++-------- .../api/test/routers/integrations.test.ts | 6 +++--- packages/api/tsconfig.json | 2 +- .../0111.do.change_type_in_integration.sql | 9 ++++++++ .../0111.undo.change_type_in_integration.sql | 9 ++++++++ 9 files changed, 51 insertions(+), 28 deletions(-) create mode 100755 packages/db/migrations/0111.do.change_type_in_integration.sql create mode 100755 packages/db/migrations/0111.undo.change_type_in_integration.sql diff --git a/packages/api/src/entity/integration.ts b/packages/api/src/entity/integration.ts index 0d1895f64..2aba8d7fb 100644 --- a/packages/api/src/entity/integration.ts +++ b/packages/api/src/entity/integration.ts @@ -9,10 +9,6 @@ import { } from 'typeorm' import { User } from './user' -export enum IntegrationType { - Readwise = 'READWISE', -} - @Entity({ name: 'integrations' }) export class Integration { @PrimaryGeneratedColumn('uuid') @@ -22,8 +18,8 @@ export class Integration { @JoinColumn({ name: 'user_id' }) user!: User - @Column('enum', { enum: IntegrationType }) - type!: IntegrationType + @Column('text') + type!: string @Column('varchar', { length: 255 }) token!: string diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 8e8519d65..e5a380c68 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -6,6 +6,7 @@ import { IntegrationsError, IntegrationsErrorCode, IntegrationsSuccess, + IntegrationType, MutationDeleteIntegrationArgs, MutationSetIntegrationArgs, SetIntegrationError, @@ -94,7 +95,10 @@ export const setIntegrationResolver = authorized< if (!integrationToSave.id || integrationToSave.enabled) { // create a task to sync all the pages if new integration or enable integration - const taskName = await enqueueSyncWithIntegration(user.id, input.type) + const taskName = await enqueueSyncWithIntegration( + user.id, + input.type as string + ) log.info('enqueued task', taskName) // update task name in integration @@ -122,7 +126,10 @@ export const setIntegrationResolver = authorized< }) return { - integration, + integration: { + ...integration, + type: integration.type as IntegrationType, + }, } } catch (error) { log.error(error) @@ -151,7 +158,10 @@ export const integrationsResolver = authorized< }) return { - integrations, + integrations: integrations.map((integration) => ({ + ...integration, + type: integration.type as IntegrationType, + })), } } catch (error) { log.error(error) @@ -215,7 +225,10 @@ export const deleteIntegrationResolver = authorized< }) return { - integration: deletedIntegration, + integration: { + ...deletedIntegration, + type: deletedIntegration.type as IntegrationType, + }, } } catch (error) { log.error(error) diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 03ccea762..bffa55c94 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -54,7 +54,7 @@ export function integrationsServiceRouter() { const integration = await getRepository(Integration).findOneBy({ user: { id: userId }, - type: req.params.integrationType.toUpperCase() as IntegrationType, + type: req.params.integrationType.toUpperCase(), enabled: true, }) if (!integration) { @@ -125,7 +125,7 @@ export function integrationsServiceRouter() { ;[pages, count] = (await searchPages( { from: after, size, dateFilters }, userId - ))! + )) as [Page[], number] const pageIds = pages.map((p) => p.id) logger.info('syncing pages', { pageIds }) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index e8b15eccb..6ff8b5e28 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -8,7 +8,6 @@ import { CreateTaskError } from './errors' import { buildLogger } from './logger' import { nanoid } from 'nanoid' import { google } from '@google-cloud/tasks/build/protos/protos' -import { IntegrationType } from '../entity/integration' import { signFeatureToken } from '../services/features' import { Recommendation } from '../elastic/types' import View = google.cloud.tasks.v2.Task.View @@ -280,7 +279,7 @@ export const enqueueReminder = async ( export const enqueueSyncWithIntegration = async ( userId: string, - integrationType: IntegrationType + integrationType: string ): Promise => { const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env // use pubsub data format to send the userId to the task handler diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 376b5c9d0..5e5b54f46 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -8,10 +8,7 @@ import { } from '../../src/generated/graphql' import { expect } from 'chai' import { getRepository } from '../../src/entity/utils' -import { - Integration, - IntegrationType as DataIntegrationType, -} from '../../src/entity/integration' +import { Integration } from '../../src/entity/integration' import nock from 'nock' import { READWISE_API_URL } from '../../src/services/integrations' @@ -91,10 +88,10 @@ describe('Integrations resolvers', () => { before(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: DataIntegrationType.Readwise, + type: 'READWISE', token: 'fakeToken', }) - integrationType = existingIntegration.type + integrationType = existingIntegration.type as IntegrationType }) after(async () => { @@ -190,7 +187,7 @@ describe('Integrations resolvers', () => { otherUser = await createTestUser('otherUser') existingIntegration = await getRepository(Integration).save({ user: { id: otherUser.id }, - type: DataIntegrationType.Readwise, + type: 'READWISE', token: 'fakeToken', }) integrationId = existingIntegration.id @@ -216,7 +213,7 @@ describe('Integrations resolvers', () => { before(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: DataIntegrationType.Readwise, + type: 'READWISE', token: 'fakeToken', }) integrationId = existingIntegration.id @@ -316,7 +313,7 @@ describe('Integrations resolvers', () => { before(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: DataIntegrationType.Readwise, + type: 'READWISE', token: 'fakeToken', }) }) @@ -362,7 +359,7 @@ describe('Integrations resolvers', () => { beforeEach(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: DataIntegrationType.Readwise, + type: 'READWISE', token: 'fakeToken', taskName: 'some task name', }) diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index eb3ef2360..d5cf2507f 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -8,7 +8,7 @@ import { } from '../../src/datalayer/pubsub' import { User } from '../../src/entity/user' import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' -import { Integration, IntegrationType } from '../../src/entity/integration' +import { Integration } from '../../src/entity/integration' import { getRepository } from '../../src/entity/utils' import { Highlight, @@ -95,7 +95,7 @@ describe('Integrations routers', () => { context('when integration not found', () => { before(() => { - integrationType = IntegrationType.Readwise + integrationType = 'READWISE' data = { message: { data: Buffer.from( @@ -125,7 +125,7 @@ describe('Integrations routers', () => { before(async () => { integration = await getRepository(Integration).save({ user: { id: user.id }, - type: IntegrationType.Readwise, + type: 'READWISE', token: 'token', }) integrationType = integration.type diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index 7c8caecfd..ee1fb782a 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -7,5 +7,5 @@ "outDir": "dist" }, "include": ["src", "test"], - "exclude": ["./src/generated", "./test"] + "exclude": ["./src/generated"] } diff --git a/packages/db/migrations/0111.do.change_type_in_integration.sql b/packages/db/migrations/0111.do.change_type_in_integration.sql new file mode 100755 index 000000000..ecdac43f6 --- /dev/null +++ b/packages/db/migrations/0111.do.change_type_in_integration.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: change_type_in_integration +-- Description: Change type field in integration table + +BEGIN; + +ALTER TABLE omnivore.integrations ALTER COLUMN "type" TYPE text; + +COMMIT; diff --git a/packages/db/migrations/0111.undo.change_type_in_integration.sql b/packages/db/migrations/0111.undo.change_type_in_integration.sql new file mode 100755 index 000000000..0cd785f81 --- /dev/null +++ b/packages/db/migrations/0111.undo.change_type_in_integration.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: change_type_in_integration +-- Description: Change type field in integration table + +BEGIN; + +ALTER TABLE omnivore.integrations ALTER COLUMN "type" TYPE omnivore.integration_type USING "type"::omnivore.integration_type; + +COMMIT; From 4ce4cd0a62d98dac55e344eb89c86b67d7330d54 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Feb 2023 12:17:26 +0800 Subject: [PATCH 02/48] Add a type for export or import to the integration table --- packages/api/src/entity/integration.ts | 15 ++++- packages/api/src/generated/graphql.ts | 56 ++++++++++++++++++- packages/api/src/generated/schema.graphql | 23 +++++++- .../api/src/resolvers/integrations/index.ts | 32 ++--------- packages/api/src/routers/svc/integrations.ts | 7 ++- packages/api/src/schema.ts | 25 ++++++++- packages/api/src/services/integrations.ts | 11 ++-- .../api/test/resolvers/integrations.test.ts | 45 +++++++-------- .../api/test/routers/integrations.test.ts | 24 ++++---- .../0111.do.change_type_in_integration.sql | 9 ++- .../0111.undo.change_type_in_integration.sql | 8 ++- 11 files changed, 174 insertions(+), 81 deletions(-) diff --git a/packages/api/src/entity/integration.ts b/packages/api/src/entity/integration.ts index 2aba8d7fb..a051ac0db 100644 --- a/packages/api/src/entity/integration.ts +++ b/packages/api/src/entity/integration.ts @@ -9,6 +9,11 @@ import { } from 'typeorm' import { User } from './user' +export enum IntegrationType { + Export = 'EXPORT', + Import = 'IMPORT', +} + @Entity({ name: 'integrations' }) export class Integration { @PrimaryGeneratedColumn('uuid') @@ -18,8 +23,14 @@ export class Integration { @JoinColumn({ name: 'user_id' }) user!: User - @Column('text') - type!: string + @Column('varchar', { length: 40 }) + name!: string + + @Column('enum', { + enum: IntegrationType, + default: IntegrationType.Export, + }) + type!: IntegrationType @Column('varchar', { length: 255 }) token!: string diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 22d51b8b4..656149b99 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -940,18 +940,37 @@ export enum HighlightType { Redaction = 'REDACTION' } +export type ImportFromIntegrationError = { + __typename?: 'ImportFromIntegrationError'; + errorCodes: Array; +}; + +export enum ImportFromIntegrationErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess; + +export type ImportFromIntegrationSuccess = { + __typename?: 'ImportFromIntegrationSuccess'; + success: Scalars['Boolean']; +}; + export type Integration = { __typename?: 'Integration'; createdAt: Scalars['Date']; enabled: Scalars['Boolean']; id: Scalars['ID']; + name: Scalars['String']; token: Scalars['String']; type: IntegrationType; updatedAt: Scalars['Date']; }; export enum IntegrationType { - Readwise = 'READWISE' + Export = 'EXPORT', + Import = 'IMPORT' } export type IntegrationsError = { @@ -1223,6 +1242,7 @@ export type Mutation = { generateApiKey: GenerateApiKeyResult; googleLogin: LoginResult; googleSignup: GoogleSignupResult; + importFromIntegration: ImportFromIntegrationResult; joinGroup: JoinGroupResult; leaveGroup: LeaveGroupResult; logOut: LogOutResult; @@ -1389,6 +1409,11 @@ export type MutationGoogleSignupArgs = { }; +export type MutationImportFromIntegrationArgs = { + integrationId: Scalars['ID']; +}; + + export type MutationJoinGroupArgs = { inviteCode: Scalars['String']; }; @@ -2387,8 +2412,9 @@ export enum SetIntegrationErrorCode { export type SetIntegrationInput = { enabled: Scalars['Boolean']; id?: InputMaybe; + name: Scalars['String']; token: Scalars['String']; - type: IntegrationType; + type?: InputMaybe; }; export type SetIntegrationResult = SetIntegrationError | SetIntegrationSuccess; @@ -3398,6 +3424,10 @@ export type ResolversTypes = { HighlightStats: ResolverTypeWrapper; HighlightType: HighlightType; ID: ResolverTypeWrapper; + ImportFromIntegrationError: ResolverTypeWrapper; + ImportFromIntegrationErrorCode: ImportFromIntegrationErrorCode; + ImportFromIntegrationResult: ResolversTypes['ImportFromIntegrationError'] | ResolversTypes['ImportFromIntegrationSuccess']; + ImportFromIntegrationSuccess: ResolverTypeWrapper; Int: ResolverTypeWrapper; Integration: ResolverTypeWrapper; IntegrationType: IntegrationType; @@ -3842,6 +3872,9 @@ export type ResolversParentTypes = { HighlightReply: HighlightReply; HighlightStats: HighlightStats; ID: Scalars['ID']; + ImportFromIntegrationError: ImportFromIntegrationError; + ImportFromIntegrationResult: ResolversParentTypes['ImportFromIntegrationError'] | ResolversParentTypes['ImportFromIntegrationSuccess']; + ImportFromIntegrationSuccess: ImportFromIntegrationSuccess; Int: Scalars['Int']; Integration: Integration; IntegrationsError: IntegrationsError; @@ -4764,10 +4797,25 @@ export type HighlightStatsResolvers; }; +export type ImportFromIntegrationErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type ImportFromIntegrationResultResolvers = { + __resolveType: TypeResolveFn<'ImportFromIntegrationError' | 'ImportFromIntegrationSuccess', ParentType, ContextType>; +}; + +export type ImportFromIntegrationSuccessResolvers = { + success?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type IntegrationResolvers = { createdAt?: Resolver; enabled?: Resolver; id?: Resolver; + name?: Resolver; token?: Resolver; type?: Resolver; updatedAt?: Resolver; @@ -4975,6 +5023,7 @@ export type MutationResolvers>; googleLogin?: Resolver>; googleSignup?: Resolver>; + importFromIntegration?: Resolver>; joinGroup?: Resolver>; leaveGroup?: Resolver>; logOut?: Resolver; @@ -6086,6 +6135,9 @@ export type Resolvers = { Highlight?: HighlightResolvers; HighlightReply?: HighlightReplyResolvers; HighlightStats?: HighlightStatsResolvers; + ImportFromIntegrationError?: ImportFromIntegrationErrorResolvers; + ImportFromIntegrationResult?: ImportFromIntegrationResultResolvers; + ImportFromIntegrationSuccess?: ImportFromIntegrationSuccessResolvers; Integration?: IntegrationResolvers; IntegrationsError?: IntegrationsErrorResolvers; IntegrationsResult?: IntegrationsResultResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 247ff2eda..651eafbff 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -835,17 +835,34 @@ enum HighlightType { REDACTION } +type ImportFromIntegrationError { + errorCodes: [ImportFromIntegrationErrorCode!]! +} + +enum ImportFromIntegrationErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess + +type ImportFromIntegrationSuccess { + success: Boolean! +} + type Integration { createdAt: Date! enabled: Boolean! id: ID! + name: String! token: String! type: IntegrationType! updatedAt: Date! } enum IntegrationType { - READWISE + EXPORT + IMPORT } type IntegrationsError { @@ -1093,6 +1110,7 @@ type Mutation { generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! googleLogin(input: GoogleLoginInput!): LoginResult! googleSignup(input: GoogleSignupInput!): GoogleSignupResult! + importFromIntegration(integrationId: ID!): ImportFromIntegrationResult! joinGroup(inviteCode: String!): JoinGroupResult! leaveGroup(groupId: ID!): LeaveGroupResult! logOut: LogOutResult! @@ -1775,8 +1793,9 @@ enum SetIntegrationErrorCode { input SetIntegrationInput { enabled: Boolean! id: ID + name: String! token: String! - type: IntegrationType! + type: IntegrationType } union SetIntegrationResult = SetIntegrationError | SetIntegrationSuccess diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index e5a380c68..ab731b3f5 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -6,7 +6,6 @@ import { IntegrationsError, IntegrationsErrorCode, IntegrationsSuccess, - IntegrationType, MutationDeleteIntegrationArgs, MutationSetIntegrationArgs, SetIntegrationError, @@ -65,27 +64,17 @@ export const setIntegrationResolver = authorized< } } else { // Create - const existingIntegration = await getRepository(Integration).findOneBy({ - user: { id: uid }, - type: input.type, - }) - - if (existingIntegration) { - return { - errorCodes: [SetIntegrationErrorCode.AlreadyExists], - } - } - // validate token - if (!(await validateToken(input.token, input.type))) { + if (!(await validateToken(input.token, input.name))) { return { errorCodes: [SetIntegrationErrorCode.InvalidToken], } } integrationToSave = { ...integrationToSave, + name: input.name, + type: input.type ?? undefined, token: input.token, - type: input.type, enabled: true, } } @@ -126,10 +115,7 @@ export const setIntegrationResolver = authorized< }) return { - integration: { - ...integration, - type: integration.type as IntegrationType, - }, + integration, } } catch (error) { log.error(error) @@ -158,10 +144,7 @@ export const integrationsResolver = authorized< }) return { - integrations: integrations.map((integration) => ({ - ...integration, - type: integration.type as IntegrationType, - })), + integrations, } } catch (error) { log.error(error) @@ -225,10 +208,7 @@ export const deleteIntegrationResolver = authorized< }) return { - integration: { - ...deletedIntegration, - type: deletedIntegration.type as IntegrationType, - }, + integration, } } catch (error) { log.error(error) diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index bffa55c94..b9656ab9b 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -24,10 +24,10 @@ const logger = buildLogger('app.dispatch') export function integrationsServiceRouter() { const router = express.Router() - router.post('/:integrationType/:action', async (req, res) => { + router.post('/:integrationName/:action', async (req, res) => { logger.info('start to sync with integration', { action: req.params.action, - integrationType: req.params.integrationType, + integrationName: req.params.integrationName, }) const { message: msgStr, expired } = readPushSubscription(req) @@ -54,7 +54,8 @@ export function integrationsServiceRouter() { const integration = await getRepository(Integration).findOneBy({ user: { id: userId }, - type: req.params.integrationType.toUpperCase(), + name: req.params.integrationName.toUpperCase(), + type: IntegrationType.Export, enabled: true, }) if (!integration) { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 3beeaff93..7751f0c33 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1903,6 +1903,7 @@ const schema = gql` type Integration { id: ID! + name: String! type: IntegrationType! token: String! enabled: Boolean! @@ -1911,7 +1912,8 @@ const schema = gql` } enum IntegrationType { - READWISE + EXPORT + IMPORT } type SetIntegrationError { @@ -1928,7 +1930,8 @@ const schema = gql` input SetIntegrationInput { id: ID - type: IntegrationType! + name: String! + type: IntegrationType token: String! enabled: Boolean! } @@ -2416,6 +2419,23 @@ const schema = gql` UNAUTHORIZED } + union ImportFromIntegrationResult = + ImportFromIntegrationSuccess + | ImportFromIntegrationError + + type ImportFromIntegrationSuccess { + success: Boolean! + } + + type ImportFromIntegrationError { + errorCodes: [ImportFromIntegrationErrorCode!]! + } + + enum ImportFromIntegrationErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2506,6 +2526,7 @@ const schema = gql` ): UploadImportFileResult! markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult! bulkAction(query: String, action: BulkActionType!): BulkActionResult! + importFromIntegration(integrationId: ID!): ImportFromIntegrationResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/integrations.ts b/packages/api/src/services/integrations.ts index 91d0df654..e6a5fa8d2 100644 --- a/packages/api/src/services/integrations.ts +++ b/packages/api/src/services/integrations.ts @@ -1,4 +1,3 @@ -import { IntegrationType } from '../generated/graphql' import { env } from '../env' import axios from 'axios' import { wait } from '../utils/helpers' @@ -38,10 +37,10 @@ export const READWISE_API_URL = 'https://readwise.io/api/v2' export const validateToken = async ( token: string, - type: IntegrationType + name: string ): Promise => { - switch (type) { - case IntegrationType.Readwise: + switch (name) { + case 'READWISE': return validateReadwiseToken(token) default: return false @@ -97,8 +96,8 @@ export const syncWithIntegration = async ( pages: Page[] ): Promise => { let result = true - switch (integration.type) { - case IntegrationType.Readwise: { + switch (integration.name) { + case 'READWISE': { const highlights = pages.flatMap(pageToReadwiseHighlight) // If there are no highlights, we will skip the sync if (highlights.length > 0) { diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 5e5b54f46..c3a8d4b44 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -2,10 +2,7 @@ import 'mocha' import { User } from '../../src/entity/user' import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' import { generateFakeUuid, graphqlRequest, request } from '../util' -import { - IntegrationType, - SetIntegrationErrorCode, -} from '../../src/generated/graphql' +import { SetIntegrationErrorCode } from '../../src/generated/graphql' import { expect } from 'chai' import { getRepository } from '../../src/entity/utils' import { Integration } from '../../src/entity/integration' @@ -34,14 +31,14 @@ describe('Integrations resolvers', () => { const validToken = 'valid-token' const query = ( id = '', - type: IntegrationType = IntegrationType.Readwise, + name = 'READWISE', token: string = 'test token', enabled = true ) => ` mutation { setIntegration(input: { id: "${id}", - type: ${type}, + name: ${name}, token: "${token}", enabled: ${enabled}, }) { @@ -59,7 +56,7 @@ describe('Integrations resolvers', () => { ` let integrationId: string let token: string - let integrationType: IntegrationType + let integrationName: string let enabled: boolean let scope: nock.Scope @@ -88,10 +85,10 @@ describe('Integrations resolvers', () => { before(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: 'READWISE', + name: 'READWISE', token: 'fakeToken', }) - integrationType = existingIntegration.type as IntegrationType + integrationName = existingIntegration.name }) after(async () => { @@ -100,7 +97,7 @@ describe('Integrations resolvers', () => { it('returns AlreadyExists error code', async () => { const res = await graphqlRequest( - query(integrationId, integrationType), + query(integrationId, integrationName), authToken ) expect(res.body.data.setIntegration.errorCodes).to.eql([ @@ -117,7 +114,7 @@ describe('Integrations resolvers', () => { it('returns InvalidToken error code', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token), + query(integrationId, integrationName, token), authToken ) expect(res.body.data.setIntegration.errorCodes).to.eql([ @@ -134,13 +131,13 @@ describe('Integrations resolvers', () => { afterEach(async () => { await deleteTestIntegrations(loginUser.id, { user: { id: loginUser.id }, - type: integrationType, + name: integrationName, }) }) it('creates new integration', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token), + query(integrationId, integrationName, token), authToken ) expect(res.body.data.setIntegration.integration.enabled).to.be.true @@ -148,7 +145,7 @@ describe('Integrations resolvers', () => { it('creates new cloud task to sync all existing articles and highlights', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token), + query(integrationId, integrationName, token), authToken ) const integration = await getRepository(Integration).findOneBy({ @@ -170,7 +167,7 @@ describe('Integrations resolvers', () => { it('returns NotFound error code', async () => { const res = await graphqlRequest( - query(integrationId, integrationType), + query(integrationId, integrationName), authToken ) expect(res.body.data.setIntegration.errorCodes).to.eql([ @@ -187,7 +184,7 @@ describe('Integrations resolvers', () => { otherUser = await createTestUser('otherUser') existingIntegration = await getRepository(Integration).save({ user: { id: otherUser.id }, - type: 'READWISE', + name: 'READWISE', token: 'fakeToken', }) integrationId = existingIntegration.id @@ -200,7 +197,7 @@ describe('Integrations resolvers', () => { it('returns Unauthorized error code', async () => { const res = await graphqlRequest( - query(integrationId, integrationType), + query(integrationId, integrationName), authToken ) expect(res.body.data.setIntegration.errorCodes).to.eql([ @@ -213,7 +210,7 @@ describe('Integrations resolvers', () => { before(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: 'READWISE', + name: 'READWISE', token: 'fakeToken', }) integrationId = existingIntegration.id @@ -237,7 +234,7 @@ describe('Integrations resolvers', () => { it('disables integration', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token, enabled), + query(integrationId, integrationName, token, enabled), authToken ) expect(res.body.data.setIntegration.integration.enabled).to.be @@ -246,7 +243,7 @@ describe('Integrations resolvers', () => { it('deletes cloud task', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token, enabled), + query(integrationId, integrationName, token, enabled), authToken ) const integration = await getRepository(Integration).findOneBy({ @@ -270,7 +267,7 @@ describe('Integrations resolvers', () => { it('enables integration', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token, enabled), + query(integrationId, integrationName, token, enabled), authToken ) expect(res.body.data.setIntegration.integration.enabled).to.be @@ -279,7 +276,7 @@ describe('Integrations resolvers', () => { it('creates new cloud task to sync all existing articles and highlights', async () => { const res = await graphqlRequest( - query(integrationId, integrationType, token, enabled), + query(integrationId, integrationName, token, enabled), authToken ) const integration = await getRepository(Integration).findOneBy({ @@ -313,7 +310,7 @@ describe('Integrations resolvers', () => { before(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: 'READWISE', + name: 'READWISE', token: 'fakeToken', }) }) @@ -359,7 +356,7 @@ describe('Integrations resolvers', () => { beforeEach(async () => { existingIntegration = await getRepository(Integration).save({ user: { id: loginUser.id }, - type: 'READWISE', + name: 'READWISE', token: 'fakeToken', taskName: 'some task name', }) diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index d5cf2507f..f6bb67d30 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -26,11 +26,11 @@ describe('Integrations routers', () => { let token: string describe('sync with integrations', () => { - const endpoint = (token: string, type = 'type', action = 'action') => - `/svc/pubsub/integrations/${type}/${action}?token=${token}` + const endpoint = (token: string, name = 'name', action = 'action') => + `/svc/pubsub/integrations/${name}/${action}?token=${token}` let action: string let data: PubSubRequestBody - let integrationType: string + let integrationName: string context('when token is invalid', () => { before(() => { @@ -95,7 +95,7 @@ describe('Integrations routers', () => { context('when integration not found', () => { before(() => { - integrationType = 'READWISE' + integrationName = 'READWISE' data = { message: { data: Buffer.from( @@ -108,7 +108,7 @@ describe('Integrations routers', () => { it('returns 200 with No integration found', async () => { const res = await request - .post(endpoint(token, integrationType)) + .post(endpoint(token, integrationName)) .send(data) .expect(200) expect(res.text).to.eql('No integration found') @@ -125,10 +125,10 @@ describe('Integrations routers', () => { before(async () => { integration = await getRepository(Integration).save({ user: { id: user.id }, - type: 'READWISE', + name: 'READWISE', token: 'token', }) - integrationType = integration.type + integrationName = integration.name // create page page = await createTestElasticPage(user.id) ctx = { @@ -177,7 +177,7 @@ describe('Integrations routers', () => { }) context('when action is sync_updated', () => { - before(async () => { + before(() => { action = 'sync_updated' }) @@ -208,7 +208,7 @@ describe('Integrations routers', () => { it('returns 200 with OK', async () => { const res = await request - .post(endpoint(token, integrationType, action)) + .post(endpoint(token, integrationName, action)) .send(data) .expect(200) expect(res.text).to.eql('OK') @@ -240,7 +240,7 @@ describe('Integrations routers', () => { it('returns 200 with OK', async () => { const res = await request - .post(endpoint(token, integrationType, action)) + .post(endpoint(token, integrationName, action)) .send(data) .expect(200) expect(res.text).to.eql('OK') @@ -275,7 +275,7 @@ describe('Integrations routers', () => { it('returns 200 with OK', async () => { const res = await request - .post(endpoint(token, integrationType, action)) + .post(endpoint(token, integrationName, action)) .send(data) .expect(200) expect(res.text).to.eql('OK') @@ -313,7 +313,7 @@ describe('Integrations routers', () => { it('returns 200 with OK', async () => { const res = await request - .post(endpoint(token, integrationType, action)) + .post(endpoint(token, integrationName, action)) .send(data) .expect(200) expect(res.text).to.eql('OK') diff --git a/packages/db/migrations/0111.do.change_type_in_integration.sql b/packages/db/migrations/0111.do.change_type_in_integration.sql index ecdac43f6..ebaee00a7 100755 --- a/packages/db/migrations/0111.do.change_type_in_integration.sql +++ b/packages/db/migrations/0111.do.change_type_in_integration.sql @@ -4,6 +4,13 @@ BEGIN; -ALTER TABLE omnivore.integrations ALTER COLUMN "type" TYPE text; +ALTER TABLE omnivore.integrations RENAME COLUMN "type" TO "name"; +ALTER TABLE omnivore.integrations + DROP CONSTRAINT integrations_user_id_type_key, + ALTER COLUMN "name" TYPE VARCHAR(40) USING "name"::VARCHAR(40); +DROP TYPE omnivore.integration_type; +CREATE TYPE omnivore.integration_type AS ENUM ('EXPORT', 'IMPORT'); +ALTER TABLE omnivore.integrations + ADD COLUMN "type" omnivore.integration_type NOT NULL DEFAULT 'EXPORT'; COMMIT; diff --git a/packages/db/migrations/0111.undo.change_type_in_integration.sql b/packages/db/migrations/0111.undo.change_type_in_integration.sql index 0cd785f81..4b36d4b5b 100755 --- a/packages/db/migrations/0111.undo.change_type_in_integration.sql +++ b/packages/db/migrations/0111.undo.change_type_in_integration.sql @@ -4,6 +4,12 @@ BEGIN; -ALTER TABLE omnivore.integrations ALTER COLUMN "type" TYPE omnivore.integration_type USING "type"::omnivore.integration_type; +ALTER TABLE omnivore.integrations DROP COLUMN "type"; +DROP TYPE omnivore.integration_type; +CREATE TYPE omnivore.integration_type AS ENUM ('READWISE', 'POCKET'); +ALTER TABLE omnivore.integrations + ALTER COLUMN "name" TYPE omnivore.integration_type USING "name"::omnivore.integration_type, + ADD CONSTRAINT integrations_user_id_type_key UNIQUE (user_id, "name"); +ALTER TABLE omnivore.integrations RENAME COLUMN "name" TO "type"; COMMIT; From 0bf4119c92e98f426d171d4e9cfc29a257fc3ab6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Feb 2023 23:47:08 +0800 Subject: [PATCH 03/48] Refactor with factor pattern --- .../api/src/resolvers/integrations/index.ts | 5 +- packages/api/src/routers/svc/integrations.ts | 8 +- packages/api/src/services/integrations.ts | 169 ------------------ .../api/src/services/integrations/index.ts | 12 ++ .../src/services/integrations/integration.ts | 22 +++ .../api/src/services/integrations/pocket.ts | 5 + .../api/src/services/integrations/readwise.ts | 134 ++++++++++++++ packages/api/src/utils/createTask.ts | 4 +- .../api/test/resolvers/integrations.test.ts | 94 ++++------ .../api/test/routers/integrations.test.ts | 2 +- 10 files changed, 218 insertions(+), 237 deletions(-) delete mode 100644 packages/api/src/services/integrations.ts create mode 100644 packages/api/src/services/integrations/index.ts create mode 100644 packages/api/src/services/integrations/integration.ts create mode 100644 packages/api/src/services/integrations/pocket.ts create mode 100644 packages/api/src/services/integrations/readwise.ts diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index ab731b3f5..afc5e59d2 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -17,7 +17,7 @@ import { User } from '../../entity/user' import { Integration } from '../../entity/integration' import { analytics } from '../../utils/analytics' import { env } from '../../env' -import { validateToken } from '../../services/integrations' +import { getIntegrationService } from '../../services/integrations' import { deleteTask, enqueueSyncWithIntegration } from '../../utils/createTask' export const setIntegrationResolver = authorized< @@ -64,8 +64,9 @@ export const setIntegrationResolver = authorized< } } else { // Create + const integrationService = getIntegrationService(input.name) // validate token - if (!(await validateToken(input.token, input.name))) { + if (!(await integrationService.validateToken(input.token))) { return { errorCodes: [SetIntegrationErrorCode.InvalidToken], } diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index b9656ab9b..60aa8ff10 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -65,6 +65,7 @@ export function integrationsServiceRouter() { } const action = req.params.action.toUpperCase() + const integrationService = getIntegrationService(integration.name) if (action === 'SYNC_UPDATED') { // get updated page by id let id: string | undefined @@ -100,7 +101,7 @@ export function integrationsServiceRouter() { pageId: page.id, }) - const synced = await syncWithIntegration(integration, [page]) + const synced = await integrationService.exportPages(integration, [page]) if (!synced) { logger.info('failed to sync page', { integrationId: integration.id, @@ -131,7 +132,10 @@ export function integrationsServiceRouter() { logger.info('syncing pages', { pageIds }) - const synced = await syncWithIntegration(integration, pages) + const synced = await integrationService.exportPages( + integration, + pages + ) if (!synced) { logger.info('failed to sync pages', { pageIds, diff --git a/packages/api/src/services/integrations.ts b/packages/api/src/services/integrations.ts deleted file mode 100644 index e6a5fa8d2..000000000 --- a/packages/api/src/services/integrations.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { env } from '../env' -import axios from 'axios' -import { wait } from '../utils/helpers' -import { HighlightType, Page } from '../elastic/types' -import { getHighlightUrl } from './highlights' -import { Integration } from '../entity/integration' -import { getRepository } from '../entity/utils' - -interface ReadwiseHighlight { - // The highlight text, (technically the only field required in a highlight object) - text: string - // The title of the page the highlight is on - title?: string - // The author of the page the highlight is on - author?: string - // The URL of the page image - image_url?: string - // The URL of the page - source_url?: string - // A meaningful unique identifier for your app - source_type?: string - // One of: books, articles, tweets or podcasts - category?: string - // Annotation note attached to the specific highlight - note?: string - // Highlight's location in the source text. Used to order the highlights - location?: number - // One of: page, order or time_offset - location_type?: string - // A datetime representing when the highlight was taken in the ISO 8601 format - highlighted_at?: string - // Unique url of the specific highlight - highlight_url?: string -} - -export const READWISE_API_URL = 'https://readwise.io/api/v2' - -export const validateToken = async ( - token: string, - name: string -): Promise => { - switch (name) { - case 'READWISE': - return validateReadwiseToken(token) - default: - return false - } -} - -const validateReadwiseToken = async (token: string): Promise => { - const authUrl = `${env.readwise.apiUrl || READWISE_API_URL}/auth` - try { - const response = await axios.get(authUrl, { - headers: { - Authorization: `Token ${token}`, - }, - }) - return response.status === 204 - } catch (error) { - console.log('error validating readwise token', error) - return false - } -} - -const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => { - if (!page.highlights) return [] - const category = page.siteName === 'Twitter' ? 'tweets' : 'articles' - return ( - page.highlights - // filter out highlights with no quote and are not of type Highlight - .filter( - (highlight) => - highlight.type === HighlightType.Highlight && highlight.quote - ) - .map((highlight) => { - return { - text: highlight.quote!, - title: page.title, - author: page.author || undefined, - highlight_url: getHighlightUrl(page.slug, highlight.id), - highlighted_at: new Date(highlight.createdAt).toISOString(), - category, - image_url: page.image || undefined, - // location: highlight.highlightPositionAnchorIndex || undefined, - location_type: 'order', - note: highlight.annotation || undefined, - source_type: 'omnivore', - source_url: page.url, - } - }) - ) -} - -export const syncWithIntegration = async ( - integration: Integration, - pages: Page[] -): Promise => { - let result = true - switch (integration.name) { - case 'READWISE': { - const highlights = pages.flatMap(pageToReadwiseHighlight) - // If there are no highlights, we will skip the sync - if (highlights.length > 0) { - result = await syncWithReadwise(integration.token, highlights) - } - break - } - default: - return false - } - // update integration syncedAt if successful - if (result) { - console.log('updating integration syncedAt') - await getRepository(Integration).update(integration.id, { - syncedAt: new Date(), - }) - } - return result -} - -export const syncWithReadwise = async ( - token: string, - highlights: ReadwiseHighlight[], - retryCount = 0 -): Promise => { - const url = `${env.readwise.apiUrl || READWISE_API_URL}/highlights` - try { - const response = await axios.post( - url, - { - highlights, - }, - { - headers: { - Authorization: `Token ${token}`, - ContentType: 'application/json', - }, - } - ) - return response.status === 200 - } catch (error) { - if (axios.isAxiosError(error)) { - if (error.response) { - if (error.response.status === 429 && retryCount < 3) { - console.log('Readwise API rate limit exceeded, retrying...') - // wait for Retry-After seconds in the header if rate limited - // max retry count is 3 - const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds - await wait(parseInt(retryAfter, 10) * 1000) - return syncWithReadwise(token, highlights, retryCount + 1) - } - // The request was made and the server responded with a status code - // that falls out of the range of 2xx - console.error('Readwise error, response data', error.response.data) - } else if (error.request) { - // The request was made but no response was received - // `error.request` is an instance of XMLHttpRequest in the browser and an instance of - // http.ClientRequest in node.js - console.error('Readwise error, request', error.request) - } else { - // Something happened in setting up the request that triggered an Error - console.error('Error', error.message) - } - } else { - console.error('Error syncing with readwise', error) - } - return false - } -} diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts new file mode 100644 index 000000000..966c4584d --- /dev/null +++ b/packages/api/src/services/integrations/index.ts @@ -0,0 +1,12 @@ +import { ReadwiseIntegration } from './readwise' +import { IntegrationService } from './integration' + +const integrations: IntegrationService[] = [new ReadwiseIntegration()] + +export const getIntegrationService = (name: string): IntegrationService => { + const service = integrations.find((s) => s.name === name) + if (!service) { + throw new Error(`Integration service not found: ${name}`) + } + return service +} diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts new file mode 100644 index 000000000..c025c2d31 --- /dev/null +++ b/packages/api/src/services/integrations/integration.ts @@ -0,0 +1,22 @@ +import { Integration } from '../../entity/integration' +import { Page } from '../../elastic/types' + +export abstract class IntegrationService { + abstract name: string + + validateToken = async (token: string): Promise => { + return Promise.resolve(true) + } + exportPages = async ( + integration: Integration, + pages: Page[] + ): Promise => { + return Promise.resolve(true) + } + importPages = async ( + integration: Integration, + pages: Page[] + ): Promise => { + return Promise.resolve(true) + } +} diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts new file mode 100644 index 000000000..ac9399274 --- /dev/null +++ b/packages/api/src/services/integrations/pocket.ts @@ -0,0 +1,5 @@ +import { IntegrationService } from './integration' + +export class PocketIntegration extends IntegrationService { + name = 'POCKET' +} diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts new file mode 100644 index 000000000..16c4e6021 --- /dev/null +++ b/packages/api/src/services/integrations/readwise.ts @@ -0,0 +1,134 @@ +import { env } from '../../env' +import axios from 'axios' +import { Page } from '../../elastic/types' +import { getHighlightUrl } from '../highlights' +import { getRepository } from '../../entity/utils' +import { Integration } from '../../entity/integration' +import { wait } from '../../utils/helpers' +import { IntegrationService } from './integration' + +interface ReadwiseHighlight { + // The highlight text, (technically the only field required in a highlight object) + text: string + // The title of the page the highlight is on + title?: string + // The author of the page the highlight is on + author?: string + // The URL of the page image + image_url?: string + // The URL of the page + source_url?: string + // A meaningful unique identifier for your app + source_type?: string + // One of: books, articles, tweets or podcasts + category?: string + // Annotation note attached to the specific highlight + note?: string + // Highlight's location in the source text. Used to order the highlights + location?: number + // One of: page, order or time_offset + location_type?: string + // A datetime representing when the highlight was taken in the ISO 8601 format + highlighted_at?: string + // Unique url of the specific highlight + highlight_url?: string +} + +export const READWISE_API_URL = 'https://readwise.io/api/v2' + +export class ReadwiseIntegration extends IntegrationService { + name = 'READWISE' + validateToken = async (token: string): Promise => { + const authUrl = `${env.readwise.apiUrl || READWISE_API_URL}/auth` + try { + const response = await axios.get(authUrl, { + headers: { + Authorization: `Token ${token}`, + }, + }) + return response.status === 204 + } catch (error) { + console.log('error validating readwise token', error) + return false + } + } + exportPages = async ( + integration: Integration, + pages: Page[] + ): Promise => { + let result = true + + const highlights = pages.flatMap(this.pageToReadwiseHighlight) + // If there are no highlights, we will skip the sync + if (highlights.length > 0) { + result = await this.syncWithReadwise(integration.token, highlights) + } + + // update integration syncedAt if successful + if (result) { + console.log('updating integration syncedAt') + await getRepository(Integration).update(integration.id, { + syncedAt: new Date(), + }) + } + return result + } + + pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => { + if (!page.highlights) return [] + return page.highlights.map((highlight) => { + return { + text: highlight.quote, + title: page.title, + author: page.author || undefined, + highlight_url: getHighlightUrl(page.slug, highlight.id), + highlighted_at: new Date(highlight.createdAt).toISOString(), + category: 'articles', + image_url: page.image || undefined, + location: highlight.highlightPositionPercent || undefined, + location_type: 'order', + note: highlight.annotation || undefined, + source_type: 'omnivore', + source_url: page.url, + } + }) + } + + syncWithReadwise = async ( + token: string, + highlights: ReadwiseHighlight[], + retryCount = 0 + ): Promise => { + const url = `${env.readwise.apiUrl || READWISE_API_URL}/highlights` + try { + const response = await axios.post( + url, + { + highlights, + }, + { + headers: { + Authorization: `Token ${token}`, + ContentType: 'application/json', + }, + } + ) + return response.status === 200 + } catch (error) { + if ( + axios.isAxiosError(error) && + error.response?.status === 429 && + retryCount < 3 + ) { + console.log('Readwise API rate limit exceeded, retrying...') + // wait for Retry-After seconds in the header if rate limited + // max retry count is 3 + const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds + await wait(parseInt(retryAfter, 10) * 1000) + return this.syncWithReadwise(token, highlights, retryCount + 1) + } + console.log('Error creating highlights in Readwise', error) + return false + } + } +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 6ff8b5e28..d83f7b7c8 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -279,7 +279,7 @@ export const enqueueReminder = async ( export const enqueueSyncWithIntegration = async ( userId: string, - integrationType: string + integrationName: string ): Promise => { const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env // use pubsub data format to send the userId to the task handler @@ -304,7 +304,7 @@ export const enqueueSyncWithIntegration = async ( payload, taskHandlerUrl: `${ env.queue.integrationTaskHandlerUrl - }/${integrationType.toLowerCase()}/sync_all?token=${PUBSUB_VERIFICATION_TOKEN}`, + }/${integrationName.toLowerCase()}/sync_all?token=${PUBSUB_VERIFICATION_TOKEN}`, priority: 'low', }) diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index c3a8d4b44..91d2d08ca 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -7,7 +7,7 @@ import { expect } from 'chai' import { getRepository } from '../../src/entity/utils' import { Integration } from '../../src/entity/integration' import nock from 'nock' -import { READWISE_API_URL } from '../../src/services/integrations' +import { READWISE_API_URL } from '../../src/services/integrations/readwise' describe('Integrations resolvers', () => { let loginUser: User @@ -38,7 +38,7 @@ describe('Integrations resolvers', () => { mutation { setIntegration(input: { id: "${id}", - name: ${name}, + name: "${name}", token: "${token}", enabled: ${enabled}, }) { @@ -68,6 +68,7 @@ describe('Integrations resolvers', () => { .get('/auth') .reply(204) .persist() + integrationName = 'READWISE' }) after(() => { @@ -79,80 +80,51 @@ describe('Integrations resolvers', () => { integrationId = '' }) - context('when integration exists', () => { - let existingIntegration: Integration - - before(async () => { - existingIntegration = await getRepository(Integration).save({ - user: { id: loginUser.id }, - name: 'READWISE', - token: 'fakeToken', - }) - integrationName = existingIntegration.name + context('when token is invalid', () => { + before(() => { + token = 'invalid token' }) - after(async () => { - await deleteTestIntegrations(loginUser.id, [existingIntegration.id]) - }) - - it('returns AlreadyExists error code', async () => { + it('returns InvalidToken error code', async () => { const res = await graphqlRequest( - query(integrationId, integrationName), + query(integrationId, integrationName, token), authToken ) expect(res.body.data.setIntegration.errorCodes).to.eql([ - SetIntegrationErrorCode.AlreadyExists, + SetIntegrationErrorCode.InvalidToken, ]) }) }) - context('when integration does not exist', () => { - context('when token is invalid', () => { - before(() => { - token = 'invalid token' - }) + context('when token is valid', () => { + before(() => { + token = validToken + }) - it('returns InvalidToken error code', async () => { - const res = await graphqlRequest( - query(integrationId, integrationName, token), - authToken - ) - expect(res.body.data.setIntegration.errorCodes).to.eql([ - SetIntegrationErrorCode.InvalidToken, - ]) + afterEach(async () => { + await deleteTestIntegrations(loginUser.id, { + user: { id: loginUser.id }, + name: integrationName, }) }) - context('when token is valid', () => { - before(() => { - token = validToken - }) + it('creates new integration', async () => { + const res = await graphqlRequest( + query(integrationId, integrationName, token), + authToken + ) + expect(res.body.data.setIntegration.integration.enabled).to.be.true + }) - afterEach(async () => { - await deleteTestIntegrations(loginUser.id, { - user: { id: loginUser.id }, - name: integrationName, - }) - }) - - it('creates new integration', async () => { - const res = await graphqlRequest( - query(integrationId, integrationName, token), - authToken - ) - expect(res.body.data.setIntegration.integration.enabled).to.be.true - }) - - it('creates new cloud task to sync all existing articles and highlights', async () => { - const res = await graphqlRequest( - query(integrationId, integrationName, token), - authToken - ) - const integration = await getRepository(Integration).findOneBy({ - id: res.body.data.setIntegration.integration.id, - }) - expect(integration?.taskName).not.to.be.null + it('creates new cloud task to sync all existing articles and highlights', async () => { + const res = await graphqlRequest( + query(integrationId, integrationName, token), + authToken + ) + const integration = await getRepository(Integration).findOneBy({ + id: res.body.data.setIntegration.integration.id, }) + expect(integration?.taskName).not.to.be.null }) }) }) @@ -192,7 +164,7 @@ describe('Integrations resolvers', () => { after(async () => { await deleteTestUser(otherUser.id) - await deleteTestIntegrations(loginUser.id, [existingIntegration.id]) + await deleteTestIntegrations(otherUser.id, [existingIntegration.id]) }) it('returns Unauthorized error code', async () => { diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index f6bb67d30..9ff06905d 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -17,10 +17,10 @@ import { PageContext, } from '../../src/elastic/types' import nock from 'nock' -import { READWISE_API_URL } from '../../src/services/integrations' import { addHighlightToPage } from '../../src/elastic/highlights' import { getHighlightUrl } from '../../src/services/highlights' import { deletePage } from '../../src/elastic/pages' +import { READWISE_API_URL } from '../../src/services/integrations/readwise' describe('Integrations routers', () => { let token: string From 94d8903ec12c9e66493a2fa1239a79c178f552f2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Feb 2023 10:28:07 +0800 Subject: [PATCH 04/48] Add pocket integration support --- .../api/src/resolvers/integrations/index.ts | 9 ++- packages/api/src/routers/svc/integrations.ts | 7 +-- .../src/services/integrations/integration.ts | 9 +-- .../api/src/services/integrations/pocket.ts | 61 +++++++++++++++++++ .../api/src/services/integrations/readwise.ts | 2 +- packages/api/src/util.ts | 10 +++ 6 files changed, 83 insertions(+), 15 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index afc5e59d2..9aa2c8b3e 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -14,7 +14,7 @@ import { } from '../../generated/graphql' import { getRepository } from '../../entity/utils' import { User } from '../../entity/user' -import { Integration } from '../../entity/integration' +import { Integration, IntegrationType } from '../../entity/integration' import { analytics } from '../../utils/analytics' import { env } from '../../env' import { getIntegrationService } from '../../services/integrations' @@ -83,8 +83,11 @@ export const setIntegrationResolver = authorized< // save integration const integration = await getRepository(Integration).save(integrationToSave) - if (!integrationToSave.id || integrationToSave.enabled) { - // create a task to sync all the pages if new integration or enable integration + if ( + integration.type === IntegrationType.Export && + (!integrationToSave.id || integrationToSave.enabled) + ) { + // create a task to sync all the pages if new integration or enable integration (export type) const taskName = await enqueueSyncWithIntegration( user.id, input.type as string diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 60aa8ff10..6f9ed2b8a 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -101,7 +101,7 @@ export function integrationsServiceRouter() { pageId: page.id, }) - const synced = await integrationService.exportPages(integration, [page]) + const synced = await integrationService.export(integration, [page]) if (!synced) { logger.info('failed to sync page', { integrationId: integration.id, @@ -132,10 +132,7 @@ export function integrationsServiceRouter() { logger.info('syncing pages', { pageIds }) - const synced = await integrationService.exportPages( - integration, - pages - ) + const synced = await integrationService.export(integration, pages) if (!synced) { logger.info('failed to sync pages', { pageIds, diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index c025c2d31..aad49a369 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -7,16 +7,13 @@ export abstract class IntegrationService { validateToken = async (token: string): Promise => { return Promise.resolve(true) } - exportPages = async ( + export = async ( integration: Integration, pages: Page[] ): Promise => { return Promise.resolve(true) } - importPages = async ( - integration: Integration, - pages: Page[] - ): Promise => { - return Promise.resolve(true) + import = async (integration: Integration): Promise => { + return Promise.resolve() } } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index ac9399274..9ba79d3d7 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,5 +1,66 @@ import { IntegrationService } from './integration' +import { Integration } from '../../entity/integration' +import axios from 'axios' +import { env } from '../../env' +import { PubSub } from '@google-cloud/pubsub' + +interface PocketResponse { + list: { + [key: string]: PocketItem + } +} + +interface PocketItem { + given_url: string +} export class PocketIntegration extends IntegrationService { name = 'POCKET' + POCKET_API_URL = 'https://getpocket.com/v3' + IMPORT_TOPIC = 'importURL' + + retrievePocketData = async ( + accessToken: string, + since: number + ): Promise => { + const url = `${this.POCKET_API_URL}/get` + try { + const response = await axios.post(url, { + consumer_key: env.pocket.consumerKey, + access_token: accessToken, + state: 'all', + detailType: 'simple', + since, + }) + return response.data + } catch (error) { + console.log('error retrieving pocket data', error) + throw error + } + } + + import = async (integration: Integration): Promise => { + const syncAt = integration.syncedAt + ? integration.syncedAt.getTime() / 1000 + : 0 + const pocketData = await this.retrievePocketData(integration.token, syncAt) + const pocketItems = Object.values(pocketData.list) + // publish pocket items to queue + const client = new PubSub() + await Promise.all( + pocketItems.map((item) => { + return client + .topic(this.IMPORT_TOPIC) + .publishMessage({ + data: JSON.stringify({ + url: item.given_url, + }), + }) + .catch((err) => { + console.log('error publishing to pubsub', err) + return undefined + }) + }) + ) + } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 16c4e6021..ab069c119 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -52,7 +52,7 @@ export class ReadwiseIntegration extends IntegrationService { return false } } - exportPages = async ( + export = async ( integration: Integration, pages: Page[] ): Promise => { diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 1b8ff76f0..f8c6f9d4c 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -98,6 +98,10 @@ interface BackendEnv { gcp: { location: string } + + pocket: { + consumerKey: string + } } /*** @@ -154,6 +158,7 @@ const nullableEnvVars = [ 'AZURE_SPEECH_REGION', 'GCP_LOCATION', 'RECOMMENDATION_TASK_HANDLER_URL', + 'POCKET_CONSUMER_KEY', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -284,6 +289,10 @@ export function getEnv(): BackendEnv { location: parse('GCP_LOCATION'), } + const pocket = { + consumerKey: parse('POCKET_CONSUMER_KEY'), + } + return { pg, client, @@ -304,6 +313,7 @@ export function getEnv(): BackendEnv { readwise, azure, gcp, + pocket, } } From e2d050288e4578db81021f6d8adcbac48ded40bd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Feb 2023 12:33:37 +0800 Subject: [PATCH 05/48] write the list of urls to a csv file and upload it to gcs --- .../src/services/integrations/integration.ts | 4 +- .../api/src/services/integrations/pocket.ts | 39 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index aad49a369..1581a0842 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -13,7 +13,7 @@ export abstract class IntegrationService { ): Promise => { return Promise.resolve(true) } - import = async (integration: Integration): Promise => { - return Promise.resolve() + import = async (integration: Integration): Promise => { + return Promise.resolve(0) } } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 9ba79d3d7..6370ad986 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -2,7 +2,10 @@ import { IntegrationService } from './integration' import { Integration } from '../../entity/integration' import axios from 'axios' import { env } from '../../env' -import { PubSub } from '@google-cloud/pubsub' +import { DateTime } from 'luxon' +import { uploadToBucket } from '../../utils/uploads' +import { v4 as uuidv4 } from 'uuid' +import { getRepository } from '../../entity/utils' interface PocketResponse { list: { @@ -17,7 +20,6 @@ interface PocketItem { export class PocketIntegration extends IntegrationService { name = 'POCKET' POCKET_API_URL = 'https://getpocket.com/v3' - IMPORT_TOPIC = 'importURL' retrievePocketData = async ( accessToken: string, @@ -39,28 +41,25 @@ export class PocketIntegration extends IntegrationService { } } - import = async (integration: Integration): Promise => { + import = async (integration: Integration): Promise => { const syncAt = integration.syncedAt ? integration.syncedAt.getTime() / 1000 : 0 const pocketData = await this.retrievePocketData(integration.token, syncAt) const pocketItems = Object.values(pocketData.list) - // publish pocket items to queue - const client = new PubSub() - await Promise.all( - pocketItems.map((item) => { - return client - .topic(this.IMPORT_TOPIC) - .publishMessage({ - data: JSON.stringify({ - url: item.given_url, - }), - }) - .catch((err) => { - console.log('error publishing to pubsub', err) - return undefined - }) - }) - ) + // write the list of urls to a csv file and upload it to gcs + // path style: imports///-.csv + const dateStr = DateTime.now().toISODate() + const fileUuid = uuidv4() + const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv` + const data = pocketItems.map((item) => item.given_url).join('\n') + await uploadToBucket(fullPath, Buffer.from(data, 'utf-8'), { + contentType: 'text/csv', + }) + // update the integration's syncedAt + await getRepository(Integration).update(integration.id, { + syncedAt: new Date(), + }) + return pocketItems.length } } From 02247bc0a60dab0440703e48e8704d535bfaf820 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Feb 2023 12:41:39 +0800 Subject: [PATCH 06/48] Add import from integration api implementation --- packages/api/src/generated/graphql.ts | 4 +- packages/api/src/generated/schema.graphql | 2 +- .../api/src/resolvers/function_resolvers.ts | 3 ++ .../api/src/resolvers/integrations/index.ts | 46 +++++++++++++++++++ packages/api/src/schema.ts | 2 +- 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 656149b99..dcc658e09 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -954,7 +954,7 @@ export type ImportFromIntegrationResult = ImportFromIntegrationError | ImportFro export type ImportFromIntegrationSuccess = { __typename?: 'ImportFromIntegrationSuccess'; - success: Scalars['Boolean']; + count: Scalars['Int']; }; export type Integration = { @@ -4807,7 +4807,7 @@ export type ImportFromIntegrationResultResolvers = { - success?: Resolver; + count?: Resolver; __isTypeOf?: IsTypeOfResolverFn; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 651eafbff..22eb9d4dc 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -847,7 +847,7 @@ enum ImportFromIntegrationErrorCode { union ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess type ImportFromIntegrationSuccess { - success: Boolean! + count: Int! } type Integration { diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 4e2228ccb..f5624512c 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -62,6 +62,7 @@ import { googleLoginResolver, googleSignupResolver, groupsResolver, + importFromIntegrationResolver, integrationsResolver, joinGroupResolver, labelsResolver, @@ -202,6 +203,7 @@ export const functionResolvers = { uploadImportFile: uploadImportFileResolver, markEmailAsItem: markEmailAsItemResolver, bulkAction: bulkActionResolver, + importFromIntegration: importFromIntegrationResolver, }, Query: { me: getMeUserResolver, @@ -650,4 +652,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('RecentEmails'), ...resultResolveTypeResolver('MarkEmailAsItem'), ...resultResolveTypeResolver('BulkAction'), + ...resultResolveTypeResolver('ImportFromIntegration'), } diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 9aa2c8b3e..b2cf17f88 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -3,10 +3,14 @@ import { DeleteIntegrationError, DeleteIntegrationErrorCode, DeleteIntegrationSuccess, + ImportFromIntegrationError, + ImportFromIntegrationErrorCode, + ImportFromIntegrationSuccess, IntegrationsError, IntegrationsErrorCode, IntegrationsSuccess, MutationDeleteIntegrationArgs, + MutationImportFromIntegrationArgs, MutationSetIntegrationArgs, SetIntegrationError, SetIntegrationErrorCode, @@ -222,3 +226,45 @@ export const deleteIntegrationResolver = authorized< } } }) + +export const importFromIntegrationResolver = authorized< + ImportFromIntegrationSuccess, + ImportFromIntegrationError, + MutationImportFromIntegrationArgs +>(async (_, { integrationId }, { claims: { uid }, log }) => { + log.info('importFromIntegrationResolver') + + try { + const integration = await getRepository(Integration).findOne({ + where: { id: integrationId, user: { id: uid } }, + relations: ['user'], + }) + + if (!integration) { + return { + errorCodes: [ImportFromIntegrationErrorCode.Unauthorized], + } + } + + const integrationService = getIntegrationService(integration.name) + const count = await integrationService.import(integration) + + analytics.track({ + userId: uid, + event: 'integration_import', + properties: { + integrationId, + }, + }) + + return { + count, + } + } catch (error) { + log.error(error) + + return { + errorCodes: [ImportFromIntegrationErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 7751f0c33..e45f278c7 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2424,7 +2424,7 @@ const schema = gql` | ImportFromIntegrationError type ImportFromIntegrationSuccess { - success: Boolean! + count: Int! } type ImportFromIntegrationError { From c3bfb593c6e23f6049fe5a33f58a5b914f06a86e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Feb 2023 15:49:37 +0800 Subject: [PATCH 07/48] Fix regression --- .../api/src/resolvers/integrations/index.ts | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index b2cf17f88..14a049d2d 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -39,8 +39,11 @@ export const setIntegrationResolver = authorized< } } - let integrationToSave: Partial = { + const integrationToSave: Partial = { + ...input, user, + id: input.id || undefined, + type: input.type || IntegrationType.Export, } if (input.id) { // Update @@ -59,13 +62,8 @@ export const setIntegrationResolver = authorized< } } - integrationToSave = { - ...integrationToSave, - id: existingIntegration.id, - enabled: input.enabled, - token: input.token, - taskName: existingIntegration.taskName, - } + integrationToSave.id = existingIntegration.id + integrationToSave.taskName = existingIntegration.taskName } else { // Create const integrationService = getIntegrationService(input.name) @@ -75,20 +73,13 @@ export const setIntegrationResolver = authorized< errorCodes: [SetIntegrationErrorCode.InvalidToken], } } - integrationToSave = { - ...integrationToSave, - name: input.name, - type: input.type ?? undefined, - token: input.token, - enabled: true, - } } // save integration const integration = await getRepository(Integration).save(integrationToSave) if ( - integration.type === IntegrationType.Export && + integrationToSave.type === IntegrationType.Export && (!integrationToSave.id || integrationToSave.enabled) ) { // create a task to sync all the pages if new integration or enable integration (export type) From 9c7f6fefcaaa6fa95b27e5c7b3edc66a4dd9d639 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Feb 2023 16:33:58 +0800 Subject: [PATCH 08/48] Add PocketIntegration service to the list --- packages/api/src/services/integrations/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts index 966c4584d..c5946ceed 100644 --- a/packages/api/src/services/integrations/index.ts +++ b/packages/api/src/services/integrations/index.ts @@ -1,7 +1,11 @@ import { ReadwiseIntegration } from './readwise' import { IntegrationService } from './integration' +import { PocketIntegration } from './pocket' -const integrations: IntegrationService[] = [new ReadwiseIntegration()] +const integrations: IntegrationService[] = [ + new ReadwiseIntegration(), + new PocketIntegration(), +] export const getIntegrationService = (name: string): IntegrationService => { const service = integrations.find((s) => s.name === name) From 6554d3b6015b1c6be57286e08786eaf07b43e067 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Feb 2023 17:01:18 +0800 Subject: [PATCH 09/48] Add tests --- .../api/test/resolvers/integrations.test.ts | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 91d2d08ca..4973887a8 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -3,11 +3,16 @@ import { User } from '../../src/entity/user' import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' import { generateFakeUuid, graphqlRequest, request } from '../util' import { SetIntegrationErrorCode } from '../../src/generated/graphql' -import { expect } from 'chai' +import chai, { expect } from 'chai' import { getRepository } from '../../src/entity/utils' import { Integration } from '../../src/entity/integration' import nock from 'nock' import { READWISE_API_URL } from '../../src/services/integrations/readwise' +import sinonChai from 'sinon-chai' +import sinon from 'sinon' +import * as uploads from '../../src/utils/uploads' + +chai.use(sinonChai) describe('Integrations resolvers', () => { let loginUser: User @@ -351,4 +356,81 @@ describe('Integrations resolvers', () => { }) }) }) + + describe('importFromIntegration API', () => { + const query = (integrationId: string) => ` + mutation { + importFromIntegration(integrationId: "${integrationId}") { + ... on ImportFromIntegrationSuccess { + count + } + ... on ImportFromIntegrationError { + errorCodes + } + } + } + ` + let existingIntegration: Integration + + context('when integration exists', () => { + before(async () => { + existingIntegration = await getRepository(Integration).save({ + user: { id: loginUser.id }, + name: 'POCKET', + token: 'fakeToken', + }) + + nock('https://getpocket.com', { + reqheaders: { + 'content-type': 'application/json', + }, + }) + .post('/v3/get', { + access_token: existingIntegration.token, + consumer_key: '', + state: 'all', + detailType: 'simple', + since: 0, + }) + .reply(200, { + list: { + '123': { + given_url: 'https://omnivore.app/pocket-import-test', + }, + }, + }) + + sinon.replace(uploads, 'uploadToBucket', () => { + return Promise.resolve() + }) + }) + + after(async () => { + await deleteTestIntegrations(loginUser.id, [existingIntegration.id]) + sinon.restore() + }) + + it('returns count and updates syncAt', async () => { + const res = await graphqlRequest( + query(existingIntegration.id), + authToken + ) + expect(res.body.data.importFromIntegration.count).to.eql(1) + const integration = await getRepository(Integration).findOneBy({ + id: existingIntegration.id, + }) + expect(integration?.syncedAt).not.to.be.null + }) + }) + + context('when integration does not exist', () => { + it('returns error', async () => { + const invalidIntegrationId = generateFakeUuid() + const res = await graphqlRequest(query(invalidIntegrationId), authToken) + expect(res.body.data.importFromIntegration.errorCodes).to.eql([ + 'UNAUTHORIZED', + ]) + }) + }) + }) }) From da1fdb85f927e9953d5694e4718fa1baa21df9f7 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 16 Feb 2023 17:04:06 +0800 Subject: [PATCH 10/48] Update web to work with latest integration API --- .../components/templates/integrations/Readwise.tsx | 5 +++-- .../networking/mutations/setIntegrationMutation.ts | 14 +++++++------- .../networking/queries/useGetIntegrationsQuery.tsx | 3 ++- packages/web/pages/settings/integrations.tsx | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/web/components/templates/integrations/Readwise.tsx b/packages/web/components/templates/integrations/Readwise.tsx index a6f4de6fc..4e0469896 100644 --- a/packages/web/components/templates/integrations/Readwise.tsx +++ b/packages/web/components/templates/integrations/Readwise.tsx @@ -26,7 +26,7 @@ const Header = styled(Box, { export function Readwise(): JSX.Element { const { integrations, revalidate } = useGetIntegrationsQuery() const readwiseIntegration = useMemo(() => { - return integrations.find((i) => i.type == 'READWISE') + return integrations.find((i) => i.name == 'READWISE' && i.type == 'EXPORT') }, [integrations]) return ( @@ -82,7 +82,8 @@ function AddReadwiseForm(): JSX.Element { try { const result = await setIntegrationMutation({ token, - type: 'READWISE', + name: 'READWISE', + type: 'EXPORT', enabled: true, }) if (result) { diff --git a/packages/web/lib/networking/mutations/setIntegrationMutation.ts b/packages/web/lib/networking/mutations/setIntegrationMutation.ts index cae1ff341..bda8beaf0 100644 --- a/packages/web/lib/networking/mutations/setIntegrationMutation.ts +++ b/packages/web/lib/networking/mutations/setIntegrationMutation.ts @@ -4,8 +4,9 @@ import { IntegrationType } from '../queries/useGetIntegrationsQuery' export type SetIntegrationInput = { id?: string + name: string type: IntegrationType - token: string, + token: string enabled: boolean } @@ -20,6 +21,7 @@ type SetIntegrationData = { type Integration = { id: string + name: string type: IntegrationType token: string enabled: boolean @@ -31,13 +33,12 @@ export async function setIntegrationMutation( input: SetIntegrationInput ): Promise { const mutation = gql` - mutation SetIntegration( - $input: SetIntegrationInput! - ) { + mutation SetIntegration($input: SetIntegrationInput!) { setIntegration(input: $input) { ... on SetIntegrationSuccess { integration { id + name type token enabled @@ -52,12 +53,11 @@ export async function setIntegrationMutation( } ` - const data = await gqlFetcher(mutation, { input }) as SetIntegrationResult + const data = (await gqlFetcher(mutation, { input })) as SetIntegrationResult const output = data as any const error = data.setIntegration?.errorCodes?.find(() => true) if (error) { - if (error === 'INVALID_TOKEN') - throw 'Your token is invalid.' + if (error === 'INVALID_TOKEN') throw 'Your token is invalid.' throw error } return output.setIntegration?.integration diff --git a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx index e61ef35ae..6b748c0b7 100644 --- a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx @@ -4,6 +4,7 @@ import { publicGqlFetcher } from '../networkHelpers' export interface Integration { id: string + name: string type: IntegrationType token: string enabled: boolean @@ -11,7 +12,7 @@ export interface Integration { updatedAt: Date } -export type IntegrationType = 'READWISE' +export type IntegrationType = 'EXPORT' | 'IMPORT' interface IntegrationsQueryResponse { isValidating: boolean diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 02835e9b2..501675c57 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -65,7 +65,7 @@ export default function Integrations(): JSX.Element { const router = useRouter() const readwiseConnected = useMemo(() => { - return integrations.find((i) => i.type == 'READWISE') + return integrations.find((i) => i.name == 'READWISE' && i.type == 'EXPORT') }, [integrations]) const deleteIntegration = async (id: string) => { From ae2f695df84c728a75ce81a2e01dcbc949ac16f1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 12:22:07 +0800 Subject: [PATCH 11/48] Get access token from pocket --- packages/api/src/resolvers/integrations/index.ts | 6 ++++-- .../api/src/services/integrations/integration.ts | 4 ++-- packages/api/src/services/integrations/pocket.ts | 14 ++++++++++++++ packages/api/src/services/integrations/readwise.ts | 6 +++--- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 14a049d2d..0618bf3ec 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -67,12 +67,14 @@ export const setIntegrationResolver = authorized< } else { // Create const integrationService = getIntegrationService(input.name) - // validate token - if (!(await integrationService.validateToken(input.token))) { + // authorize and get access token + const token = await integrationService.accessToken(input.token) + if (!token) { return { errorCodes: [SetIntegrationErrorCode.InvalidToken], } } + integrationToSave.token = token } // save integration diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index 1581a0842..045869111 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -4,8 +4,8 @@ import { Page } from '../../elastic/types' export abstract class IntegrationService { abstract name: string - validateToken = async (token: string): Promise => { - return Promise.resolve(true) + accessToken = async (token: string): Promise => { + return Promise.resolve('') } export = async ( integration: Integration, diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 6370ad986..e7cb53286 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -21,6 +21,20 @@ export class PocketIntegration extends IntegrationService { name = 'POCKET' POCKET_API_URL = 'https://getpocket.com/v3' + accessToken = async (token: string): Promise => { + const url = `${this.POCKET_API_URL}/oauth/authorize` + try { + const response = await axios.post<{ access_token: string }>(url, { + consumer_key: env.pocket.consumerKey, + code: token, + }) + return response.data.access_token + } catch (error) { + console.log('error validating pocket token', error) + return null + } + } + retrievePocketData = async ( accessToken: string, since: number diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index ab069c119..9c680cf0e 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -38,7 +38,7 @@ export const READWISE_API_URL = 'https://readwise.io/api/v2' export class ReadwiseIntegration extends IntegrationService { name = 'READWISE' - validateToken = async (token: string): Promise => { + accessToken = async (token: string): Promise => { const authUrl = `${env.readwise.apiUrl || READWISE_API_URL}/auth` try { const response = await axios.get(authUrl, { @@ -46,10 +46,10 @@ export class ReadwiseIntegration extends IntegrationService { Authorization: `Token ${token}`, }, }) - return response.status === 204 + return response.status === 204 ? token : null } catch (error) { console.log('error validating readwise token', error) - return false + return null } } export = async ( From 59621768c86d21bc7a79aa3f0de00bf02de3e8f9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 12:23:57 +0800 Subject: [PATCH 12/48] Get request token from pocket --- .../api/src/routers/integration_router.ts | 54 +++++++++++++++++++ packages/api/src/server.ts | 2 + 2 files changed, 56 insertions(+) create mode 100644 packages/api/src/routers/integration_router.ts diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts new file mode 100644 index 000000000..7e6d4079b --- /dev/null +++ b/packages/api/src/routers/integration_router.ts @@ -0,0 +1,54 @@ +import cors from 'cors' +import express from 'express' +import { corsConfig } from '../utils/corsConfig' +import { env } from '../env' +import axios from 'axios' +import { buildLogger } from '../utils/logger' +import { getClaimsByToken } from '../utils/auth' + +const logger = buildLogger('app.dispatch') + +export function integrationRouter() { + const router = express.Router() + // request token from pocket + router.post( + '/pocket/request-token', + cors(corsConfig), + async (req: express.Request, res: express.Response) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const token = (req.cookies.auth as string) || req.headers.authorization + const claims = await getClaimsByToken(token) + if (!claims) { + return res.status(401).send('UNAUTHORIZED') + } + + const consumerKey = env.pocket.consumerKey + const redirectUri = `${env.client.url}/settings/integrations:pocketAuthorizationFinished` + try { + // make a POST request to Pocket to get a request token + const response = await axios.post<{ code: string }>( + 'https://getpocket.com/v3/oauth/request', + { + consumer_key: consumerKey, + redirect_uri: redirectUri, + } + ) + const { code } = response.data + // store the request token in a cookie + res.cookie('pocketRequestToken', code, { + maxAge: 1000 * 60 * 60, + }) + // redirect the user to Pocket to authorize the request token + res.redirect( + `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}` + ) + } catch (e) { + logger.info('pocket/request-token exception:', e) + res.redirect( + `${env.client.url}/settings/integrations?errorCodes=UNKNOWN` + ) + } + } + ) + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 506046777..deccfade0 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -49,6 +49,7 @@ import { textToSpeechRouter } from './routers/text_to_speech' import * as httpContext from 'express-http-context' import { notificationRouter } from './routers/notification_router' import { userRouter } from './routers/user_router' +import { integrationRouter } from './routers/integration_router' const PORT = process.env.PORT || 4000 @@ -135,6 +136,7 @@ export const createApp = (): { app.use('/api/mobile-auth', mobileAuthRouter()) app.use('/api/text-to-speech', textToSpeechRouter()) app.use('/api/notification', notificationRouter()) + app.use('/api/integration', integrationRouter()) app.use('/svc/pubsub/content', contentServiceRouter()) app.use('/svc/pubsub/links', linkServiceRouter()) app.use('/svc/pubsub/newsletters', newsletterServiceRouter()) From c54ef1185fcd0dace03fa54d6349cf22ae2ac09c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 12:24:09 +0800 Subject: [PATCH 13/48] Add pocket svg --- packages/web/public/static/icons/pocket.svg | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/web/public/static/icons/pocket.svg diff --git a/packages/web/public/static/icons/pocket.svg b/packages/web/public/static/icons/pocket.svg new file mode 100644 index 000000000..c8b6eba6c --- /dev/null +++ b/packages/web/public/static/icons/pocket.svg @@ -0,0 +1 @@ + \ No newline at end of file From 2e5e3bd19ea4fabce6e1372af24254eabe2507ec Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 13:09:00 +0800 Subject: [PATCH 14/48] Add headers --- .../api/src/routers/integration_router.ts | 11 +++++- .../api/src/services/integrations/pocket.ts | 38 +++++++++++++------ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts index 7e6d4079b..7614aeacd 100644 --- a/packages/api/src/routers/integration_router.ts +++ b/packages/api/src/routers/integration_router.ts @@ -12,9 +12,10 @@ export function integrationRouter() { const router = express.Router() // request token from pocket router.post( - '/pocket/request-token', + '/pocket/auth', cors(corsConfig), async (req: express.Request, res: express.Response) => { + logger.info('pocket/request-token') // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const token = (req.cookies.auth as string) || req.headers.authorization const claims = await getClaimsByToken(token) @@ -23,7 +24,7 @@ export function integrationRouter() { } const consumerKey = env.pocket.consumerKey - const redirectUri = `${env.client.url}/settings/integrations:pocketAuthorizationFinished` + const redirectUri = `${env.client.url}/settings/integrations?state=pocketAuthorizationFinished` try { // make a POST request to Pocket to get a request token const response = await axios.post<{ code: string }>( @@ -31,6 +32,12 @@ export function integrationRouter() { { consumer_key: consumerKey, redirect_uri: redirectUri, + }, + { + headers: { + 'Content-Type': 'application/json', + 'X-Accept': 'application/json', + }, } ) const { code } = response.data diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index e7cb53286..5ad62eb65 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -20,14 +20,24 @@ interface PocketItem { export class PocketIntegration extends IntegrationService { name = 'POCKET' POCKET_API_URL = 'https://getpocket.com/v3' + headers = { + 'Content-Type': 'application/json', + 'X-Accept': 'application/json', + } accessToken = async (token: string): Promise => { const url = `${this.POCKET_API_URL}/oauth/authorize` try { - const response = await axios.post<{ access_token: string }>(url, { - consumer_key: env.pocket.consumerKey, - code: token, - }) + const response = await axios.post<{ access_token: string }>( + url, + { + consumer_key: env.pocket.consumerKey, + code: token, + }, + { + headers: this.headers, + } + ) return response.data.access_token } catch (error) { console.log('error validating pocket token', error) @@ -41,13 +51,19 @@ export class PocketIntegration extends IntegrationService { ): Promise => { const url = `${this.POCKET_API_URL}/get` try { - const response = await axios.post(url, { - consumer_key: env.pocket.consumerKey, - access_token: accessToken, - state: 'all', - detailType: 'simple', - since, - }) + const response = await axios.post( + url, + { + consumer_key: env.pocket.consumerKey, + access_token: accessToken, + state: 'all', + detailType: 'simple', + since, + }, + { + headers: this.headers, + } + ) return response.data } catch (error) { console.log('error retrieving pocket data', error) From 950d41a38797113b6de0a2fc7d62c0e9e2436ae3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 14:01:14 +0800 Subject: [PATCH 15/48] Add pocket integration to web --- packages/web/pages/settings/integrations.tsx | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 501675c57..f6ad691f7 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -18,6 +18,9 @@ import { useGetIntegrationsQuery } from '../../lib/networking/queries/useGetInte import { useGetWebhooksQuery } from '../../lib/networking/queries/useGetWebhooksQuery' import { deleteIntegrationMutation } from '../../lib/networking/mutations/deleteIntegrationMutation' import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers' +import { fetchEndpoint } from '../../lib/appConfig' +import { setIntegrationMutation } from '../../lib/networking/mutations/setIntegrationMutation' +import { cookieValue } from '../../lib/cookieHelpers' // Styles const Header = styled(Box, { @@ -78,6 +81,45 @@ export default function Integrations(): JSX.Element { } } + const redirectToPocket = () => { + // create a form and submit it to the backend + const form = document.createElement('form') + form.method = 'POST' + form.action = `${fetchEndpoint}/integration/pocket/auth` + document.body.appendChild(form) + form.submit() + } + + useEffect(() => { + const connectToPocket = async () => { + try { + // get the token from cookies + const token = cookieValue('pocketRequestToken', document.cookie) + if (!token) { + showErrorToast('There was an error connecting to Pocket.') + return + } + const result = await setIntegrationMutation({ + token, + name: 'POCKET', + type: 'IMPORT', + enabled: true, + }) + if (result) { + showSuccessToast('Connected with Pocket.') + } else { + showErrorToast('There was an error connecting to Pocket.') + } + } catch (err) { + showErrorToast('Error: ' + err) + } + } + if (!router.isReady) return + if (router.query.state == 'pocketAuthorizationFinished') { + connectToPocket() + } + }, [router]) + useEffect(() => { setIntegrationsArray([ { @@ -121,6 +163,19 @@ export default function Integrations(): JSX.Element { action: () => router.push('/settings/webhooks'), }, }, + { + icon: '/static/icons/pocket.svg', + title: 'Pocket', + subText: 'Pocket is a place to save articles, videos, and more.', + button: { + text: 'Connect to Pocket', + icon: , + style: 'ctaDarkYellow', + action: () => { + redirectToPocket() + }, + }, + }, ]) }, [readwiseConnected, router, webhooks]) From ab1154ef6f4013f6212f984b0ed9c6e25924edae Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 14:31:14 +0800 Subject: [PATCH 16/48] Add remove pocket integration to web --- .../networking/queries/useGetIntegrationsQuery.tsx | 1 + packages/web/pages/settings/integrations.tsx | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx index 6b748c0b7..e7a4c0501 100644 --- a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx @@ -35,6 +35,7 @@ export function useGetIntegrationsQuery(): IntegrationsQueryResponse { ... on IntegrationsSuccess { integrations { id + name type token enabled diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index f6ad691f7..3f81591e3 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -70,6 +70,9 @@ export default function Integrations(): JSX.Element { const readwiseConnected = useMemo(() => { return integrations.find((i) => i.name == 'READWISE' && i.type == 'EXPORT') }, [integrations]) + const isConnected = (name: string, type: string) => { + return integrations.find((i) => i.name === name && i.type === type) + } const deleteIntegration = async (id: string) => { try { @@ -121,6 +124,7 @@ export default function Integrations(): JSX.Element { }, [router]) useEffect(() => { + const pocketConnected = isConnected('POCKET', 'IMPORT') setIntegrationsArray([ { icon: '/static/icons/logseq.svg', @@ -168,16 +172,18 @@ export default function Integrations(): JSX.Element { title: 'Pocket', subText: 'Pocket is a place to save articles, videos, and more.', button: { - text: 'Connect to Pocket', + text: pocketConnected ? 'Remove' : 'Connect to Pocket', icon: , - style: 'ctaDarkYellow', + style: pocketConnected ? 'ctaWhite' : 'ctaDarkYellow', action: () => { - redirectToPocket() + pocketConnected + ? deleteIntegration(pocketConnected.id) + : redirectToPocket() }, }, }, ]) - }, [readwiseConnected, router, webhooks]) + }, [integrations]) return ( From 737957ff82298e6a393794255d805f3b2ec14a8a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 14:54:37 +0800 Subject: [PATCH 17/48] Add import from pocket to web --- .../api/src/services/integrations/pocket.ts | 3 ++ .../importFromIntegrationMutation.ts | 35 +++++++++++++++++++ packages/web/pages/settings/integrations.tsx | 16 +++++++-- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 packages/web/lib/networking/mutations/importFromIntegrationMutation.ts diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 5ad62eb65..d3086686f 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -77,6 +77,9 @@ export class PocketIntegration extends IntegrationService { : 0 const pocketData = await this.retrievePocketData(integration.token, syncAt) const pocketItems = Object.values(pocketData.list) + if (pocketItems.length === 0) { + return 0 + } // write the list of urls to a csv file and upload it to gcs // path style: imports///-.csv const dateStr = DateTime.now().toISODate() diff --git a/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts b/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts new file mode 100644 index 000000000..768ba27e4 --- /dev/null +++ b/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts @@ -0,0 +1,35 @@ +import { gqlFetcher } from '../networkHelpers' + +interface ImportFromIntegrationDataResponseData { + importFromIntegration?: ImportFromIntegrationData +} + +interface ImportFromIntegrationData { + count: number + errorCodes?: unknown[] +} + +export async function importFromIntegrationMutation( + integrationId: string +): Promise { + const mutation = ` + mutation ImportFromIntegration($integrationId: ID!) { + importFromIntegration(integrationId:$integrationId) { + ... on ImportFromIntegrationError { + errorCodes + } + ... on ImportFromIntegrationSuccess { + count + } + } + }` + + const data = await gqlFetcher(mutation, { integrationId }) + console.log('integrationId: ', data) + const output = data as ImportFromIntegrationDataResponseData | undefined + const error = output?.importFromIntegration?.errorCodes?.find(() => true) + console.log('error: ', error) + if (error) { + throw error + } +} diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 3f81591e3..861afe407 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -21,6 +21,7 @@ import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers' import { fetchEndpoint } from '../../lib/appConfig' import { setIntegrationMutation } from '../../lib/networking/mutations/setIntegrationMutation' import { cookieValue } from '../../lib/cookieHelpers' +import { importFromIntegrationMutation } from '../../lib/networking/mutations/importFromIntegrationMutation' // Styles const Header = styled(Box, { @@ -84,6 +85,15 @@ export default function Integrations(): JSX.Element { } } + const importFromIntegration = async (id: string) => { + try { + await importFromIntegrationMutation(id) + showSuccessToast('Import started') + } catch (err) { + showErrorToast('Error: ' + err) + } + } + const redirectToPocket = () => { // create a form and submit it to the backend const form = document.createElement('form') @@ -172,12 +182,12 @@ export default function Integrations(): JSX.Element { title: 'Pocket', subText: 'Pocket is a place to save articles, videos, and more.', button: { - text: pocketConnected ? 'Remove' : 'Connect to Pocket', + text: pocketConnected ? 'Import' : 'Connect to Pocket', icon: , - style: pocketConnected ? 'ctaWhite' : 'ctaDarkYellow', + style: 'ctaDarkYellow', action: () => { pocketConnected - ? deleteIntegration(pocketConnected.id) + ? importFromIntegration(pocketConnected.id) : redirectToPocket() }, }, From 3522e135f4583ed5fc45275598c852d0602550ef Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 15:19:18 +0800 Subject: [PATCH 18/48] Refresh state after connected --- packages/web/pages/settings/integrations.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 861afe407..12ae73333 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -71,9 +71,9 @@ export default function Integrations(): JSX.Element { const readwiseConnected = useMemo(() => { return integrations.find((i) => i.name == 'READWISE' && i.type == 'EXPORT') }, [integrations]) - const isConnected = (name: string, type: string) => { - return integrations.find((i) => i.name === name && i.type === type) - } + const pocketConnected = useMemo(() => { + return integrations.find((i) => i.name == 'POCKET' && i.type == 'IMPORT') + }, [integrations]) const deleteIntegration = async (id: string) => { try { @@ -120,6 +120,7 @@ export default function Integrations(): JSX.Element { }) if (result) { showSuccessToast('Connected with Pocket.') + await router.push('/settings/integrations') } else { showErrorToast('There was an error connecting to Pocket.') } @@ -134,7 +135,6 @@ export default function Integrations(): JSX.Element { }, [router]) useEffect(() => { - const pocketConnected = isConnected('POCKET', 'IMPORT') setIntegrationsArray([ { icon: '/static/icons/logseq.svg', From b58b8dd03a66a6b49fc8385c25399e7542d8533f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Feb 2023 15:21:41 +0800 Subject: [PATCH 19/48] Add height --- packages/web/pages/settings/integrations.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 12ae73333..330a6bf0b 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -213,7 +213,7 @@ export default function Integrations(): JSX.Element { css={{ width: '80%', margin: '0 auto', - height: '100%', + height: '800px', '@smDown': { width: '100%', }, From c78fe9f87d7ed720efbf9baeb250211defdbafa4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 15:40:33 +0800 Subject: [PATCH 20/48] Paginate pocket retrieve api response data --- .../api/src/services/integrations/pocket.ts | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index d3086686f..9923e4ecf 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -14,7 +14,33 @@ interface PocketResponse { } interface PocketItem { + item_id: string + resolved_id: string given_url: string + resolved_url: string + given_title: string + resolved_title: string + favorite: string + status: string + excerpt: string + word_count: string + tags: { + [key: string]: Tag + } + authors: { + [key: string]: Author + } +} + +interface Tag { + item_id: string + tag: string +} + +interface Author { + item_id: string + author_id: string + name: string } export class PocketIntegration extends IntegrationService { @@ -47,7 +73,9 @@ export class PocketIntegration extends IntegrationService { retrievePocketData = async ( accessToken: string, - since: number + since: number, + count = 100, + offset = 0 ): Promise => { const url = `${this.POCKET_API_URL}/get` try { @@ -57,13 +85,17 @@ export class PocketIntegration extends IntegrationService { consumer_key: env.pocket.consumerKey, access_token: accessToken, state: 'all', - detailType: 'simple', + detailType: 'complete', since, + sort: 'oldest', + count, + offset, }, { headers: this.headers, } ) + console.debug('pocket data', response.data) return response.data } catch (error) { console.log('error retrieving pocket data', error) From 866d70e416ed67546fd5aeb7d81c4f8f6c07597c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 16:57:14 +0800 Subject: [PATCH 21/48] Revalidate integrations after connected with pocket --- packages/web/pages/settings/integrations.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index 330a6bf0b..d4d16ce76 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -119,8 +119,8 @@ export default function Integrations(): JSX.Element { enabled: true, }) if (result) { + revalidate() showSuccessToast('Connected with Pocket.') - await router.push('/settings/integrations') } else { showErrorToast('There was an error connecting to Pocket.') } @@ -193,7 +193,7 @@ export default function Integrations(): JSX.Element { }, }, ]) - }, [integrations]) + }, [pocketConnected, readwiseConnected, webhooks]) return ( From 1c5991aded5be839776f86017f0e9665e347ea17 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 18:26:17 +0800 Subject: [PATCH 22/48] import from integration api return a boolean value --- packages/api/src/generated/graphql.ts | 4 ++-- packages/api/src/generated/schema.graphql | 2 +- packages/api/src/schema.ts | 2 +- .../lib/networking/mutations/importFromIntegrationMutation.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index dcc658e09..656149b99 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -954,7 +954,7 @@ export type ImportFromIntegrationResult = ImportFromIntegrationError | ImportFro export type ImportFromIntegrationSuccess = { __typename?: 'ImportFromIntegrationSuccess'; - count: Scalars['Int']; + success: Scalars['Boolean']; }; export type Integration = { @@ -4807,7 +4807,7 @@ export type ImportFromIntegrationResultResolvers = { - count?: Resolver; + success?: Resolver; __isTypeOf?: IsTypeOfResolverFn; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 22eb9d4dc..651eafbff 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -847,7 +847,7 @@ enum ImportFromIntegrationErrorCode { union ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess type ImportFromIntegrationSuccess { - count: Int! + success: Boolean! } type Integration { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e45f278c7..7751f0c33 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2424,7 +2424,7 @@ const schema = gql` | ImportFromIntegrationError type ImportFromIntegrationSuccess { - count: Int! + success: Boolean! } type ImportFromIntegrationError { diff --git a/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts b/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts index 768ba27e4..e74b5429b 100644 --- a/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts +++ b/packages/web/lib/networking/mutations/importFromIntegrationMutation.ts @@ -5,7 +5,7 @@ interface ImportFromIntegrationDataResponseData { } interface ImportFromIntegrationData { - count: number + success: boolean errorCodes?: unknown[] } @@ -19,7 +19,7 @@ export async function importFromIntegrationMutation( errorCodes } ... on ImportFromIntegrationSuccess { - count + success } } }` From b62b7696044e1cb76af017e06bb14de9c016f1fa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 18:26:58 +0800 Subject: [PATCH 23/48] Paginate pocket api --- .../src/services/integrations/integration.ts | 20 ++++- .../api/src/services/integrations/pocket.ts | 80 ++++++++++++------- .../api/test/resolvers/integrations.test.ts | 9 ++- 3 files changed, 77 insertions(+), 32 deletions(-) diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index 045869111..2f8ccc0db 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -1,6 +1,17 @@ import { Integration } from '../../entity/integration' import { Page } from '../../elastic/types' +export type RetrievedDataState = 'archived' | 'saved' | 'deleted' +export interface RetrievedData { + url: string + labels?: string[] + state?: RetrievedDataState +} +export interface RetrievedResult { + data: RetrievedData[] + hasMore: boolean +} + export abstract class IntegrationService { abstract name: string @@ -13,7 +24,12 @@ export abstract class IntegrationService { ): Promise => { return Promise.resolve(true) } - import = async (integration: Integration): Promise => { - return Promise.resolve(0) + retrieve = async ( + token: string, + since = 0, + count = 100, + offset = 0 + ): Promise => { + return Promise.resolve({ data: [], hasMore: false }) } } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 9923e4ecf..5bea482d6 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,16 +1,22 @@ -import { IntegrationService } from './integration' -import { Integration } from '../../entity/integration' +import { + IntegrationService, + RetrievedDataState, + RetrievedResult, +} from './integration' import axios from 'axios' import { env } from '../../env' -import { DateTime } from 'luxon' -import { uploadToBucket } from '../../utils/uploads' -import { v4 as uuidv4 } from 'uuid' -import { getRepository } from '../../entity/utils' interface PocketResponse { + status: number + complete: number list: { [key: string]: PocketItem } + since: number + search_meta: { + search_type: string + } + error: string } interface PocketItem { @@ -103,28 +109,48 @@ export class PocketIntegration extends IntegrationService { } } - import = async (integration: Integration): Promise => { - const syncAt = integration.syncedAt - ? integration.syncedAt.getTime() / 1000 - : 0 - const pocketData = await this.retrievePocketData(integration.token, syncAt) + retrieve = async ( + token: string, + since = 0, + count = 100, + offset = 0 + ): Promise => { + // const syncAt = integration.syncedAt + // ? integration.syncedAt.getTime() / 1000 + // : 0 + const pocketData = await this.retrievePocketData( + token, + since, + count, + offset + ) const pocketItems = Object.values(pocketData.list) - if (pocketItems.length === 0) { - return 0 + const statusToState: Record = { + '0': 'saved', + '1': 'archived', + '2': 'deleted', } - // write the list of urls to a csv file and upload it to gcs - // path style: imports///-.csv - const dateStr = DateTime.now().toISODate() - const fileUuid = uuidv4() - const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv` - const data = pocketItems.map((item) => item.given_url).join('\n') - await uploadToBucket(fullPath, Buffer.from(data, 'utf-8'), { - contentType: 'text/csv', - }) - // update the integration's syncedAt - await getRepository(Integration).update(integration.id, { - syncedAt: new Date(), - }) - return pocketItems.length + const data = pocketItems.map((item) => ({ + url: item.given_url, + labels: Object.values(item.tags).map((tag) => tag.tag), + state: statusToState[item.status], + })) + return { + data, + hasMore: pocketData.complete !== 1, + } + // // write the list of urls to a csv file and upload it to gcs + // // path style: imports///-.csv + // const dateStr = DateTime.now().toISODate() + // const fileUuid = uuidv4() + // const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv` + // const data = pocketItems.map((item) => item.given_url).join('\n') + // await uploadToBucket(fullPath, Buffer.from(data, 'utf-8'), { + // contentType: 'text/csv', + // }) + // // update the integration's syncedAt + // await getRepository(Integration).update(integration.id, { + // syncedAt: new Date(), + // }) } } diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 4973887a8..966acc391 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -357,12 +357,12 @@ describe('Integrations resolvers', () => { }) }) - describe('importFromIntegration API', () => { + xdescribe('importFromIntegration API', () => { const query = (integrationId: string) => ` mutation { importFromIntegration(integrationId: "${integrationId}") { ... on ImportFromIntegrationSuccess { - count + success } ... on ImportFromIntegrationError { errorCodes @@ -389,8 +389,11 @@ describe('Integrations resolvers', () => { access_token: existingIntegration.token, consumer_key: '', state: 'all', - detailType: 'simple', + detailType: 'complete', since: 0, + sort: 'oldest', + count: 100, + offset: 0, }) .reply(200, { list: { From c8ec84562bb05a34af5f3b712f0bfaa5ee4bd91b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 18:27:17 +0800 Subject: [PATCH 24/48] Create an import from integration cloud task --- .../api/src/resolvers/integrations/index.ts | 12 ++++--- packages/api/src/utils/createTask.ts | 33 +++++++++++++++++++ packages/web/pages/settings/integrations.tsx | 5 ++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 0618bf3ec..25f1643fe 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -22,7 +22,11 @@ import { Integration, IntegrationType } from '../../entity/integration' import { analytics } from '../../utils/analytics' import { env } from '../../env' import { getIntegrationService } from '../../services/integrations' -import { deleteTask, enqueueSyncWithIntegration } from '../../utils/createTask' +import { + deleteTask, + enqueueImportFromIntegration, + enqueueSyncWithIntegration, +} from '../../utils/createTask' export const setIntegrationResolver = authorized< SetIntegrationSuccess, @@ -239,8 +243,8 @@ export const importFromIntegrationResolver = authorized< } } - const integrationService = getIntegrationService(integration.name) - const count = await integrationService.import(integration) + // create a task to import all the pages + await enqueueImportFromIntegration(uid, integration.id) analytics.track({ userId: uid, @@ -251,7 +255,7 @@ export const importFromIntegrationResolver = authorized< }) return { - count, + success: true, } } catch (error) { log.error(error) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index d83f7b7c8..6e984966d 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -441,4 +441,37 @@ export const enqueueRecommendation = async ( return createdTasks[0].name } +export const enqueueImportFromIntegration = async ( + userId: string, + integrationId: string +): Promise => { + const { GOOGLE_CLOUD_PROJECT } = process.env + // use pubsub data format to send the userId to the task handler + const payload = { + userId, + integrationId, + } + + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { + return nanoid() + } + + const createdTasks = await createHttpTaskWithToken({ + project: GOOGLE_CLOUD_PROJECT, + payload, + taskHandlerUrl: `${env.queue.integrationTaskHandlerUrl}/import`, + priority: 'low', + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + export default createHttpTaskWithToken diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index d4d16ce76..5a1d472af 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -129,7 +129,10 @@ export default function Integrations(): JSX.Element { } } if (!router.isReady) return - if (router.query.state == 'pocketAuthorizationFinished') { + if ( + router.query.state == 'pocketAuthorizationFinished' && + !pocketConnected + ) { connectToPocket() } }, [router]) From 9ff0711a7324a9c8ccf5ad8d3cf4cf2b7dc7a44a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 18:28:45 +0800 Subject: [PATCH 25/48] Remove wrong comments --- packages/api/src/utils/createTask.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 6e984966d..f314b256d 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -446,7 +446,6 @@ export const enqueueImportFromIntegration = async ( integrationId: string ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env - // use pubsub data format to send the userId to the task handler const payload = { userId, integrationId, From 503ec3d41368655beba50c08b4e01719db390ddd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 18:31:50 +0800 Subject: [PATCH 26/48] Update task name --- packages/api/src/resolvers/integrations/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 25f1643fe..d7f402602 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -244,7 +244,9 @@ export const importFromIntegrationResolver = authorized< } // create a task to import all the pages - await enqueueImportFromIntegration(uid, integration.id) + const taskName = await enqueueImportFromIntegration(uid, integration.id) + // update task name in integration + await getRepository(Integration).update(integration.id, { taskName }) analytics.track({ userId: uid, From 9aa65f0d331f5de362ab748e96f8ce178275fda3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 18:46:17 +0800 Subject: [PATCH 27/48] Fix tests --- .../api/test/resolvers/integrations.test.ts | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 966acc391..c25b31715 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -9,8 +9,6 @@ import { Integration } from '../../src/entity/integration' import nock from 'nock' import { READWISE_API_URL } from '../../src/services/integrations/readwise' import sinonChai from 'sinon-chai' -import sinon from 'sinon' -import * as uploads from '../../src/utils/uploads' chai.use(sinonChai) @@ -357,7 +355,7 @@ describe('Integrations resolvers', () => { }) }) - xdescribe('importFromIntegration API', () => { + describe('importFromIntegration API', () => { const query = (integrationId: string) => ` mutation { importFromIntegration(integrationId: "${integrationId}") { @@ -380,56 +378,59 @@ describe('Integrations resolvers', () => { token: 'fakeToken', }) - nock('https://getpocket.com', { - reqheaders: { - 'content-type': 'application/json', - }, - }) - .post('/v3/get', { - access_token: existingIntegration.token, - consumer_key: '', - state: 'all', - detailType: 'complete', - since: 0, - sort: 'oldest', - count: 100, - offset: 0, - }) - .reply(200, { - list: { - '123': { - given_url: 'https://omnivore.app/pocket-import-test', - }, - }, - }) - - sinon.replace(uploads, 'uploadToBucket', () => { - return Promise.resolve() - }) + // nock('https://getpocket.com', { + // reqheaders: { + // 'content-type': 'application/json', + // }, + // }) + // .post('/v3/get', { + // access_token: existingIntegration.token, + // consumer_key: '', + // state: 'all', + // detailType: 'complete', + // since: 0, + // sort: 'oldest', + // count: 100, + // offset: 0, + // }) + // .reply(200, { + // list: { + // '123': { + // given_url: 'https://omnivore.app/pocket-import-test', + // }, + // }, + // }) + // + // sinon.replace(uploads, 'uploadToBucket', () => { + // return Promise.resolve() + // }) }) after(async () => { await deleteTestIntegrations(loginUser.id, [existingIntegration.id]) - sinon.restore() + // sinon.restore() }) - it('returns count and updates syncAt', async () => { + it('returns success and starts cloud task', async () => { const res = await graphqlRequest( query(existingIntegration.id), authToken - ) - expect(res.body.data.importFromIntegration.count).to.eql(1) + ).expect(200) + expect(res.body.data.importFromIntegration.success).to.be.true const integration = await getRepository(Integration).findOneBy({ id: existingIntegration.id, }) - expect(integration?.syncedAt).not.to.be.null + expect(integration?.taskName).not.to.be.null }) }) context('when integration does not exist', () => { it('returns error', async () => { const invalidIntegrationId = generateFakeUuid() - const res = await graphqlRequest(query(invalidIntegrationId), authToken) + const res = await graphqlRequest( + query(invalidIntegrationId), + authToken + ).expect(200) expect(res.body.data.importFromIntegration.errorCodes).to.eql([ 'UNAUTHORIZED', ]) From 56fd3d76173c07b647054321b7f8a9bbd2da3ef7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 Feb 2023 21:51:46 +0800 Subject: [PATCH 28/48] Create a router to handle importing from integrations cloud task --- packages/api/src/routers/svc/integrations.ts | 99 +++++++++++++++++++ .../src/services/integrations/integration.ts | 23 +++-- .../api/src/services/integrations/pocket.ts | 28 ++---- 3 files changed, 119 insertions(+), 31 deletions(-) diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 6f9ed2b8a..269eb1163 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -10,6 +10,9 @@ import { getRepository } from '../../entity/utils' import { syncWithIntegration } from '../../services/integrations' import { buildLogger } from '../../utils/logger' import { DateFilter } from '../../utils/search' +import { DateTime } from 'luxon' +import { createGCSFile } from '../../utils/uploads' +import { v4 as uuidv4 } from 'uuid' export interface Message { type?: EntityType @@ -19,6 +22,14 @@ export interface Message { articleId?: string } +interface ImportEvent { + userId: string + integrationId: string +} + +const isImportEvent = (event: any): event is ImportEvent => + 'userId' in event && 'integrationId' in event + const logger = buildLogger('app.dispatch') export function integrationsServiceRouter() { @@ -158,6 +169,94 @@ export function integrationsServiceRouter() { res.status(500).send(err) } }) + // import pages from integration + router.post('/import', async (req, res) => { + logger.info('start to import pages from integration') + const { message: msgStr, expired } = readPushSubscription(req) + + if (!msgStr) { + return res.status(400).send('Bad Request') + } + + if (expired) { + logger.info('discarding expired message') + return res.status(200).send('Expired') + } + + const data = JSON.parse(msgStr) + if (!isImportEvent(data)) { + logger.info('Invalid message') + return res.status(400).send('Bad Request') + } + + const userId = data.userId + const integration = await getRepository(Integration).findOneBy({ + user: { id: userId }, + id: data.integrationId, + enabled: true, + type: IntegrationType.Import, + }) + if (!integration) { + logger.info('No active integration found for user', { userId }) + return res.status(200).send('No integration found') + } + + const integrationService = getIntegrationService(integration.name) + // import pages from integration + logger.info('importing pages from integration', { + integrationId: integration.id, + }) + + // write the list of urls to a csv file and upload it to gcs + // path style: imports///-.csv + const dateStr = DateTime.now().toISODate() + const fileUuid = uuidv4() + const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv` + // open a write_stream to the file + const file = createGCSFile(fullPath) + const writeStream = file.createWriteStream({ + contentType: 'text/csv', + }) + + try { + let hasMore = true + let offset = 0 + let since = integration.syncedAt?.getTime() || 0 + while (hasMore) { + // get pages from integration + const retrieved = await integrationService.retrieve({ + token: integration.token, + since, + offset: offset, + }) + const retrievedData = retrieved.data + if (retrievedData.length === 0) { + break + } + // write the list of urls, state and labels to the stream + const csvData = retrievedData.map((page) => { + const { url, state, labels } = page + return [url, state, labels?.join(',')].join(',') + }) + writeStream.write(csvData.join('\n')) + + hasMore = !!retrieved.hasMore + offset += retrievedData.length + since = retrieved.since || Date.now() + } + // update the integration's syncedAt + await getRepository(Integration).update(integration.id, { + syncedAt: since, + }) + + res.status(200).send('OK') + } catch (err) { + logger.error('import pages from integration failed', err) + res.status(500).send(err) + } finally { + writeStream.end() + } + }) return router } diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index 2f8ccc0db..d91e63030 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -9,27 +9,30 @@ export interface RetrievedData { } export interface RetrievedResult { data: RetrievedData[] - hasMore: boolean + hasMore?: boolean + since?: number +} + +export interface RetrieveRequest { + token: string + since?: number + count?: number + offset?: number } export abstract class IntegrationService { abstract name: string accessToken = async (token: string): Promise => { - return Promise.resolve('') + return Promise.resolve(null) } export = async ( integration: Integration, pages: Page[] ): Promise => { - return Promise.resolve(true) + return Promise.resolve(false) } - retrieve = async ( - token: string, - since = 0, - count = 100, - offset = 0 - ): Promise => { - return Promise.resolve({ data: [], hasMore: false }) + retrieve = async (req: RetrieveRequest): Promise => { + return Promise.resolve({ data: [] }) } } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 5bea482d6..f16b4fd48 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -2,6 +2,7 @@ import { IntegrationService, RetrievedDataState, RetrievedResult, + RetrieveRequest, } from './integration' import axios from 'axios' import { env } from '../../env' @@ -109,18 +110,15 @@ export class PocketIntegration extends IntegrationService { } } - retrieve = async ( - token: string, + retrieve = async ({ + token, since = 0, count = 100, - offset = 0 - ): Promise => { - // const syncAt = integration.syncedAt - // ? integration.syncedAt.getTime() / 1000 - // : 0 + offset = 0, + }: RetrieveRequest): Promise => { const pocketData = await this.retrievePocketData( token, - since, + since / 1000, count, offset ) @@ -138,19 +136,7 @@ export class PocketIntegration extends IntegrationService { return { data, hasMore: pocketData.complete !== 1, + since: pocketData.since * 1000, } - // // write the list of urls to a csv file and upload it to gcs - // // path style: imports///-.csv - // const dateStr = DateTime.now().toISODate() - // const fileUuid = uuidv4() - // const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv` - // const data = pocketItems.map((item) => item.given_url).join('\n') - // await uploadToBucket(fullPath, Buffer.from(data, 'utf-8'), { - // contentType: 'text/csv', - // }) - // // update the integration's syncedAt - // await getRepository(Integration).update(integration.id, { - // syncedAt: new Date(), - // }) } } From d8b2d5476b14a30c2915a732e1842109d4652037 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 1 Mar 2023 23:13:23 +0800 Subject: [PATCH 29/48] Add Mock Storage class for cloud storage --- packages/api/test/mock_storage.ts | 55 +++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 packages/api/test/mock_storage.ts diff --git a/packages/api/test/mock_storage.ts b/packages/api/test/mock_storage.ts new file mode 100644 index 000000000..00415ab6b --- /dev/null +++ b/packages/api/test/mock_storage.ts @@ -0,0 +1,55 @@ +import { Writable } from 'stream' + +class MockStorage { + buckets: { [name: string]: MockBucket } + + constructor() { + this.buckets = {} + } + + bucket(name: string) { + return this.buckets[name] || (this.buckets[name] = new MockBucket(name)) + } +} + +export class MockBucket { + name: string + files: { [path: string]: MockFile } + + constructor(name: string) { + this.name = name + this.files = {} + } + + file(path: string) { + return this.files[path] || (this.files[path] = new MockFile(path)) + } +} + +class MockFile { + path: string + contents: Buffer + + constructor(path: string) { + this.path = path + this.contents = Buffer.alloc(0) + } + + createWriteStream() { + return new MockWriteStream(this) + } +} + +class MockWriteStream extends Writable { + file: MockFile + + constructor(file: MockFile) { + super() + this.file = file + } + + _write(chunk: Buffer, encoding: string, callback: (error?: Error) => void) { + this.file.contents = Buffer.concat([this.file.contents, chunk]) + callback() + } +} From 10c01c12f27d8e55c0e28d13ed819c4358bef2ce Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 1 Mar 2023 23:15:03 +0800 Subject: [PATCH 30/48] Add test case for integration import task handler --- .../api/src/resolvers/integrations/index.ts | 13 ++- packages/api/src/routers/svc/integrations.ts | 100 +++++++++--------- .../src/services/integrations/integration.ts | 4 +- .../api/src/services/integrations/pocket.ts | 16 +-- packages/api/src/utils/createTask.ts | 8 +- .../api/test/resolvers/integrations.test.ts | 28 ----- .../api/test/routers/integrations.test.ts | 97 ++++++++++++++--- 7 files changed, 163 insertions(+), 103 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index d7f402602..808affbe5 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -228,7 +228,7 @@ export const importFromIntegrationResolver = authorized< ImportFromIntegrationSuccess, ImportFromIntegrationError, MutationImportFromIntegrationArgs ->(async (_, { integrationId }, { claims: { uid }, log }) => { +>(async (_, { integrationId }, { claims: { uid }, log, signToken }) => { log.info('importFromIntegrationResolver') try { @@ -243,8 +243,17 @@ export const importFromIntegrationResolver = authorized< } } + const exp = Math.floor(Date.now() / 1000) + 60 * 60 * 24 // 1 day + const authToken = (await signToken( + { uid, exp }, + env.server.jwtSecret + )) as string // create a task to import all the pages - const taskName = await enqueueImportFromIntegration(uid, integration.id) + const taskName = await enqueueImportFromIntegration( + uid, + integration.id, + authToken + ) // update task name in integration await getRepository(Integration).update(integration.id, { taskName }) diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 269eb1163..2cd89229d 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -13,6 +13,8 @@ import { DateFilter } from '../../utils/search' import { DateTime } from 'luxon' import { createGCSFile } from '../../utils/uploads' import { v4 as uuidv4 } from 'uuid' +import { getClaimsByToken } from '../../utils/auth' +import { Claims } from '../../resolvers/types' export interface Message { type?: EntityType @@ -23,12 +25,11 @@ export interface Message { } interface ImportEvent { - userId: string integrationId: string } const isImportEvent = (event: any): event is ImportEvent => - 'userId' in event && 'integrationId' in event + 'integrationId' in event const logger = buildLogger('app.dispatch') @@ -169,56 +170,57 @@ export function integrationsServiceRouter() { res.status(500).send(err) } }) - // import pages from integration + // import pages from integration task handler router.post('/import', async (req, res) => { - logger.info('start to import pages from integration') - const { message: msgStr, expired } = readPushSubscription(req) - - if (!msgStr) { - return res.status(400).send('Bad Request') + logger.info('start cloud task to import pages from integration') + const token = req.cookies?.auth || req.headers?.authorization + let claims: Claims | undefined + try { + claims = await getClaimsByToken(token) + if (!claims) { + return res.status(401).send('UNAUTHORIZED') + } + } catch (err) { + logger.error('failed to get claims from token', err) + return res.status(401).send('UNAUTHORIZED') } - if (expired) { - logger.info('discarding expired message') - return res.status(200).send('Expired') - } - - const data = JSON.parse(msgStr) - if (!isImportEvent(data)) { + if (!isImportEvent(req.body)) { logger.info('Invalid message') return res.status(400).send('Bad Request') } - const userId = data.userId - const integration = await getRepository(Integration).findOneBy({ - user: { id: userId }, - id: data.integrationId, - enabled: true, - type: IntegrationType.Import, - }) - if (!integration) { - logger.info('No active integration found for user', { userId }) - return res.status(200).send('No integration found') - } - - const integrationService = getIntegrationService(integration.name) - // import pages from integration - logger.info('importing pages from integration', { - integrationId: integration.id, - }) - - // write the list of urls to a csv file and upload it to gcs - // path style: imports///-.csv - const dateStr = DateTime.now().toISODate() - const fileUuid = uuidv4() - const fullPath = `imports/${integration.user.id}/${dateStr}/URL_LIST-${fileUuid}.csv` - // open a write_stream to the file - const file = createGCSFile(fullPath) - const writeStream = file.createWriteStream({ - contentType: 'text/csv', - }) - + let writeStream: NodeJS.WritableStream | undefined try { + const userId = claims.uid + const integration = await getRepository(Integration).findOneBy({ + user: { id: userId }, + id: req.body.integrationId, + enabled: true, + type: IntegrationType.Import, + }) + if (!integration) { + logger.info('No active integration found for user', { userId }) + return res.status(200).send('No integration found') + } + + const integrationService = getIntegrationService(integration.name) + // import pages from integration + logger.info('importing pages from integration', { + integrationId: integration.id, + }) + + // write the list of urls to a csv file and upload it to gcs + // path style: imports///-.csv + const dateStr = DateTime.now().toISODate() + const fileUuid = uuidv4() + const fullPath = `imports/${userId}/${dateStr}/URL_LIST-${fileUuid}.csv` + // open a write_stream to the file + const file = createGCSFile(fullPath) + writeStream = file.createWriteStream({ + contentType: 'text/csv', + }) + let hasMore = true let offset = 0 let since = integration.syncedAt?.getTime() || 0 @@ -246,16 +248,16 @@ export function integrationsServiceRouter() { } // update the integration's syncedAt await getRepository(Integration).update(integration.id, { - syncedAt: since, + syncedAt: new Date(since), }) - - res.status(200).send('OK') } catch (err) { logger.error('import pages from integration failed', err) - res.status(500).send(err) + return res.status(500).send(err) } finally { - writeStream.end() + writeStream?.end() } + + res.status(200).send('OK') }) return router diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index d91e63030..e4c17051c 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -10,12 +10,12 @@ export interface RetrievedData { export interface RetrievedResult { data: RetrievedData[] hasMore?: boolean - since?: number + since?: number // unix timestamp in milliseconds } export interface RetrieveRequest { token: string - since?: number + since?: number // unix timestamp in milliseconds count?: number offset?: number } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index f16b4fd48..43e3646f2 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -8,12 +8,12 @@ import axios from 'axios' import { env } from '../../env' interface PocketResponse { - status: number - complete: number + status: number // 1 if success + complete: number // 1 if all items have been returned list: { [key: string]: PocketItem } - since: number + since: number // unix timestamp in seconds search_meta: { search_type: string } @@ -31,10 +31,10 @@ interface PocketItem { status: string excerpt: string word_count: string - tags: { + tags?: { [key: string]: Tag } - authors: { + authors?: { [key: string]: Author } } @@ -80,7 +80,7 @@ export class PocketIntegration extends IntegrationService { retrievePocketData = async ( accessToken: string, - since: number, + since: number, // unix timestamp in seconds count = 100, offset = 0 ): Promise => { @@ -106,7 +106,7 @@ export class PocketIntegration extends IntegrationService { return response.data } catch (error) { console.log('error retrieving pocket data', error) - throw error + throw new Error('Error retrieving pocket data') } } @@ -130,7 +130,7 @@ export class PocketIntegration extends IntegrationService { } const data = pocketItems.map((item) => ({ url: item.given_url, - labels: Object.values(item.tags).map((tag) => tag.tag), + labels: Object.values(item.tags ?? {}).map((tag) => tag.tag), state: statusToState[item.status], })) return { diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index f314b256d..b6b16262b 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -443,14 +443,17 @@ export const enqueueRecommendation = async ( export const enqueueImportFromIntegration = async ( userId: string, - integrationId: string + integrationId: string, + authToken: string ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { - userId, integrationId, } + const headers = { + Cookie: `auth=${authToken}`, + } // If there is no Google Cloud Project Id exposed, it means that we are in local environment if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { return nanoid() @@ -461,6 +464,7 @@ export const enqueueImportFromIntegration = async ( payload, taskHandlerUrl: `${env.queue.integrationTaskHandlerUrl}/import`, priority: 'low', + requestHeaders: headers, }) if (!createdTasks || !createdTasks[0].name) { diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index c25b31715..22c3e5f37 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -377,38 +377,10 @@ describe('Integrations resolvers', () => { name: 'POCKET', token: 'fakeToken', }) - - // nock('https://getpocket.com', { - // reqheaders: { - // 'content-type': 'application/json', - // }, - // }) - // .post('/v3/get', { - // access_token: existingIntegration.token, - // consumer_key: '', - // state: 'all', - // detailType: 'complete', - // since: 0, - // sort: 'oldest', - // count: 100, - // offset: 0, - // }) - // .reply(200, { - // list: { - // '123': { - // given_url: 'https://omnivore.app/pocket-import-test', - // }, - // }, - // }) - // - // sinon.replace(uploads, 'uploadToBucket', () => { - // return Promise.resolve() - // }) }) after(async () => { await deleteTestIntegrations(loginUser.id, [existingIntegration.id]) - // sinon.restore() }) it('returns success and starts cloud task', async () => { diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 9ff06905d..961aec012 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -8,7 +8,7 @@ import { } from '../../src/datalayer/pubsub' import { User } from '../../src/entity/user' import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' -import { Integration } from '../../src/entity/integration' +import { Integration, IntegrationType } from '../../src/entity/integration' import { getRepository } from '../../src/entity/utils' import { Highlight, @@ -21,13 +21,33 @@ import { addHighlightToPage } from '../../src/elastic/highlights' import { getHighlightUrl } from '../../src/services/highlights' import { deletePage } from '../../src/elastic/pages' import { READWISE_API_URL } from '../../src/services/integrations/readwise' +import sinon from 'sinon' +import { Storage } from '@google-cloud/storage' +import { MockBucket } from '../mock_storage' describe('Integrations routers', () => { + const baseUrl = '/svc/pubsub/integrations' let token: string + let user: User + let authToken: string + + before(async () => { + user = await createTestUser('fakeUser') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + const body = res.body as { authToken: string } + authToken = body.authToken + }) + + after(async () => { + await deleteTestUser(user.id) + }) describe('sync with integrations', () => { const endpoint = (token: string, name = 'name', action = 'action') => - `/svc/pubsub/integrations/${name}/${action}?token=${token}` + `${baseUrl}/${name}/${action}?token=${token}` let action: string let data: PubSubRequestBody let integrationName: string @@ -83,16 +103,6 @@ describe('Integrations routers', () => { }) context('when user exists', () => { - let user: User - - before(async () => { - user = await createTestUser('fakeUser') - }) - - after(async () => { - await deleteTestUser(user.id) - }) - context('when integration not found', () => { before(() => { integrationName = 'READWISE' @@ -323,4 +333,67 @@ describe('Integrations routers', () => { }) }) }) + + describe('import from integrations router', () => { + let integration: Integration + + before(async () => { + token = 'test token' + // create integration + integration = await getRepository(Integration).save({ + user: { id: user.id }, + name: 'POCKET', + token, + type: IntegrationType.Import, + }) + // mock cloud storage bucket + sinon.stub(Storage, 'Bucket').returns(MockBucket) + // mock Pocket API + nock('https://getpocket.com', { + reqheaders: { + 'content-type': 'application/json', + 'x-accept': 'application/json', + }, + }) + .post('/v3/get', { + access_token: token, + consumer_key: process.env.POCKET_CONSUMER_KEY, + state: 'all', + detailType: 'complete', + since: 0, + sort: 'oldest', + count: 100, + offset: 0, + }) + .reply(200, { + complete: 1, + list: { + '123': { + given_url: 'https://omnivore.app/pocket-import-test', + state: '0', + }, + }, + since: Date.now() / 1000, + }) + }) + + after(async () => { + sinon.restore() + await deleteTestIntegrations(user.id, [integration.id]) + }) + + context('when integration is pocket', () => { + it('returns 200 with OK', async () => { + const res = await request + .post(`${baseUrl}/import`) + .send({ + integrationId: integration.id, + }) + .set('Cookie', `auth=${authToken}`) + .expect(200) + + expect(res.text).to.eql('OK') + }) + }) + }) }) From 76b63efdc14b60fa571a5554ef5ba533576f6506 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 2 Mar 2023 11:02:53 +0800 Subject: [PATCH 31/48] Fix tests --- packages/api/src/utils/uploads.ts | 2 +- packages/api/test/mock_storage.ts | 4 ++-- packages/api/test/routers/integrations.test.ts | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 4c73c8780..5abee4dc1 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -9,7 +9,7 @@ import { env } from '../env' * the default app engine service account on the IAM page. We also need to * enable IAM related APIs on the project. */ -const storage = env.fileUpload?.gcsUploadSAKeyFilePath +export const storage = env.fileUpload?.gcsUploadSAKeyFilePath ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath }) : new Storage() const bucketName = env.fileUpload.gcsUploadBucket diff --git a/packages/api/test/mock_storage.ts b/packages/api/test/mock_storage.ts index 00415ab6b..d0dad7340 100644 --- a/packages/api/test/mock_storage.ts +++ b/packages/api/test/mock_storage.ts @@ -1,6 +1,6 @@ import { Writable } from 'stream' -class MockStorage { +export class MockStorage { buckets: { [name: string]: MockBucket } constructor() { @@ -12,7 +12,7 @@ class MockStorage { } } -export class MockBucket { +class MockBucket { name: string files: { [path: string]: MockFile } diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 961aec012..847aa37aa 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -22,8 +22,8 @@ import { getHighlightUrl } from '../../src/services/highlights' import { deletePage } from '../../src/elastic/pages' import { READWISE_API_URL } from '../../src/services/integrations/readwise' import sinon from 'sinon' -import { Storage } from '@google-cloud/storage' -import { MockBucket } from '../mock_storage' +import { MockStorage } from '../mock_storage' +import * as uploads from '../../src/utils/uploads' describe('Integrations routers', () => { const baseUrl = '/svc/pubsub/integrations' @@ -347,7 +347,8 @@ describe('Integrations routers', () => { type: IntegrationType.Import, }) // mock cloud storage bucket - sinon.stub(Storage, 'Bucket').returns(MockBucket) + // @ts-ignore + sinon.replace(uploads, 'storage', new MockStorage('test-bucket')) // mock Pocket API nock('https://getpocket.com', { reqheaders: { From 06b6583f298e0688c27f10622be0fa8a8aa4ae43 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 2 Mar 2023 22:21:09 +0800 Subject: [PATCH 32/48] Mock cloud storage bucket --- packages/api/src/utils/uploads.ts | 2 +- packages/api/test/mock_storage.ts | 4 ++-- .../api/test/routers/integrations.test.ts | 19 ++++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 5abee4dc1..4c73c8780 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -9,7 +9,7 @@ import { env } from '../env' * the default app engine service account on the IAM page. We also need to * enable IAM related APIs on the project. */ -export const storage = env.fileUpload?.gcsUploadSAKeyFilePath +const storage = env.fileUpload?.gcsUploadSAKeyFilePath ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath }) : new Storage() const bucketName = env.fileUpload.gcsUploadBucket diff --git a/packages/api/test/mock_storage.ts b/packages/api/test/mock_storage.ts index d0dad7340..00415ab6b 100644 --- a/packages/api/test/mock_storage.ts +++ b/packages/api/test/mock_storage.ts @@ -1,6 +1,6 @@ import { Writable } from 'stream' -export class MockStorage { +class MockStorage { buckets: { [name: string]: MockBucket } constructor() { @@ -12,7 +12,7 @@ export class MockStorage { } } -class MockBucket { +export class MockBucket { name: string files: { [path: string]: MockFile } diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 847aa37aa..2947a1476 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -22,8 +22,8 @@ import { getHighlightUrl } from '../../src/services/highlights' import { deletePage } from '../../src/elastic/pages' import { READWISE_API_URL } from '../../src/services/integrations/readwise' import sinon from 'sinon' -import { MockStorage } from '../mock_storage' -import * as uploads from '../../src/utils/uploads' +import { Storage } from '@google-cloud/storage' +import { MockBucket } from '../mock_storage' describe('Integrations routers', () => { const baseUrl = '/svc/pubsub/integrations' @@ -346,9 +346,7 @@ describe('Integrations routers', () => { token, type: IntegrationType.Import, }) - // mock cloud storage bucket - // @ts-ignore - sinon.replace(uploads, 'storage', new MockStorage('test-bucket')) + // mock Pocket API nock('https://getpocket.com', { reqheaders: { @@ -385,7 +383,14 @@ describe('Integrations routers', () => { context('when integration is pocket', () => { it('returns 200 with OK', async () => { - const res = await request + // mock cloud storage + const mockBucket = new MockBucket('test') + sinon.replace( + Storage.prototype, + 'bucket', + sinon.fake.returns(mockBucket as never) + ) + await request .post(`${baseUrl}/import`) .send({ integrationId: integration.id, @@ -393,7 +398,7 @@ describe('Integrations routers', () => { .set('Cookie', `auth=${authToken}`) .expect(200) - expect(res.text).to.eql('OK') + sinon.restore() }) }) }) From 0204e27aee8e2ab9d3c097a4b1ef45ca69e57628 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Mar 2023 08:21:08 +0800 Subject: [PATCH 33/48] mock before test --- .../api/test/routers/integrations.test.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 2947a1476..15e43cd5c 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -24,6 +24,7 @@ import { READWISE_API_URL } from '../../src/services/integrations/readwise' import sinon from 'sinon' import { Storage } from '@google-cloud/storage' import { MockBucket } from '../mock_storage' +import { env } from '../../src/env' describe('Integrations routers', () => { const baseUrl = '/svc/pubsub/integrations' @@ -356,7 +357,7 @@ describe('Integrations routers', () => { }) .post('/v3/get', { access_token: token, - consumer_key: process.env.POCKET_CONSUMER_KEY, + consumer_key: env.pocket.consumerKey, state: 'all', detailType: 'complete', since: 0, @@ -374,6 +375,14 @@ describe('Integrations routers', () => { }, since: Date.now() / 1000, }) + + // mock cloud storage + const mockBucket = new MockBucket('test') + sinon.replace( + Storage.prototype, + 'bucket', + sinon.fake.returns(mockBucket as never) + ) }) after(async () => { @@ -383,22 +392,13 @@ describe('Integrations routers', () => { context('when integration is pocket', () => { it('returns 200 with OK', async () => { - // mock cloud storage - const mockBucket = new MockBucket('test') - sinon.replace( - Storage.prototype, - 'bucket', - sinon.fake.returns(mockBucket as never) - ) - await request + return request .post(`${baseUrl}/import`) .send({ integrationId: integration.id, }) .set('Cookie', `auth=${authToken}`) .expect(200) - - sinon.restore() }) }) }) From 9a46e935d9eb31f86fd352c7d14139c72569a8aa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Mar 2023 08:35:11 +0800 Subject: [PATCH 34/48] Add [] to the labels in csv --- packages/api/src/routers/svc/integrations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 2cd89229d..ec6d20bef 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -238,7 +238,7 @@ export function integrationsServiceRouter() { // write the list of urls, state and labels to the stream const csvData = retrievedData.map((page) => { const { url, state, labels } = page - return [url, state, labels?.join(',')].join(',') + return [url, state, `[${labels?.join(',') || ''}]`].join(',') }) writeStream.write(csvData.join('\n')) From 4b578bbebba974f80524b784881b25801648cdb3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Mar 2023 09:00:42 +0800 Subject: [PATCH 35/48] Process labels and state in the csv file --- packages/api/src/routers/svc/integrations.ts | 2 +- packages/import-handler/src/csv.ts | 5 ++- packages/import-handler/src/index.ts | 11 ++++- packages/import-handler/test/csv/csv.test.ts | 43 ++++++++++++++++++- .../import-handler/test/csv/data/complex.csv | 2 + packages/import-handler/test/util.ts | 9 +++- 6 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 packages/import-handler/test/csv/data/complex.csv diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index ec6d20bef..981028576 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -238,7 +238,7 @@ export function integrationsServiceRouter() { // write the list of urls, state and labels to the stream const csvData = retrievedData.map((page) => { const { url, state, labels } = page - return [url, state, `[${labels?.join(',') || ''}]`].join(',') + return [url, state, `"[${labels?.join(',') || ''}]"`].join(',') }) writeStream.write(csvData.join('\n')) diff --git a/packages/import-handler/src/csv.ts b/packages/import-handler/src/csv.ts index c505cc39a..66835127c 100644 --- a/packages/import-handler/src/csv.ts +++ b/packages/import-handler/src/csv.ts @@ -13,7 +13,10 @@ export const importCsv = async (ctx: ImportContext, stream: Stream) => { for await (const row of parser) { try { const url = new URL(row[0]) - await ctx.urlHandler(ctx, url) + const state = row.length > 1 ? row[1] : undefined + // labels follows format: "[label1, label2]" + const labels = row.length > 2 ? row[2].slice(1, -1).split(',') : undefined + await ctx.urlHandler(ctx, url, state, labels) ctx.countImported += 1 } catch (error) { console.log('invalid url', row, error) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index ded094b5d..209edd852 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -1,7 +1,7 @@ import { Storage } from '@google-cloud/storage' import { importCsv } from './csv' import * as path from 'path' -import { importMatterArchive, importMatterHistoryCsv } from './matterHistory' +import { importMatterArchive } from './matterHistory' import { Stream } from 'node:stream' import { v4 as uuid } from 'uuid' import { CONTENT_FETCH_URL, createCloudTask, emailUserUrl } from './task' @@ -13,6 +13,8 @@ import { Readability } from '@omnivore/readability' import * as Sentry from '@sentry/serverless' +export type RetrievedDataState = 'archived' | 'saved' | 'deleted' + Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, tracesSampleRate: 0, @@ -24,7 +26,12 @@ const storage = new Storage() const CONTENT_TYPES = ['text/csv', 'application/zip'] -export type UrlHandler = (ctx: ImportContext, url: URL) => Promise +export type UrlHandler = ( + ctx: ImportContext, + url: URL, + state?: RetrievedDataState, + labels?: string[] +) => Promise export type ContentHandler = ( ctx: ImportContext, url: URL, diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 0f695d69e..8526d25d5 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -4,7 +4,7 @@ import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' import { importCsv } from '../../src/csv' -import { ImportContext } from '../../src' +import { ImportContext, RetrievedDataState } from '../../src' import { stubImportCtx } from '../util' chai.use(chaiString) @@ -28,3 +28,44 @@ describe('Load a simple CSV file', () => { ]) }) }) + +describe('Load a complex CSV file', () => { + it('should call the handler for each URL, state and labels', async () => { + const results: { + url: URL + state?: RetrievedDataState + labels?: string[] + }[] = [] + const stream = fs.createReadStream('./test/csv/data/complex.csv') + const stub = stubImportCtx() + stub.urlHandler = ( + ctx: ImportContext, + url, + state, + labels + ): Promise => { + results.push({ + url, + state, + labels, + }) + return Promise.resolve() + } + + await importCsv(stub, stream) + expect(stub.countFailed).to.equal(0) + expect(stub.countImported).to.equal(2) + expect(results).to.eql([ + { + url: new URL('https://omnivore.app'), + state: 'archived', + labels: ['test'], + }, + { + url: new URL('https://google.com'), + state: 'saved', + labels: ['test', 'development'], + }, + ]) + }) +}) diff --git a/packages/import-handler/test/csv/data/complex.csv b/packages/import-handler/test/csv/data/complex.csv new file mode 100644 index 000000000..b2aa9cd64 --- /dev/null +++ b/packages/import-handler/test/csv/data/complex.csv @@ -0,0 +1,2 @@ +"https://omnivore.app",archived,"[test]" +"https://google.com",saved,"[test,development]" diff --git a/packages/import-handler/test/util.ts b/packages/import-handler/test/util.ts index 58cd9dd10..68027ad33 100644 --- a/packages/import-handler/test/util.ts +++ b/packages/import-handler/test/util.ts @@ -1,12 +1,17 @@ import { Readability } from '@omnivore/readability' -import { ImportContext } from '../src' +import { ImportContext, RetrievedDataState } from '../src' export const stubImportCtx = () => { return { userId: '', countImported: 0, countFailed: 0, - urlHandler: (ctx: ImportContext, url: URL): Promise => { + urlHandler: ( + ctx: ImportContext, + url: URL, + state?: RetrievedDataState, + labels?: string[] + ): Promise => { return Promise.resolve() }, contentHandler: ( From 52399b37027d499ca0cb8faab22a799b83a88a80 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Mar 2023 09:03:43 +0800 Subject: [PATCH 36/48] Add state and labels to the cloud task --- packages/import-handler/src/index.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index 209edd852..d1cfd9ebb 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -73,13 +73,17 @@ const shouldHandle = (data: StorageEvent) => { const importURL = async ( userId: string, url: URL, - source: string + source: string, + state?: RetrievedDataState, + labels?: string[] ): Promise => { return createCloudTask(CONTENT_FETCH_URL, { userId, source, url: url.toString(), saveRequestId: uuid(), + state, + labels, }) } @@ -129,10 +133,21 @@ const handlerForFile = (name: string): importHandlerFunc | undefined => { return undefined } -const urlHandler = async (ctx: ImportContext, url: URL): Promise => { +const urlHandler = async ( + ctx: ImportContext, + url: URL, + state?: RetrievedDataState, + labels?: string[] +): Promise => { try { // Imports are stored in the format imports//-.csv - const result = await importURL(ctx.userId, url, 'csv-importer') + const result = await importURL( + ctx.userId, + url, + 'csv-importer', + state, + labels + ) if (result) { ctx.countImported += 1 } From 5c46903fbe7caf61c628e0da5b57784ca93c2f8f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Mar 2023 09:06:40 +0800 Subject: [PATCH 37/48] Add labels and state to puppeteer-parse --- packages/puppeteer-parse/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 1ac32eb14..2f25293b9 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -248,6 +248,8 @@ async function fetchContent(req, res) { let url = getUrl(req); const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); + const state = req.body.state + const labels = req.body.labels let logRecord = { url, @@ -256,6 +258,8 @@ async function fetchContent(req, res) { labels: { source: 'parseContent', }, + state, + labelsToAdd: labels }; console.info(`Article parsing request`, logRecord); @@ -367,6 +371,8 @@ async function fetchContent(req, res) { title, originalContent: content, parseResult: readabilityResult, + state, + labels, }); logRecord.totalTime = Date.now() - functionStartTime; From fcf08fc36462e04172ed56d20121129966f33861 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 3 Mar 2023 09:27:38 +0800 Subject: [PATCH 38/48] Add state and labels to savePageInput --- packages/api/src/elastic/types.ts | 2 ++ packages/api/src/generated/graphql.ts | 3 +++ packages/api/src/generated/schema.graphql | 3 +++ packages/api/src/schema.ts | 3 +++ .../api/src/services/integrations/integration.ts | 5 ++--- packages/api/src/services/integrations/pocket.ts | 10 +++++----- packages/api/src/services/save_page.ts | 16 ++++++++++++++-- packages/import-handler/src/index.ts | 15 +++++++++++---- packages/import-handler/test/csv/csv.test.ts | 8 ++++---- .../import-handler/test/csv/data/complex.csv | 4 ++-- packages/import-handler/test/util.ts | 4 ++-- 11 files changed, 51 insertions(+), 22 deletions(-) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index eb38976c2..d2257748d 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -72,6 +72,8 @@ export enum ArticleSavingRequestStatus { Processing = 'PROCESSING', Succeeded = 'SUCCEEDED', Deleted = 'DELETED', + + Archived = 'ARCHIVED', } export enum HighlightType { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 656149b99..e78f0c88f 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -188,6 +188,7 @@ export enum ArticleSavingRequestErrorCode { export type ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess; export enum ArticleSavingRequestStatus { + Archived = 'ARCHIVED', Deleted = 'DELETED', Failed = 'FAILED', Processing = 'PROCESSING', @@ -2224,9 +2225,11 @@ export type SaveFilterSuccess = { export type SavePageInput = { clientRequestId: Scalars['ID']; + labels?: InputMaybe>>; originalContent: Scalars['String']; parseResult?: InputMaybe; source: Scalars['String']; + state?: InputMaybe; title?: InputMaybe; url: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 651eafbff..e4d52b4d0 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -153,6 +153,7 @@ enum ArticleSavingRequestErrorCode { union ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess enum ArticleSavingRequestStatus { + ARCHIVED DELETED FAILED PROCESSING @@ -1619,9 +1620,11 @@ type SaveFilterSuccess { input SavePageInput { clientRequestId: ID! + labels: [String] originalContent: String! parseResult: ParseResult source: String! + state: ArticleSavingRequestStatus title: String url: String! } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 7751f0c33..efd434e94 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -554,6 +554,8 @@ const schema = gql` title: String originalContent: String! parseResult: ParseResult + state: ArticleSavingRequestStatus + labels: [String] } input SaveUrlInput { @@ -1073,6 +1075,7 @@ const schema = gql` SUCCEEDED FAILED DELETED + ARCHIVED } type ArticleSavingRequest { diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index e4c17051c..3c2a44a08 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -1,11 +1,10 @@ import { Integration } from '../../entity/integration' -import { Page } from '../../elastic/types' +import { ArticleSavingRequestStatus, Page } from '../../elastic/types' -export type RetrievedDataState = 'archived' | 'saved' | 'deleted' export interface RetrievedData { url: string labels?: string[] - state?: RetrievedDataState + state?: ArticleSavingRequestStatus } export interface RetrievedResult { data: RetrievedData[] diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 43e3646f2..0b25928ab 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,11 +1,11 @@ import { IntegrationService, - RetrievedDataState, RetrievedResult, RetrieveRequest, } from './integration' import axios from 'axios' import { env } from '../../env' +import { ArticleSavingRequestStatus } from '../../elastic/types' interface PocketResponse { status: number // 1 if success @@ -123,10 +123,10 @@ export class PocketIntegration extends IntegrationService { offset ) const pocketItems = Object.values(pocketData.list) - const statusToState: Record = { - '0': 'saved', - '1': 'archived', - '2': 'deleted', + const statusToState: Record = { + '0': ArticleSavingRequestStatus.Succeeded, + '1': ArticleSavingRequestStatus.Archived, + '2': ArticleSavingRequestStatus.Deleted, } const data = pocketItems.map((item) => ({ url: item.given_url, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 2158d1ace..2066a029e 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -107,6 +107,9 @@ export const savePage = async ( userId: saver.userId, url: articleToSave.url, }) + const archivedAt = + input.state === ArticleSavingRequestStatus.Archived ? new Date() : null + if (existingPage) { pageId = existingPage.id slug = existingPage.slug @@ -116,7 +119,7 @@ export const savePage = async ( { // update the page with the new content ...articleToSave, - archivedAt: null, // unarchive if it was archived + archivedAt, // unarchive if it was archived id: pageId, // we don't want to update the id slug, // we don't want to update the slug createdAt: existingPage.createdAt, // we don't want to update the createdAt @@ -145,7 +148,13 @@ export const savePage = async ( } } } else { - const newPageId = await createPage(articleToSave, ctx) + const newPageId = await createPage( + { + ...articleToSave, + archivedAt, + }, + ctx + ) if (!newPageId) { return { errorCodes: [SaveErrorCode.Unknown], @@ -177,6 +186,9 @@ export const savePage = async ( } } } + // TODO: add labels to page + // if (pageId && input.labels) { + // } return { clientRequestId: pageId, diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index d1cfd9ebb..f02557156 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -13,7 +13,14 @@ import { Readability } from '@omnivore/readability' import * as Sentry from '@sentry/serverless' -export type RetrievedDataState = 'archived' | 'saved' | 'deleted' +export enum ArticleSavingRequestStatus { + Failed = 'FAILED', + Processing = 'PROCESSING', + Succeeded = 'SUCCEEDED', + Deleted = 'DELETED', + + Archived = 'ARCHIVED', +} Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -29,7 +36,7 @@ const CONTENT_TYPES = ['text/csv', 'application/zip'] export type UrlHandler = ( ctx: ImportContext, url: URL, - state?: RetrievedDataState, + state?: ArticleSavingRequestStatus, labels?: string[] ) => Promise export type ContentHandler = ( @@ -74,7 +81,7 @@ const importURL = async ( userId: string, url: URL, source: string, - state?: RetrievedDataState, + state?: ArticleSavingRequestStatus, labels?: string[] ): Promise => { return createCloudTask(CONTENT_FETCH_URL, { @@ -136,7 +143,7 @@ const handlerForFile = (name: string): importHandlerFunc | undefined => { const urlHandler = async ( ctx: ImportContext, url: URL, - state?: RetrievedDataState, + state?: ArticleSavingRequestStatus, labels?: string[] ): Promise => { try { diff --git a/packages/import-handler/test/csv/csv.test.ts b/packages/import-handler/test/csv/csv.test.ts index 8526d25d5..e87d5aa10 100644 --- a/packages/import-handler/test/csv/csv.test.ts +++ b/packages/import-handler/test/csv/csv.test.ts @@ -4,7 +4,7 @@ import { expect } from 'chai' import chaiString from 'chai-string' import * as fs from 'fs' import { importCsv } from '../../src/csv' -import { ImportContext, RetrievedDataState } from '../../src' +import { ArticleSavingRequestStatus, ImportContext } from '../../src' import { stubImportCtx } from '../util' chai.use(chaiString) @@ -33,7 +33,7 @@ describe('Load a complex CSV file', () => { it('should call the handler for each URL, state and labels', async () => { const results: { url: URL - state?: RetrievedDataState + state?: ArticleSavingRequestStatus labels?: string[] }[] = [] const stream = fs.createReadStream('./test/csv/data/complex.csv') @@ -58,12 +58,12 @@ describe('Load a complex CSV file', () => { expect(results).to.eql([ { url: new URL('https://omnivore.app'), - state: 'archived', + state: 'ARCHIVED', labels: ['test'], }, { url: new URL('https://google.com'), - state: 'saved', + state: 'SUCCEEDED', labels: ['test', 'development'], }, ]) diff --git a/packages/import-handler/test/csv/data/complex.csv b/packages/import-handler/test/csv/data/complex.csv index b2aa9cd64..c215da398 100644 --- a/packages/import-handler/test/csv/data/complex.csv +++ b/packages/import-handler/test/csv/data/complex.csv @@ -1,2 +1,2 @@ -"https://omnivore.app",archived,"[test]" -"https://google.com",saved,"[test,development]" +"https://omnivore.app",ARCHIVED,"[test]" +"https://google.com",SUCCEEDED,"[test,development]" diff --git a/packages/import-handler/test/util.ts b/packages/import-handler/test/util.ts index 68027ad33..a60564ae2 100644 --- a/packages/import-handler/test/util.ts +++ b/packages/import-handler/test/util.ts @@ -1,5 +1,5 @@ import { Readability } from '@omnivore/readability' -import { ImportContext, RetrievedDataState } from '../src' +import { ArticleSavingRequestStatus, ImportContext } from '../src' export const stubImportCtx = () => { return { @@ -9,7 +9,7 @@ export const stubImportCtx = () => { urlHandler: ( ctx: ImportContext, url: URL, - state?: RetrievedDataState, + state?: ArticleSavingRequestStatus, labels?: string[] ): Promise => { return Promise.resolve() From 87d3f5b3d8f77e51b4d10309fea7129836d4cdd5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sun, 5 Mar 2023 22:53:27 +0800 Subject: [PATCH 39/48] Add labels and state to savePage API --- packages/api/src/elastic/types.ts | 2 +- packages/api/src/entity/label.ts | 2 +- packages/api/src/generated/graphql.ts | 5 +-- packages/api/src/generated/schema.graphql | 5 +-- packages/api/src/resolvers/labels/index.ts | 4 +-- packages/api/src/schema.ts | 5 +-- packages/api/src/services/labels.ts | 42 +++++++++++++++++++++- packages/api/src/services/save_page.ts | 20 ++++++----- packages/import-handler/src/index.ts | 4 ++- 9 files changed, 68 insertions(+), 21 deletions(-) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index d2257748d..320604c3b 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -86,7 +86,7 @@ export interface Label { id: string name: string color: string - description?: string + description?: string | null createdAt?: Date } diff --git a/packages/api/src/entity/label.ts b/packages/api/src/entity/label.ts index 30d145f5d..0088e3319 100644 --- a/packages/api/src/entity/label.ts +++ b/packages/api/src/entity/label.ts @@ -24,7 +24,7 @@ export class Label { color!: string @Column('text', { nullable: true }) - description?: string + description?: string | null @CreateDateColumn() createdAt!: Date diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index e78f0c88f..0b98217dd 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -402,7 +402,7 @@ export enum CreateLabelErrorCode { } export type CreateLabelInput = { - color: Scalars['String']; + color?: InputMaybe; description?: InputMaybe; name: Scalars['String']; }; @@ -2187,6 +2187,7 @@ export type SaveError = { export enum SaveErrorCode { EmbeddedHighlightFailed = 'EMBEDDED_HIGHLIGHT_FAILED', + EmbeddedLabelFailed = 'EMBEDDED_LABEL_FAILED', Unauthorized = 'UNAUTHORIZED', Unknown = 'UNKNOWN' } @@ -2225,7 +2226,7 @@ export type SaveFilterSuccess = { export type SavePageInput = { clientRequestId: Scalars['ID']; - labels?: InputMaybe>>; + labels?: InputMaybe>; originalContent: Scalars['String']; parseResult?: InputMaybe; source: Scalars['String']; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index e4d52b4d0..4d95ef831 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -350,7 +350,7 @@ enum CreateLabelErrorCode { } input CreateLabelInput { - color: String! + color: String description: String name: String! } @@ -1584,6 +1584,7 @@ type SaveError { enum SaveErrorCode { EMBEDDED_HIGHLIGHT_FAILED + EMBEDDED_LABEL_FAILED UNAUTHORIZED UNKNOWN } @@ -1620,7 +1621,7 @@ type SaveFilterSuccess { input SavePageInput { clientRequestId: ID! - labels: [String] + labels: [CreateLabelInput!] originalContent: String! parseResult: ParseResult source: String! diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index 31bf420f8..b27c9bf3b 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -1,4 +1,4 @@ -import { authorized } from '../../utils/helpers' +import { authorized, generateRandomColor } from '../../utils/helpers' import { CreateLabelError, CreateLabelErrorCode, @@ -114,7 +114,7 @@ export const createLabelResolver = authorized< const label = await getRepository(Label).save({ user, name, - color, + color: color || generateRandomColor(), description: description || '', }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index efd434e94..2bfee3e6e 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -513,6 +513,7 @@ const schema = gql` UNKNOWN UNAUTHORIZED EMBEDDED_HIGHLIGHT_FAILED + EMBEDDED_LABEL_FAILED } type SaveError { @@ -555,7 +556,7 @@ const schema = gql` originalContent: String! parseResult: ParseResult state: ArticleSavingRequestStatus - labels: [String] + labels: [CreateLabelInput!] } input SaveUrlInput { @@ -1430,7 +1431,7 @@ const schema = gql` input CreateLabelInput { name: String! @sanitize(maxLength: 64) - color: String! @sanitize(pattern: "^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$") + color: String @sanitize(pattern: "^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$") description: String @sanitize(maxLength: 100) } diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index b20baeee2..dd7ffa31d 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -2,7 +2,7 @@ import { Label } from '../entity/label' import { ILike, In } from 'typeorm' import { PageContext } from '../elastic/types' import { User } from '../entity/user' -import { addLabelInPage } from '../elastic/labels' +import { addLabelInPage, updateLabelsInPage } from '../elastic/labels' import { getRepository } from '../entity/utils' import { Link } from '../entity/link' import DataLoader from 'dataloader' @@ -103,3 +103,43 @@ export const createLabel = async ( user: { id: userId }, }) } + +export const addLabelsToNewPage = async ( + ctx: PageContext, + pageId: string, + labels: { + name: string + color?: string | null + description?: string | null + }[] +): Promise => { + const user = await getRepository(User).findOneBy({ + id: ctx.uid, + }) + if (!user) { + console.log('user not found') + return false + } + + const labelEntities = await getRepository(Label).findBy({ + user: { id: user.id }, + name: In(labels.map((l) => l.name)), + }) + + const existingLabels = labelEntities.map((l) => l.name) + const newLabels = labels.filter((l) => !existingLabels.includes(l.name)) + // create new labels + const newLabelEntities = await getRepository(Label).save( + newLabels.map((l) => ({ + ...l, + color: l.color || generateRandomColor(), + user, + })) + ) + // add all labels to page + return updateLabelsInPage( + pageId, + [...newLabelEntities, ...labelEntities], + ctx + ) +} diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 2066a029e..2215f0761 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -22,6 +22,7 @@ import { } from '../utils/helpers' import { parsePreparedContent } from '../utils/parser' import { createPageSaveRequest } from './create_page_save_request' +import { addLabelsToNewPage } from './labels' type SaveContext = { pubsub: PubsubClient @@ -174,21 +175,22 @@ export const savePage = async ( type: HighlightType.Highlight, } - if ( - !(await addHighlightToPage(pageId, highlight, { - pubsub: ctx.pubsub, - uid: ctx.uid, - })) - ) { + if (!(await addHighlightToPage(pageId, highlight, ctx))) { return { errorCodes: [SaveErrorCode.EmbeddedHighlightFailed], message: 'Failed to save highlight', } } } - // TODO: add labels to page - // if (pageId && input.labels) { - // } + // add labels to page + if (pageId && input.labels) { + if (!(await addLabelsToNewPage(ctx, pageId, input.labels))) { + return { + errorCodes: [SaveErrorCode.EmbeddedLabelFailed], + message: 'Failed to save labels', + } + } + } return { clientRequestId: pageId, diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index f02557156..557a57a4a 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -90,7 +90,9 @@ const importURL = async ( url: url.toString(), saveRequestId: uuid(), state, - labels, + labels: labels?.map((l) => { + return { name: l } + }), }) } From ce273b5172acfb1a1da34391f714ad5be59205a6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Mar 2023 11:48:06 +0800 Subject: [PATCH 40/48] Add test case for archives page and saves labels in savePage API --- .../src/services/create_page_save_request.ts | 2 ++ packages/api/src/services/save_page.ts | 3 +- packages/api/test/resolvers/article.test.ts | 35 ++++++++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 7d2b504a5..f515d1e8c 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -64,6 +64,7 @@ export const createPageSaveRequest = async ( models: DataModels, pubsub: PubsubClient = createPubSubClient(), articleSavingRequestId = uuidv4(), + archivedAt?: Date | null, priority?: 'low' | 'high' ): Promise => { try { @@ -116,6 +117,7 @@ export const createPageSaveRequest = async ( state: ArticleSavingRequestStatus.Processing, createdAt: new Date(), savedAt: new Date(), + archivedAt, } // create processing page diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 2215f0761..ebca00339 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -140,7 +140,8 @@ export const savePage = async ( articleToSave.url, ctx.models, ctx.pubsub, - input.clientRequestId + input.clientRequestId, + archivedAt ) } catch (e) { return { diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index def6ab376..ceebb1f15 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -10,6 +10,7 @@ import { deletePage, deletePagesByParam, getPageById, + getPageByParam, updatePage, } from '../../src/elastic/pages' import { @@ -208,7 +209,13 @@ const searchQuery = (keyword = '') => { ` } -const savePageQuery = (url: string, title: string, originalContent: string) => { +const savePageQuery = ( + url: string, + title: string, + originalContent: string, + state: ArticleSavingRequestStatus | null = null, + labels: string[] | null = null +) => { return ` mutation { savePage( @@ -218,6 +225,12 @@ const savePageQuery = (url: string, title: string, originalContent: string) => { clientRequestId: "${generateFakeUuid()}", title: "${title}", originalContent: "${originalContent}" + state: ${state} + labels: ${ + labels + ? '[' + labels.map((label) => `{ name: "${label}" }`) + ']' + : null + } } ) { ... on SaveSuccess { @@ -611,6 +624,26 @@ describe('Article API', () => { expect(allLinks.body.data.articles.edges[0].node.url).to.eq(url) }) }) + + context('when we also want to save labels and archives the page', () => { + after(async () => { + await deletePagesByParam({ url }, ctx) + }) + + it('saves the labels and archives the page', async () => { + const state = ArticleSavingRequestStatus.Archived + const labels = ['test name', 'test name 2'] + await graphqlRequest( + savePageQuery(url, title, originalContent, state, labels), + authToken + ).expect(200) + await refreshIndex() + + const savedPage = await getPageByParam({ url }) + expect(savedPage?.archivedAt).to.not.be.null + expect(savedPage?.labels?.map((l) => l.name)).to.eql(labels) + }) + }) }) describe('SaveUrl', () => { From abd42f70648c0931d543c79d72ca6f53283ad8bc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Mar 2023 14:34:50 +0800 Subject: [PATCH 41/48] Add labels and state to saveUrl API --- packages/api/src/generated/graphql.ts | 5 +- packages/api/src/generated/schema.graphql | 5 +- packages/api/src/resolvers/article/index.ts | 4 +- .../resolvers/article_saving_request/index.ts | 8 +++- packages/api/src/routers/article_router.ts | 3 +- packages/api/src/routers/svc/links.ts | 9 ++-- packages/api/src/schema.ts | 5 +- .../src/services/create_page_save_request.ts | 37 +++++++++++---- packages/api/src/services/labels.ts | 25 ++++------ packages/api/src/services/save_file.ts | 10 +++- packages/api/src/services/save_page.ts | 34 +++++++------- packages/api/src/services/save_url.ts | 25 +++++++--- packages/api/src/utils/helpers.ts | 3 +- packages/api/test/resolvers/article.test.ts | 47 +++++++++++++++++-- 14 files changed, 147 insertions(+), 73 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 0b98217dd..da0ef3fba 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2187,14 +2187,15 @@ export type SaveError = { export enum SaveErrorCode { EmbeddedHighlightFailed = 'EMBEDDED_HIGHLIGHT_FAILED', - EmbeddedLabelFailed = 'EMBEDDED_LABEL_FAILED', Unauthorized = 'UNAUTHORIZED', Unknown = 'UNKNOWN' } export type SaveFileInput = { clientRequestId: Scalars['ID']; + labels?: InputMaybe>; source: Scalars['String']; + state?: InputMaybe; uploadFileId: Scalars['ID']; url: Scalars['String']; }; @@ -2245,7 +2246,9 @@ export type SaveSuccess = { export type SaveUrlInput = { clientRequestId: Scalars['ID']; + labels?: InputMaybe>; source: Scalars['String']; + state?: InputMaybe; url: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 4d95ef831..1ae8efa8d 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1584,14 +1584,15 @@ type SaveError { enum SaveErrorCode { EMBEDDED_HIGHLIGHT_FAILED - EMBEDDED_LABEL_FAILED UNAUTHORIZED UNKNOWN } input SaveFileInput { clientRequestId: ID! + labels: [CreateLabelInput!] source: String! + state: ArticleSavingRequestStatus uploadFileId: ID! url: String! } @@ -1639,7 +1640,9 @@ type SaveSuccess { input SaveUrlInput { clientRequestId: ID! + labels: [CreateLabelInput!] source: String! + state: ArticleSavingRequestStatus url: String! } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index abe36814b..c7e703c10 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -248,7 +248,7 @@ export const createArticleResolver = authorized< source !== 'puppeteer-parse' && FORCE_PUPPETEER_URLS.some((regex) => regex.test(url)) ) { - await createPageSaveRequest(uid, url, models) + await createPageSaveRequest({ userId: uid, url }) return DUMMY_RESPONSE } else if (!skipParsing && preparedDocument?.document) { const parseResults = await traceAs>( @@ -264,7 +264,7 @@ export const createArticleResolver = authorized< } else if (!preparedDocument?.document) { // We have a URL but no document, so we try to send this to puppeteer // and return a dummy response. - await createPageSaveRequest(uid, url, models) + await createPageSaveRequest({ userId: uid, url }) return DUMMY_RESPONSE } diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index a6ce2a8ba..2c431dfb0 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -25,7 +25,7 @@ export const createArticleSavingRequestResolver = authorized< CreateArticleSavingRequestSuccess, CreateArticleSavingRequestError, MutationCreateArticleSavingRequestArgs ->(async (_, { input: { url } }, { models, claims, pubsub }) => { +>(async (_, { input: { url } }, { claims, pubsub }) => { analytics.track({ userId: claims.uid, event: 'link_saved', @@ -37,7 +37,11 @@ export const createArticleSavingRequestResolver = authorized< }) try { - const request = await createPageSaveRequest(claims.uid, url, models, pubsub) + const request = await createPageSaveRequest({ + userId: claims.uid, + url, + pubsub, + }) return { articleSavingRequest: request, } diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 988a94a2b..dc28281c1 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -59,8 +59,7 @@ export function articleRouter() { return res.status(400).send({ errorCode: 'BAD_DATA' }) } - const models = initModels(kx, false) - const result = await createPageSaveRequest(uid, url, models) + const result = await createPageSaveRequest({ userId: uid, url }) if (isSiteBlockedForParse(url)) { return res diff --git a/packages/api/src/routers/svc/links.ts b/packages/api/src/routers/svc/links.ts index b656203c0..4e6c23ed0 100644 --- a/packages/api/src/routers/svc/links.ts +++ b/packages/api/src/routers/svc/links.ts @@ -3,9 +3,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { kx } from '../../datalayer/knex_config' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { initModels } from '../../server' interface CreateLinkRequestMessage { url: string @@ -39,10 +37,11 @@ export function linkServiceRouter() { } const msg = data as CreateLinkRequestMessage - const models = initModels(kx, false) - try { - const request = await createPageSaveRequest(msg.userId, msg.url, models) + const request = await createPageSaveRequest({ + userId: msg.userId, + url: msg.url, + }) console.log('create link request', request) res.status(200).send(request) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 2bfee3e6e..a09570776 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -513,7 +513,6 @@ const schema = gql` UNKNOWN UNAUTHORIZED EMBEDDED_HIGHLIGHT_FAILED - EMBEDDED_LABEL_FAILED } type SaveError { @@ -531,6 +530,8 @@ const schema = gql` source: String! clientRequestId: ID! uploadFileId: ID! + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } input ParseResult { @@ -563,6 +564,8 @@ const schema = gql` url: String! source: String! clientRequestId: ID! + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } union SaveResult = SaveSuccess | SaveError diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index f515d1e8c..77c3748ee 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -8,7 +8,7 @@ import { getPageByParam, updatePage, } from '../elastic/pages' -import { ArticleSavingRequestStatus, PageType } from '../elastic/types' +import { ArticleSavingRequestStatus, Label, PageType } from '../elastic/types' import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, @@ -17,6 +17,19 @@ import { import { DataModels } from '../resolvers/types' import { enqueueParseRequest } from '../utils/createTask' import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers' +import * as privateIpLib from 'private-ip' +import { getRepository } from '../entity/utils' +import { User } from '../entity/user' + +interface PageSaveRequest { + userId: string + url: string + pubsub?: PubsubClient + articleSavingRequestId?: string + archivedAt?: Date | null + labels?: Label[] + priority?: 'low' | 'high' +} const SAVING_CONTENT = 'Your link is being saved...' @@ -58,15 +71,15 @@ export const validateUrl = (url: string): URL => { return u } -export const createPageSaveRequest = async ( - userId: string, - url: string, - models: DataModels, - pubsub: PubsubClient = createPubSubClient(), +export const createPageSaveRequest = async ({ + userId, + url, + pubsub = createPubSubClient(), articleSavingRequestId = uuidv4(), - archivedAt?: Date | null, - priority?: 'low' | 'high' -): Promise => { + archivedAt, + priority, + labels, +}: PageSaveRequest): Promise => { try { validateUrl(url) } catch (error) { @@ -76,7 +89,10 @@ export const createPageSaveRequest = async ( }) } - const user = await models.user.get(userId) + const user = await getRepository(User).findOne({ + where: { id: userId }, + relations: ['profile'], + }) if (!user) { console.log('User not found', userId) return Promise.reject({ @@ -118,6 +134,7 @@ export const createPageSaveRequest = async ( createdAt: new Date(), savedAt: new Date(), archivedAt, + labels, } // create processing page diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index dd7ffa31d..fee84bc46 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -2,11 +2,12 @@ import { Label } from '../entity/label' import { ILike, In } from 'typeorm' import { PageContext } from '../elastic/types' import { User } from '../entity/user' -import { addLabelInPage, updateLabelsInPage } from '../elastic/labels' +import { addLabelInPage } from '../elastic/labels' import { getRepository } from '../entity/utils' import { Link } from '../entity/link' import DataLoader from 'dataloader' import { generateRandomColor } from '../utils/helpers' +import { CreateLabelInput } from '../generated/graphql' const batchGetLabelsFromLinkIds = async ( linkIds: readonly string[] @@ -104,21 +105,16 @@ export const createLabel = async ( }) } -export const addLabelsToNewPage = async ( +export const createLabels = async ( ctx: PageContext, - pageId: string, - labels: { - name: string - color?: string | null - description?: string | null - }[] -): Promise => { + labels: CreateLabelInput[] +): Promise => { const user = await getRepository(User).findOneBy({ id: ctx.uid, }) if (!user) { - console.log('user not found') - return false + console.error('user not found') + return [] } const labelEntities = await getRepository(Label).findBy({ @@ -136,10 +132,5 @@ export const addLabelsToNewPage = async ( user, })) ) - // add all labels to page - return updateLabelsInPage( - pageId, - [...newLabelEntities, ...labelEntities], - ctx - ) + return [...labelEntities, ...newLabelEntities] } diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index 3664d71ad..7e12f7150 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -36,10 +36,18 @@ export const saveFile = async ( await getStorageFileDetails(input.uploadFileId, uploadFile.fileName) - await ctx.authTrx(async (tx) => { + const uploadFileData = await ctx.authTrx(async (tx) => { return ctx.models.uploadFile.setFileUploadComplete(input.uploadFileId, tx) }) + if (!uploadFileData) { + return { + errorCodes: [SaveErrorCode.Unknown], + } + } + + // TODO: save labels and archive state + return { clientRequestId: input.clientRequestId, url: `${homePageURL()}/${saver.profile.username}/links/${ diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index ebca00339..78edd5c09 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -22,7 +22,7 @@ import { } from '../utils/helpers' import { parsePreparedContent } from '../utils/parser' import { createPageSaveRequest } from './create_page_save_request' -import { addLabelsToNewPage } from './labels' +import { createLabels } from './labels' type SaveContext = { pubsub: PubsubClient @@ -108,8 +108,13 @@ export const savePage = async ( userId: saver.userId, url: articleToSave.url, }) + // save state const archivedAt = input.state === ArticleSavingRequestStatus.Archived ? new Date() : null + // add labels to page + const labels = input.labels + ? await createLabels(ctx, input.labels) + : undefined if (existingPage) { pageId = existingPage.id @@ -124,6 +129,7 @@ export const savePage = async ( id: pageId, // we don't want to update the id slug, // we don't want to update the slug createdAt: existingPage.createdAt, // we don't want to update the createdAt + labels, }, ctx )) @@ -135,14 +141,14 @@ export const savePage = async ( } } else if (shouldParseInBackend(input)) { try { - await createPageSaveRequest( - saver.userId, - articleToSave.url, - ctx.models, - ctx.pubsub, - input.clientRequestId, - archivedAt - ) + await createPageSaveRequest({ + userId: saver.userId, + url: articleToSave.url, + pubsub: ctx.pubsub, + articleSavingRequestId: input.clientRequestId, + archivedAt, + labels, + }) } catch (e) { return { errorCodes: [SaveErrorCode.Unknown], @@ -154,6 +160,7 @@ export const savePage = async ( { ...articleToSave, archivedAt, + labels, }, ctx ) @@ -183,15 +190,6 @@ export const savePage = async ( } } } - // add labels to page - if (pageId && input.labels) { - if (!(await addLabelsToNewPage(ctx, pageId, input.labels))) { - return { - errorCodes: [SaveErrorCode.EmbeddedLabelFailed], - message: 'Failed to save labels', - } - } - } return { clientRequestId: pageId, diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index ed26080dc..7b5695285 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -4,6 +4,8 @@ import { homePageURL } from '../env' import { SaveErrorCode, SaveResult, SaveUrlInput } from '../generated/graphql' import { DataModels } from '../resolvers/types' import { createPageSaveRequest } from './create_page_save_request' +import { ArticleSavingRequestStatus } from '../elastic/types' +import { createLabels } from './labels' type SaveContext = { pubsub: PubsubClient @@ -16,13 +18,22 @@ export const saveUrl = async ( input: SaveUrlInput ): Promise => { try { - const pageSaveRequest = await createPageSaveRequest( - saver.id, - input.url, - ctx.models, - ctx.pubsub, - input.clientRequestId - ) + // save state + const archivedAt = + input.state === ArticleSavingRequestStatus.Archived ? new Date() : null + // add labels to page + const labels = input.labels + ? await createLabels({ ...ctx, uid: saver.id }, input.labels) + : undefined + + const pageSaveRequest = await createPageSaveRequest({ + userId: saver.id, + url: input.url, + pubsub: ctx.pubsub, + articleSavingRequestId: input.clientRequestId, + archivedAt, + labels, + }) return { clientRequestId: pageSaveRequest.id, diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 76ad7776c..a51e120b1 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -18,6 +18,7 @@ import path from 'path' import normalizeUrl from 'normalize-url' import wordsCounter from 'word-counting' import _ from 'underscore' +import { User } from '../entity/user' interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -187,7 +188,7 @@ export const pageError = async ( } export const pageToArticleSavingRequest = ( - user: UserData, + user: User, page: Page ): ArticleSavingRequest => ({ ...page, diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index ceebb1f15..cbcbaddae 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -37,6 +37,8 @@ import { graphqlRequest, request, } from '../util' +import sinon from 'sinon' +import * as createTask from '../../src/utils/createTask' chai.use(chaiString) @@ -266,7 +268,11 @@ const saveFileQuery = (url: string, uploadFileId: string) => { ` } -const saveUrlQuery = (url: string) => { +const saveUrlQuery = ( + url: string, + state: ArticleSavingRequestStatus | null = null, + labels: string[] | null = null +) => { return ` mutation { saveUrl( @@ -274,6 +280,12 @@ const saveUrlQuery = (url: string) => { url: "${url}", source: "test", clientRequestId: "${generateFakeUuid()}", + state: ${state} + labels: ${ + labels + ? '[' + labels.map((label) => `{ name: "${label}" }`) + ']' + : null + } } ) { ... on SaveSuccess { @@ -650,15 +662,23 @@ describe('Article API', () => { let query = '' let url = 'https://blog.omnivore.app/new-url-1' + before(() => { + sinon.replace(createTask, 'enqueueParseRequest', sinon.fake.resolves('')) + }) + beforeEach(() => { query = saveUrlQuery(url) }) - context('when we save a new url', () => { - after(async () => { - await deletePagesByParam({ url }, ctx) - }) + after(() => { + sinon.restore() + }) + afterEach(async () => { + await deletePagesByParam({ url }, ctx) + }) + + context('when we save a new url', () => { it('should return a slugged url', async () => { const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.saveUrl.url).to.startsWith( @@ -666,6 +686,23 @@ describe('Article API', () => { ) }) }) + + context('when we save labels', () => { + it('saves the labels and archives the page', async () => { + url = 'https://blog.omnivore.app/new-url-2' + const state = ArticleSavingRequestStatus.Archived + const labels = ['test name', 'test name 2'] + await graphqlRequest( + saveUrlQuery(url, state, labels), + authToken + ).expect(200) + await refreshIndex() + + const savedPage = await getPageByParam({ url }) + expect(savedPage?.archivedAt).to.not.be.null + expect(savedPage?.labels?.map((l) => l.name)).to.eql(labels) + }) + }) }) describe('setBookmarkArticle', () => { From 9857af27e2135b01d92001269a22aa80444b3f6d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Mar 2023 15:26:04 +0800 Subject: [PATCH 42/48] Add labels and state to createArticle API --- packages/api/src/generated/graphql.ts | 2 + packages/api/src/generated/schema.graphql | 2 + packages/api/src/resolvers/article/index.ts | 39 +++++++++++-------- packages/api/src/schema.ts | 2 + packages/api/src/services/save_file.ts | 35 +++++++++++++++-- packages/api/test/resolvers/article.test.ts | 16 +++++++- packages/api/test/services/save_email.test.ts | 2 +- .../services/save_newsletter_email.test.ts | 4 +- 8 files changed, 79 insertions(+), 23 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index da0ef3fba..3c9449a96 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -265,9 +265,11 @@ export enum CreateArticleErrorCode { export type CreateArticleInput = { articleSavingRequestId?: InputMaybe; + labels?: InputMaybe>; preparedDocument?: InputMaybe; skipParsing?: InputMaybe; source?: InputMaybe; + state?: InputMaybe; uploadFileId?: InputMaybe; url: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 1ae8efa8d..82d0ef913 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -223,9 +223,11 @@ enum CreateArticleErrorCode { input CreateArticleInput { articleSavingRequestId: ID + labels: [CreateLabelInput!] preparedDocument: PreparedDocumentInput skipParsing: Boolean source: String + state: ArticleSavingRequestStatus uploadFileId: ID url: String! } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index c7e703c10..307f737e6 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -101,6 +101,7 @@ import { makeStorageFilePublic, } from '../../utils/uploads' import { WithDataSourcesContext } from '../types' +import { createLabels } from '../../services/labels' enum ArticleFormat { Markdown = 'markdown', @@ -146,6 +147,8 @@ export const createArticleResolver = authorized< uploadFileId, skipParsing, source, + state, + labels: inputLabels, }, }, ctx @@ -219,6 +222,19 @@ export const createArticleResolver = authorized< isArchived: false, }, } + // save state + let archivedAt = + state === ArticleSavingRequestStatus.Archived ? new Date() : null + if (pageId) { + const reminder = await models.reminder.getByRequestId(uid, pageId) + if (reminder && reminder.archiveUntil) { + archivedAt = new Date() + } + } + // add labels to page + const labels = inputLabels + ? await createLabels(ctx, inputLabels) + : undefined if (uploadFileId) { /* We do not trust the values from client, lookup upload file by querying @@ -248,7 +264,7 @@ export const createArticleResolver = authorized< source !== 'puppeteer-parse' && FORCE_PUPPETEER_URLS.some((regex) => regex.test(url)) ) { - await createPageSaveRequest({ userId: uid, url }) + await createPageSaveRequest({ userId: uid, url, archivedAt, labels }) return DUMMY_RESPONSE } else if (!skipParsing && preparedDocument?.document) { const parseResults = await traceAs>( @@ -264,7 +280,7 @@ export const createArticleResolver = authorized< } else if (!preparedDocument?.document) { // We have a URL but no document, so we try to send this to puppeteer // and return a dummy response. - await createPageSaveRequest({ userId: uid, url }) + await createPageSaveRequest({ userId: uid, url, archivedAt, labels }) return DUMMY_RESPONSE } @@ -287,14 +303,6 @@ export const createArticleResolver = authorized< saveTime, }) - let archive = false - if (pageId) { - const reminder = await models.reminder.getByRequestId(uid, pageId) - if (reminder) { - archive = reminder.archiveUntil || false - } - } - log.info('New article saving', { parsedArticle: Object.assign({}, articleToSave, { content: undefined, @@ -308,7 +316,6 @@ export const createArticleResolver = authorized< }, }) - let uploadFileUrlOverride = '' if (uploadFileId) { const uploadFileData = await authTrx(async (tx) => { return models.uploadFile.setFileUploadComplete(uploadFileId, tx) @@ -322,12 +329,11 @@ export const createArticleResolver = authorized< pageId ) } - uploadFileUrlOverride = await makeStorageFilePublic( - uploadFileData.id, - uploadFileData.fileName - ) + await makeStorageFilePublic(uploadFileData.id, uploadFileData.fileName) } - + // save page's state and labels + articleToSave.archivedAt = archivedAt + articleToSave.labels = labels if ( pageId || (pageId = ( @@ -338,7 +344,6 @@ export const createArticleResolver = authorized< )?.id) ) { // update existing page's state from processing to succeeded - articleToSave.archivedAt = archive ? saveTime : null const updated = await updatePage(pageId, articleToSave, { ...ctx, uid, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index a09570776..afcd9c787 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -488,6 +488,8 @@ const schema = gql` uploadFileId: ID skipParsing: Boolean source: String + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } enum CreateArticleErrorCode { UNABLE_TO_FETCH diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index 7e12f7150..e7624b786 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -2,8 +2,15 @@ import { Knex } from 'knex' import { PubsubClient } from '../datalayer/pubsub' import { UserData } from '../datalayer/user/model' import { homePageURL } from '../env' -import { SaveErrorCode, SaveFileInput, SaveResult } from '../generated/graphql' +import { + ArticleSavingRequestStatus, + SaveErrorCode, + SaveFileInput, + SaveResult, +} from '../generated/graphql' import { DataModels } from '../resolvers/types' +import { createLabels } from './labels' +import { updatePage } from '../elastic/pages' import { getStorageFileDetails } from '../utils/uploads' type SaveContext = { @@ -22,7 +29,7 @@ export const saveFile = async ( input: SaveFileInput ): Promise => { console.log('saving file with input', input) - + const pageId = input.clientRequestId const uploadFile = await ctx.models.uploadFile.getWhere({ id: input.uploadFileId, userId: saver.id, @@ -46,7 +53,29 @@ export const saveFile = async ( } } - // TODO: save labels and archive state + // save state + const archivedAt = + input.state === ArticleSavingRequestStatus.Archived ? new Date() : null + // add labels to page + const labels = input.labels + ? await createLabels({ ...ctx, uid: saver.id }, input.labels) + : undefined + if (input.state || input.labels) { + const updated = await updatePage( + pageId, + { + archivedAt, + labels, + }, + ctx + ) + if (!updated) { + console.log('error updating page', pageId) + return { + errorCodes: [SaveErrorCode.Unknown], + } + } + } return { clientRequestId: input.clientRequestId, diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index cbcbaddae..88671648d 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -39,6 +39,7 @@ import { } from '../util' import sinon from 'sinon' import * as createTask from '../../src/utils/createTask' +import * as uploads from '../../src/utils/uploads' chai.use(chaiString) @@ -643,6 +644,7 @@ describe('Article API', () => { }) it('saves the labels and archives the page', async () => { + url = 'https://blog.omnivore.app/new-url-2' const state = ArticleSavingRequestStatus.Archived const labels = ['test name', 'test name 2'] await graphqlRequest( @@ -849,15 +851,27 @@ describe('Article API', () => { }) }) - xdescribe('SaveFile', () => { + describe('SaveFile', () => { let query = '' let url = '' let uploadFileId = '' + before(() => { + sinon.replace( + uploads, + 'getStorageFileDetails', + sinon.fake.resolves({ fileUrl: 'fake url', md5Hash: 'fake hash' }) + ) + }) + beforeEach(() => { query = saveFileQuery(url, uploadFileId) }) + after(() => { + sinon.restore() + }) + context('when the file is not uploaded', () => { before(async () => { url = 'fake url' diff --git a/packages/api/test/services/save_email.test.ts b/packages/api/test/services/save_email.test.ts index 975c9ad0d..a98cbd659 100644 --- a/packages/api/test/services/save_email.test.ts +++ b/packages/api/test/services/save_email.test.ts @@ -22,7 +22,7 @@ describe('saveEmail', () => { }) it('doesnt fail if saved twice', async () => { - nock('https://blog.omnivore.app').get('/fake-url').reply(404) + nock('https://blog.omnivore.app').get('/fake-url').reply(200) const url = 'https://blog.omnivore.app/fake-url' const title = 'fake title' diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index 7859aed45..a1b0fbd92 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -50,7 +50,8 @@ describe('saveNewsletterEmail', () => { }) it('adds the newsletter to the library', async () => { - nock('https://blog.omnivore.app').get('/fake-url').reply(404) + nock('https://blog.omnivore.app').get('/fake-url').reply(200) + nock('https://blog.omnivore.app').head('/fake-url').reply(200) const url = 'https://blog.omnivore.app/fake-url' await saveNewsletterEmail( @@ -88,6 +89,7 @@ describe('saveNewsletterEmail', () => { }) it('adds a Newsletter label to that page', async () => { + nock('https://blog.omnivore.app').get('/new-fake-url').reply(200) const url = 'https://blog.omnivore.app/new-fake-url' const newLabel = { name: 'Newsletter', From 39b79f5cd2be82119982c5815e85eb3c9a26a6f2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Mar 2023 15:57:43 +0800 Subject: [PATCH 43/48] Mock sending emails while testing --- packages/api/test/resolvers/recent_emails.test.ts | 4 ++++ packages/api/test/utils/parser.test.ts | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/api/test/resolvers/recent_emails.test.ts b/packages/api/test/resolvers/recent_emails.test.ts index 6b0ffcb3b..7d87414be 100644 --- a/packages/api/test/resolvers/recent_emails.test.ts +++ b/packages/api/test/resolvers/recent_emails.test.ts @@ -6,6 +6,8 @@ import { graphqlRequest, request } from '../util' import { getRepository } from '../../src/entity/utils' import { ReceivedEmail } from '../../src/entity/received_email' import { NewsletterEmail } from '../../src/entity/newsletter_email' +import sinon from 'sinon' +import * as sendEmail from '../../src/utils/sendEmail' describe('Recent Emails Resolver', () => { const recentEmailsQuery = ` @@ -121,11 +123,13 @@ describe('Recent Emails Resolver', () => { to: newsletterEmail.address, type: 'non-article', }) + sinon.replace(sendEmail, 'sendEmail', sinon.fake.resolves(true)) }) after(async () => { // clean up await getRepository(ReceivedEmail).delete(recentEmail.id) + sinon.restore() }) it('marks email as item', async () => { diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index bf5889155..4a8bdc1de 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -21,7 +21,7 @@ const load = (path: string): string => { return fs.readFileSync(path, 'utf8') } -describe('parseMetadata', async () => { +describe('parseMetadata', () => { it('gets author, title, image, description', async () => { const html = load('./test/utils/data/substack-post.html') const metadata = await parsePageMetadata(html) @@ -36,7 +36,7 @@ describe('parseMetadata', async () => { }) }) -describe('parsePreparedContent', async () => { +describe('parsePreparedContent', () => { it('gets published date when JSONLD fails to load', async () => { nock('https://stratechery.com:443', { encodedQueryParams: true }) .get('/wp-json/oembed/1.0/embed') @@ -78,7 +78,7 @@ describe('parsePreparedContent', async () => { }) }) -describe('parsePreparedContent', async () => { +describe('parsePreparedContent', () => { nock('https://oembeddata').get('/').reply(200, { version: '1.0', provider_name: 'Hippocratic Adventures', From 42dc3f73301f1ef0ddaea9851f266c8507c8686f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Mar 2023 16:24:25 +0800 Subject: [PATCH 44/48] Fix some external requests not being mocked --- packages/api/src/utils/parser.ts | 2 +- packages/api/test/services/save_email.test.ts | 8 ++++++-- packages/api/test/services/save_newsletter_email.test.ts | 1 + packages/api/test/utils/parser.test.ts | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 8316a4619..8c894d968 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -384,7 +384,7 @@ const getJSONLdLinkMetadata = async ( return result } catch (error) { - logger.warning(`Unable to get JSONLD link of the article`, error) + logger.warning(`Unable to get JSONLD link of the article`, { error }) return result } } diff --git a/packages/api/test/services/save_email.test.ts b/packages/api/test/services/save_email.test.ts index a98cbd659..a7cc8cfd9 100644 --- a/packages/api/test/services/save_email.test.ts +++ b/packages/api/test/services/save_email.test.ts @@ -11,19 +11,23 @@ import { User } from '../../src/entity/user' describe('saveEmail', () => { const fakeContent = 'fake content' let user: User + let scope: nock.Scope before(async () => { // create test user user = await createTestUser('fakeUser') + scope = nock('https://blog.omnivore.app') + .get('/fake-url') + .reply(200) + .persist() }) after(async () => { await deleteTestUser(user.id) + scope.persist(false) }) it('doesnt fail if saved twice', async () => { - nock('https://blog.omnivore.app').get('/fake-url').reply(200) - const url = 'https://blog.omnivore.app/fake-url' const title = 'fake title' const author = 'fake author' diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index a1b0fbd92..8d9d28a2c 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -90,6 +90,7 @@ describe('saveNewsletterEmail', () => { it('adds a Newsletter label to that page', async () => { nock('https://blog.omnivore.app').get('/new-fake-url').reply(200) + nock('https://blog.omnivore.app').head('/new-fake-url').reply(200) const url = 'https://blog.omnivore.app/new-fake-url' const newLabel = { name: 'Newsletter', diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 4a8bdc1de..34900d830 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -22,9 +22,9 @@ const load = (path: string): string => { } describe('parseMetadata', () => { - it('gets author, title, image, description', async () => { + it('gets author, title, image, description', () => { const html = load('./test/utils/data/substack-post.html') - const metadata = await parsePageMetadata(html) + const metadata = parsePageMetadata(html) expect(metadata?.author).to.deep.equal('Omnivore') expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting') expect(metadata?.previewImage).to.deep.equal( From ef800ded81f3e51d1ae031520e3887da21f1152a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 6 Mar 2023 16:41:29 +0800 Subject: [PATCH 45/48] Fix some external requests not being mocked --- .../api/test/resolvers/article_saving_request.test.ts | 10 ++++++++++ packages/api/test/resolvers/integrations.test.ts | 5 +++++ packages/api/test/routers/article.test.ts | 10 ++++++++++ packages/api/test/routers/integrations.test.ts | 2 +- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/api/test/resolvers/article_saving_request.test.ts b/packages/api/test/resolvers/article_saving_request.test.ts index 2d38bf026..d6faa5b95 100644 --- a/packages/api/test/resolvers/article_saving_request.test.ts +++ b/packages/api/test/resolvers/article_saving_request.test.ts @@ -15,6 +15,8 @@ import { import * as createTask from '../../src/utils/createTask' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' +import sinon from 'sinon' +import * as createTask from '../../src/utils/createTask' const articleSavingRequestQuery = ({ id, @@ -89,6 +91,14 @@ describe('ArticleSavingRequest API', () => { }) describe('createArticleSavingRequest', () => { + before(() => { + sinon.replace(createTask, 'enqueueParseRequest', sinon.fake.resolves('')) + }) + + after(() => { + sinon.restore() + }) + it('returns the article saving request', async () => { const res = await graphqlRequest( createArticleSavingRequestMutation('https://blog.omnivore.app'), diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 22c3e5f37..79c130294 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -86,6 +86,11 @@ describe('Integrations resolvers', () => { context('when token is invalid', () => { before(() => { token = 'invalid token' + nock(READWISE_API_URL, { + reqheaders: { Authorization: `Token ${token}` }, + }) + .get('/auth') + .reply(401) }) it('returns InvalidToken error code', async () => { diff --git a/packages/api/test/routers/article.test.ts b/packages/api/test/routers/article.test.ts index ec0c133f7..49fc7fe99 100644 --- a/packages/api/test/routers/article.test.ts +++ b/packages/api/test/routers/article.test.ts @@ -5,6 +5,8 @@ import nock from 'nock' import 'mocha' import { env } from '../../src/env' import { User } from '../../src/entity/user' +import sinon from 'sinon' +import * as createTask from '../../src/utils/createTask' describe('/article/save API', () => { let user: User @@ -33,6 +35,14 @@ describe('/article/save API', () => { describe('POST /article/save', () => { const url = 'https://blog.omnivore.app' + before(() => { + sinon.replace(createTask, 'enqueueParseRequest', sinon.fake.resolves('')) + }) + + after(() => { + sinon.restore() + }) + context('when token and url are valid', () => { it('should create an article saving request', async () => { const response = await request diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 15e43cd5c..4429bcc08 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -65,7 +65,7 @@ describe('Integrations routers', () => { context('when token is valid', () => { before(() => { - token = process.env.PUBSUB_VERIFICATION_TOKEN! + token = process.env.PUBSUB_VERIFICATION_TOKEN as string }) context('when data is expired', () => { From c07ada02185e3f325ce3f4262c7c7054caf278a3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 15:13:48 +0800 Subject: [PATCH 46/48] resolve conflicts --- packages/api/src/resolvers/article/index.ts | 2 +- .../resolvers/article_saving_request/index.ts | 9 +- packages/api/src/routers/article_router.ts | 2 - packages/api/src/routers/svc/integrations.ts | 10 +- .../src/services/create_page_save_request.ts | 16 ++- packages/api/src/services/save_page.ts | 19 +--- packages/api/src/utils/createTask.ts | 107 ++++-------------- packages/api/src/utils/helpers.ts | 24 ++-- packages/api/tsconfig.json | 2 +- 9 files changed, 64 insertions(+), 127 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 307f737e6..bc9f6603c 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -72,6 +72,7 @@ import { UpdatesSinceSuccess, } from '../../generated/graphql' import { createPageSaveRequest } from '../../services/create_page_save_request' +import { createLabels } from '../../services/labels' import { parsedContentToPage } from '../../services/save_page' import { traceAs } from '../../tracing' import { Merge } from '../../util' @@ -101,7 +102,6 @@ import { makeStorageFilePublic, } from '../../utils/uploads' import { WithDataSourcesContext } from '../types' -import { createLabels } from '../../services/labels' enum ArticleFormat { Markdown = 'markdown', diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 2c431dfb0..2417b3d0d 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -1,5 +1,7 @@ /* eslint-disable prefer-const */ import { getPageByParam } from '../../elastic/pages' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' import { env } from '../../env' import { ArticleSavingRequestError, @@ -60,11 +62,14 @@ export const articleSavingRequestResolver = authorized< ArticleSavingRequestSuccess, ArticleSavingRequestError, QueryArticleSavingRequestArgs ->(async (_, { id, url }, { models, claims }) => { +>(async (_, { id, url }, { claims }) => { if (!id && !url) { return { errorCodes: [ArticleSavingRequestErrorCode.BadData] } } - const user = await models.user.get(claims.uid) + const user = await getRepository(User).findOne({ + where: { id: claims.uid }, + relations: ['profile'], + }) if (!user) { return { errorCodes: [ArticleSavingRequestErrorCode.Unauthorized] } } diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index dc28281c1..c35b39302 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -4,7 +4,6 @@ import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import { kx } from '../datalayer/knex_config' import { createPubSubClient } from '../datalayer/pubsub' import { getPageById, updatePage } from '../elastic/pages' import { Speech, SpeechState } from '../entity/speech' @@ -12,7 +11,6 @@ import { getRepository } from '../entity/utils' import { env } from '../env' import { CreateArticleErrorCode } from '../generated/graphql' import { Claims } from '../resolvers/types' -import { initModels } from '../server' import { createPageSaveRequest } from '../services/create_page_save_request' import { getClaimsByToken } from '../utils/auth' import { isSiteBlockedForParse } from '../utils/blocked' diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 981028576..efc8f85d0 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -2,19 +2,19 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import express from 'express' +import { DateTime } from 'luxon' +import { v4 as uuidv4 } from 'uuid' import { EntityType, readPushSubscription } from '../../datalayer/pubsub' import { getPageById, searchPages } from '../../elastic/pages' import { Page } from '../../elastic/types' import { Integration, IntegrationType } from '../../entity/integration' import { getRepository } from '../../entity/utils' -import { syncWithIntegration } from '../../services/integrations' +import { Claims } from '../../resolvers/types' +import { getIntegrationService } from '../../services/integrations' +import { getClaimsByToken } from '../../utils/auth' import { buildLogger } from '../../utils/logger' import { DateFilter } from '../../utils/search' -import { DateTime } from 'luxon' import { createGCSFile } from '../../utils/uploads' -import { v4 as uuidv4 } from 'uuid' -import { getClaimsByToken } from '../../utils/auth' -import { Claims } from '../../resolvers/types' export interface Message { type?: EntityType diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 77c3748ee..335421e76 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -9,17 +9,14 @@ import { updatePage, } from '../elastic/pages' import { ArticleSavingRequestStatus, Label, PageType } from '../elastic/types' +import { User } from '../entity/user' +import { getRepository } from '../entity/utils' import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, } from '../generated/graphql' -// TODO: switch to a proper Entity instead of using the old data models. -import { DataModels } from '../resolvers/types' import { enqueueParseRequest } from '../utils/createTask' import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers' -import * as privateIpLib from 'private-ip' -import { getRepository } from '../entity/utils' -import { User } from '../entity/user' interface PageSaveRequest { userId: string @@ -157,7 +154,14 @@ export const createPageSaveRequest = async ({ ) } // enqueue task to parse page - await enqueueParseRequest(url, userId, page.id, priority) + await enqueueParseRequest({ + url, + userId, + saveRequestId: page.id, + priority, + archivedAt, + labels, + }) return pageToArticleSavingRequest(user, page) } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 78edd5c09..4a43f6e3f 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -109,10 +109,10 @@ export const savePage = async ( url: articleToSave.url, }) // save state - const archivedAt = + articleToSave.archivedAt = input.state === ArticleSavingRequestStatus.Archived ? new Date() : null // add labels to page - const labels = input.labels + articleToSave.labels = input.labels ? await createLabels(ctx, input.labels) : undefined @@ -125,11 +125,9 @@ export const savePage = async ( { // update the page with the new content ...articleToSave, - archivedAt, // unarchive if it was archived id: pageId, // we don't want to update the id slug, // we don't want to update the slug createdAt: existingPage.createdAt, // we don't want to update the createdAt - labels, }, ctx )) @@ -146,8 +144,8 @@ export const savePage = async ( url: articleToSave.url, pubsub: ctx.pubsub, articleSavingRequestId: input.clientRequestId, - archivedAt, - labels, + archivedAt: articleToSave.archivedAt, + labels: articleToSave.labels, }) } catch (e) { return { @@ -156,14 +154,7 @@ export const savePage = async ( } } } else { - const newPageId = await createPage( - { - ...articleToSave, - archivedAt, - labels, - }, - ctx - ) + const newPageId = await createPage(articleToSave, ctx) if (!newPageId) { return { errorCodes: [SaveErrorCode.Unknown], diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index b6b16262b..85fe44b4d 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -2,14 +2,14 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ // Imports the Google Cloud Tasks library. import { CloudTasksClient, protos } from '@google-cloud/tasks' +import { google } from '@google-cloud/tasks/build/protos/protos' import axios from 'axios' +import { nanoid } from 'nanoid' +import { Label, Recommendation } from '../elastic/types' import { env } from '../env' +import { signFeatureToken } from '../services/features' import { CreateTaskError } from './errors' import { buildLogger } from './logger' -import { nanoid } from 'nanoid' -import { google } from '@google-cloud/tasks/build/protos/protos' -import { signFeatureToken } from '../services/features' -import { Recommendation } from '../elastic/types' import View = google.cloud.tasks.v2.Task.View const logger = buildLogger('app.dispatch') @@ -194,18 +194,30 @@ export const deleteTask = async ( * @param queue - Queue name * @returns Name of the task created */ -export const enqueueParseRequest = async ( - url: string, - userId: string, - saveRequestId: string, - priority: 'low' | 'high' = 'high', - queue = env.queue.name -): Promise => { +export const enqueueParseRequest = async ({ + url, + userId, + saveRequestId, + priority = 'high', + queue = env.queue.name, + archivedAt, + labels, +}: { + url: string + userId: string + saveRequestId: string + priority?: 'low' | 'high' + queue?: string + archivedAt?: Date | null + labels?: Label[] +}): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { url, userId, saveRequestId, + archivedAt, + labels, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment @@ -245,79 +257,6 @@ export const enqueueParseRequest = async ( return createdTasks[0].name } -export const enqueueReminder = async ( - userId: string, - scheduleTime: number -): Promise => { - const { GOOGLE_CLOUD_PROJECT } = process.env - const payload = { - userId, - scheduleTime, - } - - // If there is no Google Cloud Project Id exposed, it means that we are in local environment - if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { - return nanoid() - } - - const createdTasks = await createHttpTaskWithToken({ - project: GOOGLE_CLOUD_PROJECT, - payload, - scheduleTime, - taskHandlerUrl: env.queue.reminderTaskHandlerUrl, - }) - - if (!createdTasks || !createdTasks[0].name) { - logger.error(`Unable to get the name of the task`, { - payload, - createdTasks, - }) - throw new CreateTaskError(`Unable to get the name of the task`) - } - return createdTasks[0].name -} - -export const enqueueSyncWithIntegration = async ( - userId: string, - integrationName: string -): Promise => { - const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env - // use pubsub data format to send the userId to the task handler - const payload = { - message: { - data: Buffer.from( - JSON.stringify({ - userId, - }) - ).toString('base64'), - publishTime: new Date().toISOString(), - }, - } - - // If there is no Google Cloud Project Id exposed, it means that we are in local environment - if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { - return nanoid() - } - - const createdTasks = await createHttpTaskWithToken({ - project: GOOGLE_CLOUD_PROJECT, - payload, - taskHandlerUrl: `${ - env.queue.integrationTaskHandlerUrl - }/${integrationName.toLowerCase()}/sync_all?token=${PUBSUB_VERIFICATION_TOKEN}`, - priority: 'low', - }) - - if (!createdTasks || !createdTasks[0].name) { - logger.error(`Unable to get the name of the task`, { - payload, - createdTasks, - }) - throw new CreateTaskError(`Unable to get the name of the task`) - } - return createdTasks[0].name -} - export const enqueueTextToSpeech = async ({ userId, text, diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index a51e120b1..ecf51afc0 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -1,4 +1,14 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import crypto from 'crypto' +import normalizeUrl from 'normalize-url' +import path from 'path' +import _ from 'underscore' +import slugify from 'voca/slugify' +import wordsCounter from 'word-counting' +import { RegistrationType, UserData } from '../datalayer/user/model' +import { updatePage } from '../elastic/pages' +import { ArticleSavingRequestStatus, Page } from '../elastic/types' +import { User } from '../entity/user' import { ArticleSavingRequest, CreateArticleError, @@ -6,19 +16,9 @@ import { Profile, ResolverFn, } from '../generated/graphql' -import { Claims, WithDataSourcesContext } from '../resolvers/types' -import { RegistrationType, UserData } from '../datalayer/user/model' -import crypto from 'crypto' -import slugify from 'voca/slugify' -import { Merge } from '../util' import { CreateArticlesSuccessPartial } from '../resolvers' -import { ArticleSavingRequestStatus, Page } from '../elastic/types' -import { updatePage } from '../elastic/pages' -import path from 'path' -import normalizeUrl from 'normalize-url' -import wordsCounter from 'word-counting' -import _ from 'underscore' -import { User } from '../entity/user' +import { Claims, WithDataSourcesContext } from '../resolvers/types' +import { Merge } from '../util' interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index ee1fb782a..7c8caecfd 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -7,5 +7,5 @@ "outDir": "dist" }, "include": ["src", "test"], - "exclude": ["./src/generated"] + "exclude": ["./src/generated", "./test"] } From 874247661b6605ab34a6bfb6938aa8c5a3226e73 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 16:00:57 +0800 Subject: [PATCH 47/48] resolve conflicts --- .../api/src/resolvers/integrations/index.ts | 17 ++--- .../api/src/services/integrations/readwise.ts | 51 +++++++------ packages/api/src/utils/createTask.ts | 73 +++++++++++++++++++ .../resolvers/article_saving_request.test.ts | 10 --- ...=> 0112.do.change_type_in_integration.sql} | 0 ... 0112.undo.change_type_in_integration.sql} | 0 6 files changed, 110 insertions(+), 41 deletions(-) rename packages/db/migrations/{0111.do.change_type_in_integration.sql => 0112.do.change_type_in_integration.sql} (100%) rename packages/db/migrations/{0111.undo.change_type_in_integration.sql => 0112.undo.change_type_in_integration.sql} (100%) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 808affbe5..d302aa97c 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -1,4 +1,7 @@ -import { authorized } from '../../utils/helpers' +import { Integration, IntegrationType } from '../../entity/integration' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' +import { env } from '../../env' import { DeleteIntegrationError, DeleteIntegrationErrorCode, @@ -16,17 +19,14 @@ import { SetIntegrationErrorCode, SetIntegrationSuccess, } from '../../generated/graphql' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' -import { Integration, IntegrationType } from '../../entity/integration' -import { analytics } from '../../utils/analytics' -import { env } from '../../env' import { getIntegrationService } from '../../services/integrations' +import { analytics } from '../../utils/analytics' import { deleteTask, enqueueImportFromIntegration, enqueueSyncWithIntegration, } from '../../utils/createTask' +import { authorized } from '../../utils/helpers' export const setIntegrationResolver = authorized< SetIntegrationSuccess, @@ -89,10 +89,7 @@ export const setIntegrationResolver = authorized< (!integrationToSave.id || integrationToSave.enabled) ) { // create a task to sync all the pages if new integration or enable integration (export type) - const taskName = await enqueueSyncWithIntegration( - user.id, - input.type as string - ) + const taskName = await enqueueSyncWithIntegration(user.id, input.name) log.info('enqueued task', taskName) // update task name in integration diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 9c680cf0e..1f060e87c 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,10 +1,10 @@ -import { env } from '../../env' import axios from 'axios' -import { Page } from '../../elastic/types' -import { getHighlightUrl } from '../highlights' -import { getRepository } from '../../entity/utils' +import { HighlightType, Page } from '../../elastic/types' import { Integration } from '../../entity/integration' +import { getRepository } from '../../entity/utils' +import { env } from '../../env' import { wait } from '../../utils/helpers' +import { getHighlightUrl } from '../highlights' import { IntegrationService } from './integration' interface ReadwiseHighlight { @@ -75,23 +75,32 @@ export class ReadwiseIntegration extends IntegrationService { } pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => { - if (!page.highlights) return [] - return page.highlights.map((highlight) => { - return { - text: highlight.quote, - title: page.title, - author: page.author || undefined, - highlight_url: getHighlightUrl(page.slug, highlight.id), - highlighted_at: new Date(highlight.createdAt).toISOString(), - category: 'articles', - image_url: page.image || undefined, - location: highlight.highlightPositionPercent || undefined, - location_type: 'order', - note: highlight.annotation || undefined, - source_type: 'omnivore', - source_url: page.url, - } - }) + const { highlights } = page + if (!highlights) return [] + const category = page.siteName === 'Twitter' ? 'tweets' : 'articles' + return highlights + .map((highlight) => { + // filter out highlights that are not of type highlight or have no quote + if (highlight.type !== HighlightType.Highlight || !highlight.quote) { + return undefined + } + + return { + text: highlight.quote, + title: page.title, + author: page.author || undefined, + highlight_url: getHighlightUrl(page.slug, highlight.id), + highlighted_at: new Date(highlight.createdAt).toISOString(), + category, + image_url: page.image || undefined, + // location: highlight.highlightPositionAnchorIndex || undefined, + location_type: 'order', + note: highlight.annotation || undefined, + source_type: 'omnivore', + source_url: page.url, + } + }) + .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[] } syncWithReadwise = async ( diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 85fe44b4d..89ec9a8dc 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -257,6 +257,79 @@ export const enqueueParseRequest = async ({ return createdTasks[0].name } +export const enqueueReminder = async ( + userId: string, + scheduleTime: number +): Promise => { + const { GOOGLE_CLOUD_PROJECT } = process.env + const payload = { + userId, + scheduleTime, + } + + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { + return nanoid() + } + + const createdTasks = await createHttpTaskWithToken({ + project: GOOGLE_CLOUD_PROJECT, + payload, + scheduleTime, + taskHandlerUrl: env.queue.reminderTaskHandlerUrl, + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + +export const enqueueSyncWithIntegration = async ( + userId: string, + integrationName: string +): Promise => { + const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env + // use pubsub data format to send the userId to the task handler + const payload = { + message: { + data: Buffer.from( + JSON.stringify({ + userId, + }) + ).toString('base64'), + publishTime: new Date().toISOString(), + }, + } + + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { + return nanoid() + } + + const createdTasks = await createHttpTaskWithToken({ + project: GOOGLE_CLOUD_PROJECT, + payload, + taskHandlerUrl: `${ + env.queue.integrationTaskHandlerUrl + }/${integrationName.toLowerCase()}/sync_all?token=${PUBSUB_VERIFICATION_TOKEN}`, + priority: 'low', + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + export const enqueueTextToSpeech = async ({ userId, text, diff --git a/packages/api/test/resolvers/article_saving_request.test.ts b/packages/api/test/resolvers/article_saving_request.test.ts index d6faa5b95..2d38bf026 100644 --- a/packages/api/test/resolvers/article_saving_request.test.ts +++ b/packages/api/test/resolvers/article_saving_request.test.ts @@ -15,8 +15,6 @@ import { import * as createTask from '../../src/utils/createTask' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' -import sinon from 'sinon' -import * as createTask from '../../src/utils/createTask' const articleSavingRequestQuery = ({ id, @@ -91,14 +89,6 @@ describe('ArticleSavingRequest API', () => { }) describe('createArticleSavingRequest', () => { - before(() => { - sinon.replace(createTask, 'enqueueParseRequest', sinon.fake.resolves('')) - }) - - after(() => { - sinon.restore() - }) - it('returns the article saving request', async () => { const res = await graphqlRequest( createArticleSavingRequestMutation('https://blog.omnivore.app'), diff --git a/packages/db/migrations/0111.do.change_type_in_integration.sql b/packages/db/migrations/0112.do.change_type_in_integration.sql similarity index 100% rename from packages/db/migrations/0111.do.change_type_in_integration.sql rename to packages/db/migrations/0112.do.change_type_in_integration.sql diff --git a/packages/db/migrations/0111.undo.change_type_in_integration.sql b/packages/db/migrations/0112.undo.change_type_in_integration.sql similarity index 100% rename from packages/db/migrations/0111.undo.change_type_in_integration.sql rename to packages/db/migrations/0112.undo.change_type_in_integration.sql From 014dc773e3d11a3ec63488691c14f246cb0a103c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 17:05:37 +0800 Subject: [PATCH 48/48] Fix labels not saved correctly by saveUrl --- packages/api/src/resolvers/labels/index.ts | 32 ++++++------ .../src/services/create_page_save_request.ts | 9 +++- packages/api/src/services/labels.ts | 50 +++++++++++-------- packages/api/src/utils/createTask.ts | 14 ++++-- packages/puppeteer-parse/index.js | 2 + 5 files changed, 63 insertions(+), 44 deletions(-) diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index b27c9bf3b..b39f358c8 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -1,4 +1,17 @@ -import { authorized, generateRandomColor } from '../../utils/helpers' +import { Between, ILike } from 'typeorm' +import { createPubSubClient } from '../../datalayer/pubsub' +import { getHighlightById } from '../../elastic/highlights' +import { + deleteLabel, + setLabelsForHighlight, + updateLabel, + updateLabelsInPage, +} from '../../elastic/labels' +import { getPageById } from '../../elastic/pages' +import { Label } from '../../entity/label' +import { User } from '../../entity/user' +import { getRepository, setClaims } from '../../entity/utils' +import { env } from '../../env' import { CreateLabelError, CreateLabelErrorCode, @@ -25,23 +38,10 @@ import { UpdateLabelErrorCode, UpdateLabelSuccess, } from '../../generated/graphql' -import { analytics } from '../../utils/analytics' -import { env } from '../../env' -import { User } from '../../entity/user' -import { Label } from '../../entity/label' -import { Between, ILike } from 'typeorm' -import { getRepository, setClaims } from '../../entity/utils' -import { createPubSubClient } from '../../datalayer/pubsub' import { AppDataSource } from '../../server' -import { getPageById } from '../../elastic/pages' -import { - deleteLabel, - setLabelsForHighlight, - updateLabel, - updateLabelsInPage, -} from '../../elastic/labels' -import { getHighlightById } from '../../elastic/highlights' import { getLabelsByIds } from '../../services/labels' +import { analytics } from '../../utils/analytics' +import { authorized, generateRandomColor } from '../../utils/helpers' export const labelsResolver = authorized( async (_obj, _params, { claims: { uid }, log }) => { diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 335421e76..6285c1d81 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -153,14 +153,19 @@ export const createPageSaveRequest = async ({ ctx ) } + const labelsInput = labels?.map((label) => ({ + name: label.name, + color: label.color, + description: label.description, + })) // enqueue task to parse page await enqueueParseRequest({ url, userId, saveRequestId: page.id, priority, - archivedAt, - labels, + state: archivedAt ? ArticleSavingRequestStatus.Archived : undefined, + labels: labelsInput, }) return pageToArticleSavingRequest(user, page) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index fee84bc46..102265155 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -1,13 +1,13 @@ -import { Label } from '../entity/label' -import { ILike, In } from 'typeorm' -import { PageContext } from '../elastic/types' -import { User } from '../entity/user' -import { addLabelInPage } from '../elastic/labels' -import { getRepository } from '../entity/utils' -import { Link } from '../entity/link' import DataLoader from 'dataloader' -import { generateRandomColor } from '../utils/helpers' +import { In } from 'typeorm' +import { addLabelInPage } from '../elastic/labels' +import { PageContext } from '../elastic/types' +import { Label } from '../entity/label' +import { Link } from '../entity/link' +import { User } from '../entity/user' +import { getRepository } from '../entity/utils' import { CreateLabelInput } from '../generated/graphql' +import { generateRandomColor } from '../utils/helpers' const batchGetLabelsFromLinkIds = async ( linkIds: readonly string[] @@ -40,10 +40,11 @@ export const addLabelToPage = async ( return false } - let labelEntity = await getRepository(Label).findOneBy({ - user: { id: user.id }, - name: ILike(label.name), - }) + let labelEntity = await getRepository(Label) + .createQueryBuilder() + .where({ user: { id: user.id } }) + .andWhere('LOWER(name) = LOWER(:name)', { name: label.name }) + .getOne() if (!labelEntity) { console.log('creating new label', label.name) @@ -87,10 +88,11 @@ export const createLabel = async ( description?: string } ): Promise