diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index eb38976c2..320604c3b 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -72,6 +72,8 @@ export enum ArticleSavingRequestStatus { Processing = 'PROCESSING', Succeeded = 'SUCCEEDED', Deleted = 'DELETED', + + Archived = 'ARCHIVED', } export enum HighlightType { @@ -84,7 +86,7 @@ export interface Label { id: string name: string color: string - description?: string + description?: string | null createdAt?: Date } diff --git a/packages/api/src/entity/integration.ts b/packages/api/src/entity/integration.ts index 0d1895f64..a051ac0db 100644 --- a/packages/api/src/entity/integration.ts +++ b/packages/api/src/entity/integration.ts @@ -10,7 +10,8 @@ import { import { User } from './user' export enum IntegrationType { - Readwise = 'READWISE', + Export = 'EXPORT', + Import = 'IMPORT', } @Entity({ name: 'integrations' }) @@ -22,7 +23,13 @@ export class Integration { @JoinColumn({ name: 'user_id' }) user!: User - @Column('enum', { enum: IntegrationType }) + @Column('varchar', { length: 40 }) + name!: string + + @Column('enum', { + enum: IntegrationType, + default: IntegrationType.Export, + }) type!: IntegrationType @Column('varchar', { length: 255 }) diff --git a/packages/api/src/entity/label.ts b/packages/api/src/entity/label.ts index 30d145f5d..0088e3319 100644 --- a/packages/api/src/entity/label.ts +++ b/packages/api/src/entity/label.ts @@ -24,7 +24,7 @@ export class Label { color!: string @Column('text', { nullable: true }) - description?: string + description?: string | null @CreateDateColumn() createdAt!: Date diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 22d51b8b4..3c9449a96 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -188,6 +188,7 @@ export enum ArticleSavingRequestErrorCode { export type ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess; export enum ArticleSavingRequestStatus { + Archived = 'ARCHIVED', Deleted = 'DELETED', Failed = 'FAILED', Processing = 'PROCESSING', @@ -264,9 +265,11 @@ export enum CreateArticleErrorCode { export type CreateArticleInput = { articleSavingRequestId?: InputMaybe; + labels?: InputMaybe>; preparedDocument?: InputMaybe; skipParsing?: InputMaybe; source?: InputMaybe; + state?: InputMaybe; uploadFileId?: InputMaybe; url: Scalars['String']; }; @@ -401,7 +404,7 @@ export enum CreateLabelErrorCode { } export type CreateLabelInput = { - color: Scalars['String']; + color?: InputMaybe; description?: InputMaybe; name: Scalars['String']; }; @@ -940,18 +943,37 @@ export enum HighlightType { Redaction = 'REDACTION' } +export type ImportFromIntegrationError = { + __typename?: 'ImportFromIntegrationError'; + errorCodes: Array; +}; + +export enum ImportFromIntegrationErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess; + +export type ImportFromIntegrationSuccess = { + __typename?: 'ImportFromIntegrationSuccess'; + success: Scalars['Boolean']; +}; + export type Integration = { __typename?: 'Integration'; createdAt: Scalars['Date']; enabled: Scalars['Boolean']; id: Scalars['ID']; + name: Scalars['String']; token: Scalars['String']; type: IntegrationType; updatedAt: Scalars['Date']; }; export enum IntegrationType { - Readwise = 'READWISE' + Export = 'EXPORT', + Import = 'IMPORT' } export type IntegrationsError = { @@ -1223,6 +1245,7 @@ export type Mutation = { generateApiKey: GenerateApiKeyResult; googleLogin: LoginResult; googleSignup: GoogleSignupResult; + importFromIntegration: ImportFromIntegrationResult; joinGroup: JoinGroupResult; leaveGroup: LeaveGroupResult; logOut: LogOutResult; @@ -1389,6 +1412,11 @@ export type MutationGoogleSignupArgs = { }; +export type MutationImportFromIntegrationArgs = { + integrationId: Scalars['ID']; +}; + + export type MutationJoinGroupArgs = { inviteCode: Scalars['String']; }; @@ -2167,7 +2195,9 @@ export enum SaveErrorCode { export type SaveFileInput = { clientRequestId: Scalars['ID']; + labels?: InputMaybe>; source: Scalars['String']; + state?: InputMaybe; uploadFileId: Scalars['ID']; url: Scalars['String']; }; @@ -2199,9 +2229,11 @@ export type SaveFilterSuccess = { export type SavePageInput = { clientRequestId: Scalars['ID']; + labels?: InputMaybe>; originalContent: Scalars['String']; parseResult?: InputMaybe; source: Scalars['String']; + state?: InputMaybe; title?: InputMaybe; url: Scalars['String']; }; @@ -2216,7 +2248,9 @@ export type SaveSuccess = { export type SaveUrlInput = { clientRequestId: Scalars['ID']; + labels?: InputMaybe>; source: Scalars['String']; + state?: InputMaybe; url: Scalars['String']; }; @@ -2387,8 +2421,9 @@ export enum SetIntegrationErrorCode { export type SetIntegrationInput = { enabled: Scalars['Boolean']; id?: InputMaybe; + name: Scalars['String']; token: Scalars['String']; - type: IntegrationType; + type?: InputMaybe; }; export type SetIntegrationResult = SetIntegrationError | SetIntegrationSuccess; @@ -3398,6 +3433,10 @@ export type ResolversTypes = { HighlightStats: ResolverTypeWrapper; HighlightType: HighlightType; ID: ResolverTypeWrapper; + ImportFromIntegrationError: ResolverTypeWrapper; + ImportFromIntegrationErrorCode: ImportFromIntegrationErrorCode; + ImportFromIntegrationResult: ResolversTypes['ImportFromIntegrationError'] | ResolversTypes['ImportFromIntegrationSuccess']; + ImportFromIntegrationSuccess: ResolverTypeWrapper; Int: ResolverTypeWrapper; Integration: ResolverTypeWrapper; IntegrationType: IntegrationType; @@ -3842,6 +3881,9 @@ export type ResolversParentTypes = { HighlightReply: HighlightReply; HighlightStats: HighlightStats; ID: Scalars['ID']; + ImportFromIntegrationError: ImportFromIntegrationError; + ImportFromIntegrationResult: ResolversParentTypes['ImportFromIntegrationError'] | ResolversParentTypes['ImportFromIntegrationSuccess']; + ImportFromIntegrationSuccess: ImportFromIntegrationSuccess; Int: Scalars['Int']; Integration: Integration; IntegrationsError: IntegrationsError; @@ -4764,10 +4806,25 @@ export type HighlightStatsResolvers; }; +export type ImportFromIntegrationErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type ImportFromIntegrationResultResolvers = { + __resolveType: TypeResolveFn<'ImportFromIntegrationError' | 'ImportFromIntegrationSuccess', ParentType, ContextType>; +}; + +export type ImportFromIntegrationSuccessResolvers = { + success?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type IntegrationResolvers = { createdAt?: Resolver; enabled?: Resolver; id?: Resolver; + name?: Resolver; token?: Resolver; type?: Resolver; updatedAt?: Resolver; @@ -4975,6 +5032,7 @@ export type MutationResolvers>; googleLogin?: Resolver>; googleSignup?: Resolver>; + importFromIntegration?: Resolver>; joinGroup?: Resolver>; leaveGroup?: Resolver>; logOut?: Resolver; @@ -6086,6 +6144,9 @@ export type Resolvers = { Highlight?: HighlightResolvers; HighlightReply?: HighlightReplyResolvers; HighlightStats?: HighlightStatsResolvers; + ImportFromIntegrationError?: ImportFromIntegrationErrorResolvers; + ImportFromIntegrationResult?: ImportFromIntegrationResultResolvers; + ImportFromIntegrationSuccess?: ImportFromIntegrationSuccessResolvers; Integration?: IntegrationResolvers; IntegrationsError?: IntegrationsErrorResolvers; IntegrationsResult?: IntegrationsResultResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 247ff2eda..82d0ef913 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -153,6 +153,7 @@ enum ArticleSavingRequestErrorCode { union ArticleSavingRequestResult = ArticleSavingRequestError | ArticleSavingRequestSuccess enum ArticleSavingRequestStatus { + ARCHIVED DELETED FAILED PROCESSING @@ -222,9 +223,11 @@ enum CreateArticleErrorCode { input CreateArticleInput { articleSavingRequestId: ID + labels: [CreateLabelInput!] preparedDocument: PreparedDocumentInput skipParsing: Boolean source: String + state: ArticleSavingRequestStatus uploadFileId: ID url: String! } @@ -349,7 +352,7 @@ enum CreateLabelErrorCode { } input CreateLabelInput { - color: String! + color: String description: String name: String! } @@ -835,17 +838,34 @@ enum HighlightType { REDACTION } +type ImportFromIntegrationError { + errorCodes: [ImportFromIntegrationErrorCode!]! +} + +enum ImportFromIntegrationErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union ImportFromIntegrationResult = ImportFromIntegrationError | ImportFromIntegrationSuccess + +type ImportFromIntegrationSuccess { + success: Boolean! +} + type Integration { createdAt: Date! enabled: Boolean! id: ID! + name: String! token: String! type: IntegrationType! updatedAt: Date! } enum IntegrationType { - READWISE + EXPORT + IMPORT } type IntegrationsError { @@ -1093,6 +1113,7 @@ type Mutation { generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! googleLogin(input: GoogleLoginInput!): LoginResult! googleSignup(input: GoogleSignupInput!): GoogleSignupResult! + importFromIntegration(integrationId: ID!): ImportFromIntegrationResult! joinGroup(inviteCode: String!): JoinGroupResult! leaveGroup(groupId: ID!): LeaveGroupResult! logOut: LogOutResult! @@ -1571,7 +1592,9 @@ enum SaveErrorCode { input SaveFileInput { clientRequestId: ID! + labels: [CreateLabelInput!] source: String! + state: ArticleSavingRequestStatus uploadFileId: ID! url: String! } @@ -1601,9 +1624,11 @@ type SaveFilterSuccess { input SavePageInput { clientRequestId: ID! + labels: [CreateLabelInput!] originalContent: String! parseResult: ParseResult source: String! + state: ArticleSavingRequestStatus title: String url: String! } @@ -1617,7 +1642,9 @@ type SaveSuccess { input SaveUrlInput { clientRequestId: ID! + labels: [CreateLabelInput!] source: String! + state: ArticleSavingRequestStatus url: String! } @@ -1775,8 +1802,9 @@ enum SetIntegrationErrorCode { input SetIntegrationInput { enabled: Boolean! id: ID + name: String! token: String! - type: IntegrationType! + type: IntegrationType } union SetIntegrationResult = SetIntegrationError | SetIntegrationSuccess diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index abe36814b..bc9f6603c 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -72,6 +72,7 @@ import { UpdatesSinceSuccess, } from '../../generated/graphql' import { createPageSaveRequest } from '../../services/create_page_save_request' +import { createLabels } from '../../services/labels' import { parsedContentToPage } from '../../services/save_page' import { traceAs } from '../../tracing' import { Merge } from '../../util' @@ -146,6 +147,8 @@ export const createArticleResolver = authorized< uploadFileId, skipParsing, source, + state, + labels: inputLabels, }, }, ctx @@ -219,6 +222,19 @@ export const createArticleResolver = authorized< isArchived: false, }, } + // save state + let archivedAt = + state === ArticleSavingRequestStatus.Archived ? new Date() : null + if (pageId) { + const reminder = await models.reminder.getByRequestId(uid, pageId) + if (reminder && reminder.archiveUntil) { + archivedAt = new Date() + } + } + // add labels to page + const labels = inputLabels + ? await createLabels(ctx, inputLabels) + : undefined if (uploadFileId) { /* We do not trust the values from client, lookup upload file by querying @@ -248,7 +264,7 @@ export const createArticleResolver = authorized< source !== 'puppeteer-parse' && FORCE_PUPPETEER_URLS.some((regex) => regex.test(url)) ) { - await createPageSaveRequest(uid, url, models) + await createPageSaveRequest({ userId: uid, url, archivedAt, labels }) return DUMMY_RESPONSE } else if (!skipParsing && preparedDocument?.document) { const parseResults = await traceAs>( @@ -264,7 +280,7 @@ export const createArticleResolver = authorized< } else if (!preparedDocument?.document) { // We have a URL but no document, so we try to send this to puppeteer // and return a dummy response. - await createPageSaveRequest(uid, url, models) + await createPageSaveRequest({ userId: uid, url, archivedAt, labels }) return DUMMY_RESPONSE } @@ -287,14 +303,6 @@ export const createArticleResolver = authorized< saveTime, }) - let archive = false - if (pageId) { - const reminder = await models.reminder.getByRequestId(uid, pageId) - if (reminder) { - archive = reminder.archiveUntil || false - } - } - log.info('New article saving', { parsedArticle: Object.assign({}, articleToSave, { content: undefined, @@ -308,7 +316,6 @@ export const createArticleResolver = authorized< }, }) - let uploadFileUrlOverride = '' if (uploadFileId) { const uploadFileData = await authTrx(async (tx) => { return models.uploadFile.setFileUploadComplete(uploadFileId, tx) @@ -322,12 +329,11 @@ export const createArticleResolver = authorized< pageId ) } - uploadFileUrlOverride = await makeStorageFilePublic( - uploadFileData.id, - uploadFileData.fileName - ) + await makeStorageFilePublic(uploadFileData.id, uploadFileData.fileName) } - + // save page's state and labels + articleToSave.archivedAt = archivedAt + articleToSave.labels = labels if ( pageId || (pageId = ( @@ -338,7 +344,6 @@ export const createArticleResolver = authorized< )?.id) ) { // update existing page's state from processing to succeeded - articleToSave.archivedAt = archive ? saveTime : null const updated = await updatePage(pageId, articleToSave, { ...ctx, uid, diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index a6ce2a8ba..2417b3d0d 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -1,5 +1,7 @@ /* eslint-disable prefer-const */ import { getPageByParam } from '../../elastic/pages' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' import { env } from '../../env' import { ArticleSavingRequestError, @@ -25,7 +27,7 @@ export const createArticleSavingRequestResolver = authorized< CreateArticleSavingRequestSuccess, CreateArticleSavingRequestError, MutationCreateArticleSavingRequestArgs ->(async (_, { input: { url } }, { models, claims, pubsub }) => { +>(async (_, { input: { url } }, { claims, pubsub }) => { analytics.track({ userId: claims.uid, event: 'link_saved', @@ -37,7 +39,11 @@ export const createArticleSavingRequestResolver = authorized< }) try { - const request = await createPageSaveRequest(claims.uid, url, models, pubsub) + const request = await createPageSaveRequest({ + userId: claims.uid, + url, + pubsub, + }) return { articleSavingRequest: request, } @@ -56,11 +62,14 @@ export const articleSavingRequestResolver = authorized< ArticleSavingRequestSuccess, ArticleSavingRequestError, QueryArticleSavingRequestArgs ->(async (_, { id, url }, { models, claims }) => { +>(async (_, { id, url }, { claims }) => { if (!id && !url) { return { errorCodes: [ArticleSavingRequestErrorCode.BadData] } } - const user = await models.user.get(claims.uid) + const user = await getRepository(User).findOne({ + where: { id: claims.uid }, + relations: ['profile'], + }) if (!user) { return { errorCodes: [ArticleSavingRequestErrorCode.Unauthorized] } } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 4e2228ccb..f5624512c 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -62,6 +62,7 @@ import { googleLoginResolver, googleSignupResolver, groupsResolver, + importFromIntegrationResolver, integrationsResolver, joinGroupResolver, labelsResolver, @@ -202,6 +203,7 @@ export const functionResolvers = { uploadImportFile: uploadImportFileResolver, markEmailAsItem: markEmailAsItemResolver, bulkAction: bulkActionResolver, + importFromIntegration: importFromIntegrationResolver, }, Query: { me: getMeUserResolver, @@ -650,4 +652,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('RecentEmails'), ...resultResolveTypeResolver('MarkEmailAsItem'), ...resultResolveTypeResolver('BulkAction'), + ...resultResolveTypeResolver('ImportFromIntegration'), } diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 8e8519d65..d302aa97c 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -1,24 +1,32 @@ -import { authorized } from '../../utils/helpers' +import { Integration, IntegrationType } from '../../entity/integration' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' +import { env } from '../../env' import { DeleteIntegrationError, DeleteIntegrationErrorCode, DeleteIntegrationSuccess, + ImportFromIntegrationError, + ImportFromIntegrationErrorCode, + ImportFromIntegrationSuccess, IntegrationsError, IntegrationsErrorCode, IntegrationsSuccess, MutationDeleteIntegrationArgs, + MutationImportFromIntegrationArgs, MutationSetIntegrationArgs, SetIntegrationError, SetIntegrationErrorCode, SetIntegrationSuccess, } from '../../generated/graphql' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' -import { Integration } from '../../entity/integration' +import { getIntegrationService } from '../../services/integrations' import { analytics } from '../../utils/analytics' -import { env } from '../../env' -import { validateToken } from '../../services/integrations' -import { deleteTask, enqueueSyncWithIntegration } from '../../utils/createTask' +import { + deleteTask, + enqueueImportFromIntegration, + enqueueSyncWithIntegration, +} from '../../utils/createTask' +import { authorized } from '../../utils/helpers' export const setIntegrationResolver = authorized< SetIntegrationSuccess, @@ -35,8 +43,11 @@ export const setIntegrationResolver = authorized< } } - let integrationToSave: Partial = { + const integrationToSave: Partial = { + ...input, user, + id: input.id || undefined, + type: input.type || IntegrationType.Export, } if (input.id) { // Update @@ -55,46 +66,30 @@ export const setIntegrationResolver = authorized< } } - integrationToSave = { - ...integrationToSave, - id: existingIntegration.id, - enabled: input.enabled, - token: input.token, - taskName: existingIntegration.taskName, - } + integrationToSave.id = existingIntegration.id + integrationToSave.taskName = existingIntegration.taskName } else { // Create - const existingIntegration = await getRepository(Integration).findOneBy({ - user: { id: uid }, - type: input.type, - }) - - if (existingIntegration) { - return { - errorCodes: [SetIntegrationErrorCode.AlreadyExists], - } - } - - // validate token - if (!(await validateToken(input.token, input.type))) { + const integrationService = getIntegrationService(input.name) + // authorize and get access token + const token = await integrationService.accessToken(input.token) + if (!token) { return { errorCodes: [SetIntegrationErrorCode.InvalidToken], } } - integrationToSave = { - ...integrationToSave, - token: input.token, - type: input.type, - enabled: true, - } + integrationToSave.token = token } // save integration const integration = await getRepository(Integration).save(integrationToSave) - if (!integrationToSave.id || integrationToSave.enabled) { - // create a task to sync all the pages if new integration or enable integration - const taskName = await enqueueSyncWithIntegration(user.id, input.type) + if ( + integrationToSave.type === IntegrationType.Export && + (!integrationToSave.id || integrationToSave.enabled) + ) { + // create a task to sync all the pages if new integration or enable integration (export type) + const taskName = await enqueueSyncWithIntegration(user.id, input.name) log.info('enqueued task', taskName) // update task name in integration @@ -215,7 +210,7 @@ export const deleteIntegrationResolver = authorized< }) return { - integration: deletedIntegration, + integration, } } catch (error) { log.error(error) @@ -225,3 +220,56 @@ export const deleteIntegrationResolver = authorized< } } }) + +export const importFromIntegrationResolver = authorized< + ImportFromIntegrationSuccess, + ImportFromIntegrationError, + MutationImportFromIntegrationArgs +>(async (_, { integrationId }, { claims: { uid }, log, signToken }) => { + log.info('importFromIntegrationResolver') + + try { + const integration = await getRepository(Integration).findOne({ + where: { id: integrationId, user: { id: uid } }, + relations: ['user'], + }) + + if (!integration) { + return { + errorCodes: [ImportFromIntegrationErrorCode.Unauthorized], + } + } + + const exp = Math.floor(Date.now() / 1000) + 60 * 60 * 24 // 1 day + const authToken = (await signToken( + { uid, exp }, + env.server.jwtSecret + )) as string + // create a task to import all the pages + const taskName = await enqueueImportFromIntegration( + uid, + integration.id, + authToken + ) + // update task name in integration + await getRepository(Integration).update(integration.id, { taskName }) + + analytics.track({ + userId: uid, + event: 'integration_import', + properties: { + integrationId, + }, + }) + + return { + success: true, + } + } catch (error) { + log.error(error) + + return { + errorCodes: [ImportFromIntegrationErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index 31bf420f8..b39f358c8 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -1,4 +1,17 @@ -import { authorized } from '../../utils/helpers' +import { Between, ILike } from 'typeorm' +import { createPubSubClient } from '../../datalayer/pubsub' +import { getHighlightById } from '../../elastic/highlights' +import { + deleteLabel, + setLabelsForHighlight, + updateLabel, + updateLabelsInPage, +} from '../../elastic/labels' +import { getPageById } from '../../elastic/pages' +import { Label } from '../../entity/label' +import { User } from '../../entity/user' +import { getRepository, setClaims } from '../../entity/utils' +import { env } from '../../env' import { CreateLabelError, CreateLabelErrorCode, @@ -25,23 +38,10 @@ import { UpdateLabelErrorCode, UpdateLabelSuccess, } from '../../generated/graphql' -import { analytics } from '../../utils/analytics' -import { env } from '../../env' -import { User } from '../../entity/user' -import { Label } from '../../entity/label' -import { Between, ILike } from 'typeorm' -import { getRepository, setClaims } from '../../entity/utils' -import { createPubSubClient } from '../../datalayer/pubsub' import { AppDataSource } from '../../server' -import { getPageById } from '../../elastic/pages' -import { - deleteLabel, - setLabelsForHighlight, - updateLabel, - updateLabelsInPage, -} from '../../elastic/labels' -import { getHighlightById } from '../../elastic/highlights' import { getLabelsByIds } from '../../services/labels' +import { analytics } from '../../utils/analytics' +import { authorized, generateRandomColor } from '../../utils/helpers' export const labelsResolver = authorized( async (_obj, _params, { claims: { uid }, log }) => { @@ -114,7 +114,7 @@ export const createLabelResolver = authorized< const label = await getRepository(Label).save({ user, name, - color, + color: color || generateRandomColor(), description: description || '', }) diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 988a94a2b..c35b39302 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -4,7 +4,6 @@ import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import { kx } from '../datalayer/knex_config' import { createPubSubClient } from '../datalayer/pubsub' import { getPageById, updatePage } from '../elastic/pages' import { Speech, SpeechState } from '../entity/speech' @@ -12,7 +11,6 @@ import { getRepository } from '../entity/utils' import { env } from '../env' import { CreateArticleErrorCode } from '../generated/graphql' import { Claims } from '../resolvers/types' -import { initModels } from '../server' import { createPageSaveRequest } from '../services/create_page_save_request' import { getClaimsByToken } from '../utils/auth' import { isSiteBlockedForParse } from '../utils/blocked' @@ -59,8 +57,7 @@ export function articleRouter() { return res.status(400).send({ errorCode: 'BAD_DATA' }) } - const models = initModels(kx, false) - const result = await createPageSaveRequest(uid, url, models) + const result = await createPageSaveRequest({ userId: uid, url }) if (isSiteBlockedForParse(url)) { return res diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts new file mode 100644 index 000000000..7614aeacd --- /dev/null +++ b/packages/api/src/routers/integration_router.ts @@ -0,0 +1,61 @@ +import cors from 'cors' +import express from 'express' +import { corsConfig } from '../utils/corsConfig' +import { env } from '../env' +import axios from 'axios' +import { buildLogger } from '../utils/logger' +import { getClaimsByToken } from '../utils/auth' + +const logger = buildLogger('app.dispatch') + +export function integrationRouter() { + const router = express.Router() + // request token from pocket + router.post( + '/pocket/auth', + cors(corsConfig), + async (req: express.Request, res: express.Response) => { + logger.info('pocket/request-token') + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const token = (req.cookies.auth as string) || req.headers.authorization + const claims = await getClaimsByToken(token) + if (!claims) { + return res.status(401).send('UNAUTHORIZED') + } + + const consumerKey = env.pocket.consumerKey + const redirectUri = `${env.client.url}/settings/integrations?state=pocketAuthorizationFinished` + try { + // make a POST request to Pocket to get a request token + const response = await axios.post<{ code: string }>( + 'https://getpocket.com/v3/oauth/request', + { + consumer_key: consumerKey, + redirect_uri: redirectUri, + }, + { + headers: { + 'Content-Type': 'application/json', + 'X-Accept': 'application/json', + }, + } + ) + const { code } = response.data + // store the request token in a cookie + res.cookie('pocketRequestToken', code, { + maxAge: 1000 * 60 * 60, + }) + // redirect the user to Pocket to authorize the request token + res.redirect( + `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}` + ) + } catch (e) { + logger.info('pocket/request-token exception:', e) + res.redirect( + `${env.client.url}/settings/integrations?errorCodes=UNKNOWN` + ) + } + } + ) + return router +} diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 03ccea762..efc8f85d0 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -2,14 +2,19 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import express from 'express' +import { DateTime } from 'luxon' +import { v4 as uuidv4 } from 'uuid' import { EntityType, readPushSubscription } from '../../datalayer/pubsub' import { getPageById, searchPages } from '../../elastic/pages' import { Page } from '../../elastic/types' import { Integration, IntegrationType } from '../../entity/integration' import { getRepository } from '../../entity/utils' -import { syncWithIntegration } from '../../services/integrations' +import { Claims } from '../../resolvers/types' +import { getIntegrationService } from '../../services/integrations' +import { getClaimsByToken } from '../../utils/auth' import { buildLogger } from '../../utils/logger' import { DateFilter } from '../../utils/search' +import { createGCSFile } from '../../utils/uploads' export interface Message { type?: EntityType @@ -19,15 +24,22 @@ export interface Message { articleId?: string } +interface ImportEvent { + integrationId: string +} + +const isImportEvent = (event: any): event is ImportEvent => + 'integrationId' in event + const logger = buildLogger('app.dispatch') export function integrationsServiceRouter() { const router = express.Router() - router.post('/:integrationType/:action', async (req, res) => { + router.post('/:integrationName/:action', async (req, res) => { logger.info('start to sync with integration', { action: req.params.action, - integrationType: req.params.integrationType, + integrationName: req.params.integrationName, }) const { message: msgStr, expired } = readPushSubscription(req) @@ -54,7 +66,8 @@ export function integrationsServiceRouter() { const integration = await getRepository(Integration).findOneBy({ user: { id: userId }, - type: req.params.integrationType.toUpperCase() as IntegrationType, + name: req.params.integrationName.toUpperCase(), + type: IntegrationType.Export, enabled: true, }) if (!integration) { @@ -64,6 +77,7 @@ export function integrationsServiceRouter() { } const action = req.params.action.toUpperCase() + const integrationService = getIntegrationService(integration.name) if (action === 'SYNC_UPDATED') { // get updated page by id let id: string | undefined @@ -99,7 +113,7 @@ export function integrationsServiceRouter() { pageId: page.id, }) - const synced = await syncWithIntegration(integration, [page]) + const synced = await integrationService.export(integration, [page]) if (!synced) { logger.info('failed to sync page', { integrationId: integration.id, @@ -125,12 +139,12 @@ export function integrationsServiceRouter() { ;[pages, count] = (await searchPages( { from: after, size, dateFilters }, userId - ))! + )) as [Page[], number] const pageIds = pages.map((p) => p.id) logger.info('syncing pages', { pageIds }) - const synced = await syncWithIntegration(integration, pages) + const synced = await integrationService.export(integration, pages) if (!synced) { logger.info('failed to sync pages', { pageIds, @@ -156,6 +170,95 @@ export function integrationsServiceRouter() { res.status(500).send(err) } }) + // import pages from integration task handler + router.post('/import', async (req, res) => { + logger.info('start cloud task to import pages from integration') + const token = req.cookies?.auth || req.headers?.authorization + let claims: Claims | undefined + try { + claims = await getClaimsByToken(token) + if (!claims) { + return res.status(401).send('UNAUTHORIZED') + } + } catch (err) { + logger.error('failed to get claims from token', err) + return res.status(401).send('UNAUTHORIZED') + } + + if (!isImportEvent(req.body)) { + logger.info('Invalid message') + return res.status(400).send('Bad Request') + } + + let writeStream: NodeJS.WritableStream | undefined + try { + const userId = claims.uid + const integration = await getRepository(Integration).findOneBy({ + user: { id: userId }, + id: req.body.integrationId, + enabled: true, + type: IntegrationType.Import, + }) + if (!integration) { + logger.info('No active integration found for user', { userId }) + return res.status(200).send('No integration found') + } + + const integrationService = getIntegrationService(integration.name) + // import pages from integration + logger.info('importing pages from integration', { + integrationId: integration.id, + }) + + // write the list of urls to a csv file and upload it to gcs + // path style: imports///-.csv + const dateStr = DateTime.now().toISODate() + const fileUuid = uuidv4() + const fullPath = `imports/${userId}/${dateStr}/URL_LIST-${fileUuid}.csv` + // open a write_stream to the file + const file = createGCSFile(fullPath) + writeStream = file.createWriteStream({ + contentType: 'text/csv', + }) + + let hasMore = true + let offset = 0 + let since = integration.syncedAt?.getTime() || 0 + while (hasMore) { + // get pages from integration + const retrieved = await integrationService.retrieve({ + token: integration.token, + since, + offset: offset, + }) + const retrievedData = retrieved.data + if (retrievedData.length === 0) { + break + } + // write the list of urls, state and labels to the stream + const csvData = retrievedData.map((page) => { + const { url, state, labels } = page + return [url, state, `"[${labels?.join(',') || ''}]"`].join(',') + }) + writeStream.write(csvData.join('\n')) + + hasMore = !!retrieved.hasMore + offset += retrievedData.length + since = retrieved.since || Date.now() + } + // update the integration's syncedAt + await getRepository(Integration).update(integration.id, { + syncedAt: new Date(since), + }) + } catch (err) { + logger.error('import pages from integration failed', err) + return res.status(500).send(err) + } finally { + writeStream?.end() + } + + res.status(200).send('OK') + }) return router } diff --git a/packages/api/src/routers/svc/links.ts b/packages/api/src/routers/svc/links.ts index b656203c0..4e6c23ed0 100644 --- a/packages/api/src/routers/svc/links.ts +++ b/packages/api/src/routers/svc/links.ts @@ -3,9 +3,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { kx } from '../../datalayer/knex_config' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { initModels } from '../../server' interface CreateLinkRequestMessage { url: string @@ -39,10 +37,11 @@ export function linkServiceRouter() { } const msg = data as CreateLinkRequestMessage - const models = initModels(kx, false) - try { - const request = await createPageSaveRequest(msg.userId, msg.url, models) + const request = await createPageSaveRequest({ + userId: msg.userId, + url: msg.url, + }) console.log('create link request', request) res.status(200).send(request) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 3beeaff93..afcd9c787 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -488,6 +488,8 @@ const schema = gql` uploadFileId: ID skipParsing: Boolean source: String + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } enum CreateArticleErrorCode { UNABLE_TO_FETCH @@ -530,6 +532,8 @@ const schema = gql` source: String! clientRequestId: ID! uploadFileId: ID! + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } input ParseResult { @@ -554,12 +558,16 @@ const schema = gql` title: String originalContent: String! parseResult: ParseResult + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } input SaveUrlInput { url: String! source: String! clientRequestId: ID! + state: ArticleSavingRequestStatus + labels: [CreateLabelInput!] } union SaveResult = SaveSuccess | SaveError @@ -1073,6 +1081,7 @@ const schema = gql` SUCCEEDED FAILED DELETED + ARCHIVED } type ArticleSavingRequest { @@ -1427,7 +1436,7 @@ const schema = gql` input CreateLabelInput { name: String! @sanitize(maxLength: 64) - color: String! @sanitize(pattern: "^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$") + color: String @sanitize(pattern: "^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$") description: String @sanitize(maxLength: 100) } @@ -1903,6 +1912,7 @@ const schema = gql` type Integration { id: ID! + name: String! type: IntegrationType! token: String! enabled: Boolean! @@ -1911,7 +1921,8 @@ const schema = gql` } enum IntegrationType { - READWISE + EXPORT + IMPORT } type SetIntegrationError { @@ -1928,7 +1939,8 @@ const schema = gql` input SetIntegrationInput { id: ID - type: IntegrationType! + name: String! + type: IntegrationType token: String! enabled: Boolean! } @@ -2416,6 +2428,23 @@ const schema = gql` UNAUTHORIZED } + union ImportFromIntegrationResult = + ImportFromIntegrationSuccess + | ImportFromIntegrationError + + type ImportFromIntegrationSuccess { + success: Boolean! + } + + type ImportFromIntegrationError { + errorCodes: [ImportFromIntegrationErrorCode!]! + } + + enum ImportFromIntegrationErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2506,6 +2535,7 @@ const schema = gql` ): UploadImportFileResult! markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult! bulkAction(query: String, action: BulkActionType!): BulkActionResult! + importFromIntegration(integrationId: ID!): ImportFromIntegrationResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 506046777..deccfade0 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -49,6 +49,7 @@ import { textToSpeechRouter } from './routers/text_to_speech' import * as httpContext from 'express-http-context' import { notificationRouter } from './routers/notification_router' import { userRouter } from './routers/user_router' +import { integrationRouter } from './routers/integration_router' const PORT = process.env.PORT || 4000 @@ -135,6 +136,7 @@ export const createApp = (): { app.use('/api/mobile-auth', mobileAuthRouter()) app.use('/api/text-to-speech', textToSpeechRouter()) app.use('/api/notification', notificationRouter()) + app.use('/api/integration', integrationRouter()) app.use('/svc/pubsub/content', contentServiceRouter()) app.use('/svc/pubsub/links', linkServiceRouter()) app.use('/svc/pubsub/newsletters', newsletterServiceRouter()) diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 7d2b504a5..6285c1d81 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -8,16 +8,26 @@ import { getPageByParam, updatePage, } from '../elastic/pages' -import { ArticleSavingRequestStatus, PageType } from '../elastic/types' +import { ArticleSavingRequestStatus, Label, PageType } from '../elastic/types' +import { User } from '../entity/user' +import { getRepository } from '../entity/utils' import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, } from '../generated/graphql' -// TODO: switch to a proper Entity instead of using the old data models. -import { DataModels } from '../resolvers/types' import { enqueueParseRequest } from '../utils/createTask' import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers' +interface PageSaveRequest { + userId: string + url: string + pubsub?: PubsubClient + articleSavingRequestId?: string + archivedAt?: Date | null + labels?: Label[] + priority?: 'low' | 'high' +} + const SAVING_CONTENT = 'Your link is being saved...' const isPrivateIP = privateIpLib.default @@ -58,14 +68,15 @@ export const validateUrl = (url: string): URL => { return u } -export const createPageSaveRequest = async ( - userId: string, - url: string, - models: DataModels, - pubsub: PubsubClient = createPubSubClient(), +export const createPageSaveRequest = async ({ + userId, + url, + pubsub = createPubSubClient(), articleSavingRequestId = uuidv4(), - priority?: 'low' | 'high' -): Promise => { + archivedAt, + priority, + labels, +}: PageSaveRequest): Promise => { try { validateUrl(url) } catch (error) { @@ -75,7 +86,10 @@ export const createPageSaveRequest = async ( }) } - const user = await models.user.get(userId) + const user = await getRepository(User).findOne({ + where: { id: userId }, + relations: ['profile'], + }) if (!user) { console.log('User not found', userId) return Promise.reject({ @@ -116,6 +130,8 @@ export const createPageSaveRequest = async ( state: ArticleSavingRequestStatus.Processing, createdAt: new Date(), savedAt: new Date(), + archivedAt, + labels, } // create processing page @@ -137,8 +153,20 @@ export const createPageSaveRequest = async ( ctx ) } + const labelsInput = labels?.map((label) => ({ + name: label.name, + color: label.color, + description: label.description, + })) // enqueue task to parse page - await enqueueParseRequest(url, userId, page.id, priority) + await enqueueParseRequest({ + url, + userId, + saveRequestId: page.id, + priority, + state: archivedAt ? ArticleSavingRequestStatus.Archived : undefined, + labels: labelsInput, + }) return pageToArticleSavingRequest(user, page) } diff --git a/packages/api/src/services/integrations.ts b/packages/api/src/services/integrations.ts deleted file mode 100644 index 91d0df654..000000000 --- a/packages/api/src/services/integrations.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { IntegrationType } from '../generated/graphql' -import { env } from '../env' -import axios from 'axios' -import { wait } from '../utils/helpers' -import { HighlightType, Page } from '../elastic/types' -import { getHighlightUrl } from './highlights' -import { Integration } from '../entity/integration' -import { getRepository } from '../entity/utils' - -interface ReadwiseHighlight { - // The highlight text, (technically the only field required in a highlight object) - text: string - // The title of the page the highlight is on - title?: string - // The author of the page the highlight is on - author?: string - // The URL of the page image - image_url?: string - // The URL of the page - source_url?: string - // A meaningful unique identifier for your app - source_type?: string - // One of: books, articles, tweets or podcasts - category?: string - // Annotation note attached to the specific highlight - note?: string - // Highlight's location in the source text. Used to order the highlights - location?: number - // One of: page, order or time_offset - location_type?: string - // A datetime representing when the highlight was taken in the ISO 8601 format - highlighted_at?: string - // Unique url of the specific highlight - highlight_url?: string -} - -export const READWISE_API_URL = 'https://readwise.io/api/v2' - -export const validateToken = async ( - token: string, - type: IntegrationType -): Promise => { - switch (type) { - case IntegrationType.Readwise: - return validateReadwiseToken(token) - default: - return false - } -} - -const validateReadwiseToken = async (token: string): Promise => { - const authUrl = `${env.readwise.apiUrl || READWISE_API_URL}/auth` - try { - const response = await axios.get(authUrl, { - headers: { - Authorization: `Token ${token}`, - }, - }) - return response.status === 204 - } catch (error) { - console.log('error validating readwise token', error) - return false - } -} - -const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => { - if (!page.highlights) return [] - const category = page.siteName === 'Twitter' ? 'tweets' : 'articles' - return ( - page.highlights - // filter out highlights with no quote and are not of type Highlight - .filter( - (highlight) => - highlight.type === HighlightType.Highlight && highlight.quote - ) - .map((highlight) => { - return { - text: highlight.quote!, - title: page.title, - author: page.author || undefined, - highlight_url: getHighlightUrl(page.slug, highlight.id), - highlighted_at: new Date(highlight.createdAt).toISOString(), - category, - image_url: page.image || undefined, - // location: highlight.highlightPositionAnchorIndex || undefined, - location_type: 'order', - note: highlight.annotation || undefined, - source_type: 'omnivore', - source_url: page.url, - } - }) - ) -} - -export const syncWithIntegration = async ( - integration: Integration, - pages: Page[] -): Promise => { - let result = true - switch (integration.type) { - case IntegrationType.Readwise: { - const highlights = pages.flatMap(pageToReadwiseHighlight) - // If there are no highlights, we will skip the sync - if (highlights.length > 0) { - result = await syncWithReadwise(integration.token, highlights) - } - break - } - default: - return false - } - // update integration syncedAt if successful - if (result) { - console.log('updating integration syncedAt') - await getRepository(Integration).update(integration.id, { - syncedAt: new Date(), - }) - } - return result -} - -export const syncWithReadwise = async ( - token: string, - highlights: ReadwiseHighlight[], - retryCount = 0 -): Promise => { - const url = `${env.readwise.apiUrl || READWISE_API_URL}/highlights` - try { - const response = await axios.post( - url, - { - highlights, - }, - { - headers: { - Authorization: `Token ${token}`, - ContentType: 'application/json', - }, - } - ) - return response.status === 200 - } catch (error) { - if (axios.isAxiosError(error)) { - if (error.response) { - if (error.response.status === 429 && retryCount < 3) { - console.log('Readwise API rate limit exceeded, retrying...') - // wait for Retry-After seconds in the header if rate limited - // max retry count is 3 - const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds - await wait(parseInt(retryAfter, 10) * 1000) - return syncWithReadwise(token, highlights, retryCount + 1) - } - // The request was made and the server responded with a status code - // that falls out of the range of 2xx - console.error('Readwise error, response data', error.response.data) - } else if (error.request) { - // The request was made but no response was received - // `error.request` is an instance of XMLHttpRequest in the browser and an instance of - // http.ClientRequest in node.js - console.error('Readwise error, request', error.request) - } else { - // Something happened in setting up the request that triggered an Error - console.error('Error', error.message) - } - } else { - console.error('Error syncing with readwise', error) - } - return false - } -} diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts new file mode 100644 index 000000000..c5946ceed --- /dev/null +++ b/packages/api/src/services/integrations/index.ts @@ -0,0 +1,16 @@ +import { ReadwiseIntegration } from './readwise' +import { IntegrationService } from './integration' +import { PocketIntegration } from './pocket' + +const integrations: IntegrationService[] = [ + new ReadwiseIntegration(), + new PocketIntegration(), +] + +export const getIntegrationService = (name: string): IntegrationService => { + const service = integrations.find((s) => s.name === name) + if (!service) { + throw new Error(`Integration service not found: ${name}`) + } + return service +} diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts new file mode 100644 index 000000000..3c2a44a08 --- /dev/null +++ b/packages/api/src/services/integrations/integration.ts @@ -0,0 +1,37 @@ +import { Integration } from '../../entity/integration' +import { ArticleSavingRequestStatus, Page } from '../../elastic/types' + +export interface RetrievedData { + url: string + labels?: string[] + state?: ArticleSavingRequestStatus +} +export interface RetrievedResult { + data: RetrievedData[] + hasMore?: boolean + since?: number // unix timestamp in milliseconds +} + +export interface RetrieveRequest { + token: string + since?: number // unix timestamp in milliseconds + count?: number + offset?: number +} + +export abstract class IntegrationService { + abstract name: string + + accessToken = async (token: string): Promise => { + return Promise.resolve(null) + } + export = async ( + integration: Integration, + pages: Page[] + ): Promise => { + return Promise.resolve(false) + } + retrieve = async (req: RetrieveRequest): Promise => { + return Promise.resolve({ data: [] }) + } +} diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts new file mode 100644 index 000000000..0b25928ab --- /dev/null +++ b/packages/api/src/services/integrations/pocket.ts @@ -0,0 +1,142 @@ +import { + IntegrationService, + RetrievedResult, + RetrieveRequest, +} from './integration' +import axios from 'axios' +import { env } from '../../env' +import { ArticleSavingRequestStatus } from '../../elastic/types' + +interface PocketResponse { + status: number // 1 if success + complete: number // 1 if all items have been returned + list: { + [key: string]: PocketItem + } + since: number // unix timestamp in seconds + search_meta: { + search_type: string + } + error: string +} + +interface PocketItem { + item_id: string + resolved_id: string + given_url: string + resolved_url: string + given_title: string + resolved_title: string + favorite: string + status: string + excerpt: string + word_count: string + tags?: { + [key: string]: Tag + } + authors?: { + [key: string]: Author + } +} + +interface Tag { + item_id: string + tag: string +} + +interface Author { + item_id: string + author_id: string + name: string +} + +export class PocketIntegration extends IntegrationService { + name = 'POCKET' + POCKET_API_URL = 'https://getpocket.com/v3' + headers = { + 'Content-Type': 'application/json', + 'X-Accept': 'application/json', + } + + accessToken = async (token: string): Promise => { + const url = `${this.POCKET_API_URL}/oauth/authorize` + try { + const response = await axios.post<{ access_token: string }>( + url, + { + consumer_key: env.pocket.consumerKey, + code: token, + }, + { + headers: this.headers, + } + ) + return response.data.access_token + } catch (error) { + console.log('error validating pocket token', error) + return null + } + } + + retrievePocketData = async ( + accessToken: string, + since: number, // unix timestamp in seconds + count = 100, + offset = 0 + ): Promise => { + const url = `${this.POCKET_API_URL}/get` + try { + const response = await axios.post( + url, + { + consumer_key: env.pocket.consumerKey, + access_token: accessToken, + state: 'all', + detailType: 'complete', + since, + sort: 'oldest', + count, + offset, + }, + { + headers: this.headers, + } + ) + console.debug('pocket data', response.data) + return response.data + } catch (error) { + console.log('error retrieving pocket data', error) + throw new Error('Error retrieving pocket data') + } + } + + retrieve = async ({ + token, + since = 0, + count = 100, + offset = 0, + }: RetrieveRequest): Promise => { + const pocketData = await this.retrievePocketData( + token, + since / 1000, + count, + offset + ) + const pocketItems = Object.values(pocketData.list) + const statusToState: Record = { + '0': ArticleSavingRequestStatus.Succeeded, + '1': ArticleSavingRequestStatus.Archived, + '2': ArticleSavingRequestStatus.Deleted, + } + const data = pocketItems.map((item) => ({ + url: item.given_url, + labels: Object.values(item.tags ?? {}).map((tag) => tag.tag), + state: statusToState[item.status], + })) + return { + data, + hasMore: pocketData.complete !== 1, + since: pocketData.since * 1000, + } + } +} diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts new file mode 100644 index 000000000..1f060e87c --- /dev/null +++ b/packages/api/src/services/integrations/readwise.ts @@ -0,0 +1,143 @@ +import axios from 'axios' +import { HighlightType, Page } from '../../elastic/types' +import { Integration } from '../../entity/integration' +import { getRepository } from '../../entity/utils' +import { env } from '../../env' +import { wait } from '../../utils/helpers' +import { getHighlightUrl } from '../highlights' +import { IntegrationService } from './integration' + +interface ReadwiseHighlight { + // The highlight text, (technically the only field required in a highlight object) + text: string + // The title of the page the highlight is on + title?: string + // The author of the page the highlight is on + author?: string + // The URL of the page image + image_url?: string + // The URL of the page + source_url?: string + // A meaningful unique identifier for your app + source_type?: string + // One of: books, articles, tweets or podcasts + category?: string + // Annotation note attached to the specific highlight + note?: string + // Highlight's location in the source text. Used to order the highlights + location?: number + // One of: page, order or time_offset + location_type?: string + // A datetime representing when the highlight was taken in the ISO 8601 format + highlighted_at?: string + // Unique url of the specific highlight + highlight_url?: string +} + +export const READWISE_API_URL = 'https://readwise.io/api/v2' + +export class ReadwiseIntegration extends IntegrationService { + name = 'READWISE' + accessToken = async (token: string): Promise => { + const authUrl = `${env.readwise.apiUrl || READWISE_API_URL}/auth` + try { + const response = await axios.get(authUrl, { + headers: { + Authorization: `Token ${token}`, + }, + }) + return response.status === 204 ? token : null + } catch (error) { + console.log('error validating readwise token', error) + return null + } + } + export = async ( + integration: Integration, + pages: Page[] + ): Promise => { + let result = true + + const highlights = pages.flatMap(this.pageToReadwiseHighlight) + // If there are no highlights, we will skip the sync + if (highlights.length > 0) { + result = await this.syncWithReadwise(integration.token, highlights) + } + + // update integration syncedAt if successful + if (result) { + console.log('updating integration syncedAt') + await getRepository(Integration).update(integration.id, { + syncedAt: new Date(), + }) + } + return result + } + + pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => { + const { highlights } = page + if (!highlights) return [] + const category = page.siteName === 'Twitter' ? 'tweets' : 'articles' + return highlights + .map((highlight) => { + // filter out highlights that are not of type highlight or have no quote + if (highlight.type !== HighlightType.Highlight || !highlight.quote) { + return undefined + } + + return { + text: highlight.quote, + title: page.title, + author: page.author || undefined, + highlight_url: getHighlightUrl(page.slug, highlight.id), + highlighted_at: new Date(highlight.createdAt).toISOString(), + category, + image_url: page.image || undefined, + // location: highlight.highlightPositionAnchorIndex || undefined, + location_type: 'order', + note: highlight.annotation || undefined, + source_type: 'omnivore', + source_url: page.url, + } + }) + .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[] + } + + syncWithReadwise = async ( + token: string, + highlights: ReadwiseHighlight[], + retryCount = 0 + ): Promise => { + const url = `${env.readwise.apiUrl || READWISE_API_URL}/highlights` + try { + const response = await axios.post( + url, + { + highlights, + }, + { + headers: { + Authorization: `Token ${token}`, + ContentType: 'application/json', + }, + } + ) + return response.status === 200 + } catch (error) { + if ( + axios.isAxiosError(error) && + error.response?.status === 429 && + retryCount < 3 + ) { + console.log('Readwise API rate limit exceeded, retrying...') + // wait for Retry-After seconds in the header if rate limited + // max retry count is 3 + const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds + await wait(parseInt(retryAfter, 10) * 1000) + return this.syncWithReadwise(token, highlights, retryCount + 1) + } + console.log('Error creating highlights in Readwise', error) + return false + } + } +} diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index b20baeee2..102265155 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -1,11 +1,12 @@ -import { Label } from '../entity/label' -import { ILike, In } from 'typeorm' -import { PageContext } from '../elastic/types' -import { User } from '../entity/user' -import { addLabelInPage } from '../elastic/labels' -import { getRepository } from '../entity/utils' -import { Link } from '../entity/link' import DataLoader from 'dataloader' +import { In } from 'typeorm' +import { addLabelInPage } from '../elastic/labels' +import { PageContext } from '../elastic/types' +import { Label } from '../entity/label' +import { Link } from '../entity/link' +import { User } from '../entity/user' +import { getRepository } from '../entity/utils' +import { CreateLabelInput } from '../generated/graphql' import { generateRandomColor } from '../utils/helpers' const batchGetLabelsFromLinkIds = async ( @@ -39,10 +40,11 @@ export const addLabelToPage = async ( return false } - let labelEntity = await getRepository(Label).findOneBy({ - user: { id: user.id }, - name: ILike(label.name), - }) + let labelEntity = await getRepository(Label) + .createQueryBuilder() + .where({ user: { id: user.id } }) + .andWhere('LOWER(name) = LOWER(:name)', { name: label.name }) + .getOne() if (!labelEntity) { console.log('creating new label', label.name) @@ -86,10 +88,11 @@ export const createLabel = async ( description?: string } ): Promise