From 4e388de7edb77da856830df1df0e0fbe1425a6cf Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Jul 2024 11:41:20 +0800 Subject: [PATCH 1/5] fix: export highlight in yellow in notion if color is null --- packages/api/src/services/integrations/notion.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 3732986dd..ab558989c 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -277,7 +277,9 @@ export class NotionClient implements IntegrationClient { }, annotations: { code: true, - color: highlight.color as AnnotationColor, + color: highlight.color + ? (highlight.color as AnnotationColor) + : 'yellow', }, }, ], From 2e322955f645353a9a725198cbbd38aba9f5908e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Jul 2024 12:24:25 +0800 Subject: [PATCH 2/5] fix: title not exported when item is not in notion and gets exported by attaching labels --- .../api/src/services/integrations/notion.ts | 39 +++++++++++++++++-- packages/api/src/services/library_item.ts | 4 +- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index ab558989c..b7719bac4 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -3,11 +3,12 @@ import { GetDatabaseResponse } from '@notionhq/client/build/src/api-endpoints' import axios from 'axios' import { HighlightType } from '../../entity/highlight' import { Integration } from '../../entity/integration' +import { User } from '../../entity/user' import { env } from '../../env' import { Merge } from '../../util' import { logger } from '../../utils/logger' import { getHighlightUrl } from '../highlights' -import { getItemUrl, ItemEvent } from '../library_item' +import { findLibraryItemById, getItemUrl, ItemEvent } from '../library_item' import { IntegrationClient } from './integration' type AnnotationColor = @@ -178,11 +179,38 @@ export class NotionClient implements IntegrationClient { return Promise.resolve(env.notion.authUrl) } - private itemToNotionPage = ( + private itemToNotionPage = async ( + userId: string, item: ItemEvent, settings: Settings, lastSync?: Date | null - ): NotionPage => { + ): Promise => { + if (!item.updatedAt) { + const libraryItem = await findLibraryItemById(item.id, userId, { + select: [ + 'originalUrl', + 'title', + 'author', + 'thumbnail', + 'updatedAt', + 'siteIcon', + 'savedAt', + ], + }) + + if (!libraryItem) { + throw new Error('Library item not found') + } + + item.originalUrl = libraryItem.originalUrl + item.title = libraryItem.title + item.author = libraryItem.author + item.thumbnail = libraryItem.thumbnail + item.updatedAt = libraryItem.updatedAt + item.siteIcon = libraryItem.siteIcon + item.savedAt = libraryItem.savedAt + } + return { parent: { database_id: settings.parentDatabaseId, @@ -339,10 +367,13 @@ export class NotionClient implements IntegrationClient { return false } + const userId = this.integrationData.user.id + await Promise.all( items.map(async (item) => { try { - const notionPage = this.itemToNotionPage( + const notionPage = await this.itemToNotionPage( + userId, item, settings, this.integrationData?.syncedAt diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index facb0b8c8..039ada8b4 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -963,6 +963,7 @@ export const updateLibraryItem = async ( EntityType.ITEM, { ...data, + updatedAt: new Date(), id, } as ItemEvent, userId @@ -1012,7 +1013,8 @@ export const updateLibraryItemReadingProgress = async ( reading_progress_top_percent as "readingProgressTopPercent", reading_progress_bottom_percent as "readingProgressBottomPercent", reading_progress_highest_read_anchor as "readingProgressHighestReadAnchor", - read_at as "readAt" + read_at as "readAt", + updated_at as "updatedAt" `, [id, topPercent, bottomPercent, anchorIndex] ), From 225eefd73bcfe0aba10abbcc517a1b38c84f261e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Jul 2024 13:13:50 +0800 Subject: [PATCH 3/5] fix: readwise export not working for PAGE and LABEL events --- packages/api/src/entity/integration.ts | 3 + packages/api/src/services/highlights.ts | 30 +------- .../api/src/services/integrations/index.ts | 2 +- .../api/src/services/integrations/notion.ts | 72 ++++++++++--------- .../api/src/services/integrations/readwise.ts | 41 ++++++++++- packages/api/src/services/library_item.ts | 15 +++- 6 files changed, 94 insertions(+), 69 deletions(-) diff --git a/packages/api/src/entity/integration.ts b/packages/api/src/entity/integration.ts index dcb7f03c1..7f1f5fe97 100644 --- a/packages/api/src/entity/integration.ts +++ b/packages/api/src/entity/integration.ts @@ -30,6 +30,9 @@ export class Integration { @JoinColumn({ name: 'user_id' }) user!: User + @Column('uuid', { name: 'user_id' }) + userId!: string + @Column('varchar', { length: 40 }) name!: string diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 4ea812f30..2585fb11f 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -72,10 +72,6 @@ export const createHighlight = async ( const newHighlight = await repo.createAndSave(highlight) return repo.findOneOrFail({ where: { id: newHighlight.id }, - relations: { - user: true, - libraryItem: true, - }, }) }, { @@ -89,11 +85,7 @@ export const createHighlight = async ( { id: libraryItemId, highlights: [data], - // for Readwise - originalUrl: newHighlight.libraryItem.originalUrl, - title: newHighlight.libraryItem.title, - author: newHighlight.libraryItem.author, - thumbnail: newHighlight.libraryItem.thumbnail, + updatedAt: new Date(), }, userId ) @@ -133,10 +125,6 @@ export const mergeHighlights = async ( return highlightRepo.findOneOrFail({ where: { id: newHighlight.id }, - relations: { - user: true, - libraryItem: true, - }, }) }) @@ -144,11 +132,8 @@ export const mergeHighlights = async ( EntityType.HIGHLIGHT, { id: libraryItemId, - originalUrl: newHighlight.libraryItem.originalUrl, - title: newHighlight.libraryItem.title, - author: newHighlight.libraryItem.author, - thumbnail: newHighlight.libraryItem.thumbnail, highlights: [newHighlight], + updatedAt: new Date(), }, userId ) @@ -173,10 +158,6 @@ export const updateHighlight = async ( return highlightRepo.findOneOrFail({ where: { id: highlightId }, - relations: { - libraryItem: true, - user: true, - }, }) }) @@ -185,10 +166,6 @@ export const updateHighlight = async ( EntityType.HIGHLIGHT, { id: libraryItemId, - originalUrl: updatedHighlight.libraryItem.originalUrl, - title: updatedHighlight.libraryItem.title, - author: updatedHighlight.libraryItem.author, - thumbnail: updatedHighlight.libraryItem.thumbnail, highlights: [ { ...highlight, @@ -219,9 +196,6 @@ export const deleteHighlightById = async ( const highlightRepo = tx.withRepository(highlightRepository) const highlight = await highlightRepo.findOneOrFail({ where: { id: highlightId }, - relations: { - user: true, - }, }) await highlightRepo.delete(highlightId) diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts index a8b11f916..9d4e195af 100644 --- a/packages/api/src/services/integrations/index.ts +++ b/packages/api/src/services/integrations/index.ts @@ -13,7 +13,7 @@ export const getIntegrationClient = ( ): IntegrationClient => { switch (name.toLowerCase()) { case 'readwise': - return new ReadwiseClient(token) + return new ReadwiseClient(token, integrationData) case 'pocket': return new PocketClient(token) case 'notion': diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index b7719bac4..215658891 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -3,12 +3,11 @@ import { GetDatabaseResponse } from '@notionhq/client/build/src/api-endpoints' import axios from 'axios' import { HighlightType } from '../../entity/highlight' import { Integration } from '../../entity/integration' -import { User } from '../../entity/user' import { env } from '../../env' import { Merge } from '../../util' import { logger } from '../../utils/logger' import { getHighlightUrl } from '../highlights' -import { findLibraryItemById, getItemUrl, ItemEvent } from '../library_item' +import { findLibraryItemsByIds, getItemUrl, ItemEvent } from '../library_item' import { IntegrationClient } from './integration' type AnnotationColor = @@ -179,38 +178,11 @@ export class NotionClient implements IntegrationClient { return Promise.resolve(env.notion.authUrl) } - private itemToNotionPage = async ( - userId: string, + private itemToNotionPage = ( item: ItemEvent, settings: Settings, lastSync?: Date | null - ): Promise => { - if (!item.updatedAt) { - const libraryItem = await findLibraryItemById(item.id, userId, { - select: [ - 'originalUrl', - 'title', - 'author', - 'thumbnail', - 'updatedAt', - 'siteIcon', - 'savedAt', - ], - }) - - if (!libraryItem) { - throw new Error('Library item not found') - } - - item.originalUrl = libraryItem.originalUrl - item.title = libraryItem.title - item.author = libraryItem.author - item.thumbnail = libraryItem.thumbnail - item.updatedAt = libraryItem.updatedAt - item.siteIcon = libraryItem.siteIcon - item.savedAt = libraryItem.savedAt - } - + ): NotionPage => { return { parent: { database_id: settings.parentDatabaseId, @@ -367,13 +339,45 @@ export class NotionClient implements IntegrationClient { return false } - const userId = this.integrationData.user.id + const userId = this.integrationData.userId + + // fetch the original url if not found + if (!items[0].originalUrl) { + const libraryItems = await findLibraryItemsByIds( + items.map((item) => item.id), + userId, + { + select: [ + 'id', + 'originalUrl', + 'title', + 'author', + 'thumbnail', + 'siteIcon', + 'savedAt', + ], + } + ) + + items.forEach((item) => { + const libraryItem = libraryItems.find((li) => li.id === item.id) + if (!libraryItem) { + return + } + + item.originalUrl = libraryItem.originalUrl + item.title = libraryItem.title + item.author = libraryItem.author + item.thumbnail = libraryItem.thumbnail + item.siteIcon = libraryItem.siteIcon + item.savedAt = libraryItem.savedAt + }) + } await Promise.all( items.map(async (item) => { try { - const notionPage = await this.itemToNotionPage( - userId, + const notionPage = this.itemToNotionPage( item, settings, this.integrationData?.syncedAt diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 2a6ea2fd7..360b961b2 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,8 +1,9 @@ import axios from 'axios' import { HighlightType } from '../../entity/highlight' +import { Integration } from '../../entity/integration' import { logger } from '../../utils/logger' import { getHighlightUrl } from '../highlights' -import { getItemUrl, ItemEvent } from '../library_item' +import { findLibraryItemsByIds, getItemUrl, ItemEvent } from '../library_item' import { IntegrationClient } from './integration' interface ReadwiseHighlight { @@ -43,9 +44,11 @@ export class ReadwiseClient implements IntegrationClient { baseURL: 'https://readwise.io/api/v2', timeout: 5000, // 5 seconds }) + private integrationData?: Integration - constructor(token: string) { + constructor(token: string, integration?: Integration) { this.token = token + this.integrationData = integration } accessToken = async (): Promise => { @@ -68,10 +71,42 @@ export class ReadwiseClient implements IntegrationClient { } export = async (items: ItemEvent[]): Promise => { - let result = true + if (!this.integrationData) { + logger.error('Integration data is missing') + return false + } + + if ( + items.every((item) => !item.highlights || item.highlights.length === 0) + ) { + return false + } + + const userId = this.integrationData.userId + const libraryItems = await findLibraryItemsByIds( + items.map((item) => item.id), + userId, + { + select: ['id', 'title', 'author', 'thumbnail', 'siteName'], + } + ) + console.log(libraryItems) + + items.forEach((item) => { + const libraryItem = libraryItems.find((li) => li.id === item.id) + if (!libraryItem) { + return + } + + item.title = libraryItem.title + item.author = libraryItem.author + item.thumbnail = libraryItem.thumbnail + item.siteName = libraryItem.siteName + }) const highlights = items.flatMap(this._itemToReadwiseHighlight) + let result = true // If there are no highlights, we will skip the sync if (highlights.length > 0) { result = await this._syncWithReadwise(highlights) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 039ada8b4..b7c838202 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -794,6 +794,7 @@ export const findLibraryItemsByIds = async ( userId?: string, options?: { select?: (keyof LibraryItem)[] + relations?: Array<'labels' | 'highlights'> } ) => { const selectColumns = @@ -802,12 +803,20 @@ export const findLibraryItemsByIds = async ( .filter((column) => column !== 'originalContent') .map((column) => `library_item.${column}`) return authTrx( - async (tx) => - tx + async (tx) => { + const qb = tx .createQueryBuilder(LibraryItem, 'library_item') .select(selectColumns) .where('library_item.id IN (:...ids)', { ids }) - .getMany(), + + if (options?.relations) { + options.relations.forEach((relation) => { + qb.leftJoinAndSelect(`library_item.${relation}`, relation) + }) + } + + return qb.getMany() + }, { uid: userId, replicationMode: 'replica', From d36d9cdb57a945b9681525dfe4fda84552353776 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Jul 2024 13:14:01 +0800 Subject: [PATCH 4/5] fix: update highlight --- packages/api/src/services/highlights.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 2585fb11f..468607c51 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -161,7 +161,7 @@ export const updateHighlight = async ( }) }) - const libraryItemId = updatedHighlight.libraryItem.id + const libraryItemId = updatedHighlight.libraryItemId await pubsub.entityUpdated( EntityType.HIGHLIGHT, { From 1965f0ac19846546db3802b0055eaaa6944802fc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 11 Jul 2024 13:38:22 +0800 Subject: [PATCH 5/5] fix tests --- packages/api/src/services/highlights.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 468607c51..07dc5fe2c 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -208,7 +208,7 @@ export const deleteHighlightById = async ( await enqueueUpdateHighlight({ libraryItemId: deletedHighlight.libraryItemId, - userId: deletedHighlight.user.id, + userId: deletedHighlight.userId, }) return deletedHighlight