From d67aea6d8a5376e18f788331c721ccda73f0d675 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 22 Mar 2024 17:16:33 +0800 Subject: [PATCH] deduplicate notion page by omnivore id --- packages/api/src/pubsub.ts | 8 ++-- packages/api/src/services/highlights.ts | 22 +++++---- .../api/src/services/integrations/notion.ts | 20 +++----- .../api/src/services/integrations/readwise.ts | 6 +-- packages/api/src/services/labels.ts | 46 +++++++++---------- packages/api/src/services/library_item.ts | 5 +- 6 files changed, 51 insertions(+), 56 deletions(-) diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 27babb13b..4cb0116d4 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -115,10 +115,10 @@ export const createPubSubClient = (): PubsubClient => { } export enum EntityType { - ITEM = 'page', - HIGHLIGHT = 'highlight', - LABEL = 'label', - RSS_FEED = 'feed', + ITEM = 'PAGE', + HIGHLIGHT = 'HIGHLIGHT', + LABEL = 'LABEL', + RSS_FEED = 'FEED', } export interface PubsubClient { diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 469949bb1..57bb417b1 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -69,6 +69,8 @@ export const createHighlight = async ( { id: libraryItemId, slug: newHighlight.libraryItem.slug, + originalUrl: newHighlight.libraryItem.originalUrl, + thumbnail: newHighlight.libraryItem.thumbnail, highlights: [cleanData], }, userId @@ -121,6 +123,8 @@ export const mergeHighlights = async ( { id: libraryItemId, slug: newHighlight.libraryItem.slug, + originalUrl: newHighlight.libraryItem.originalUrl, + thumbnail: newHighlight.libraryItem.thumbnail, highlights: [newHighlight], }, userId @@ -154,15 +158,15 @@ export const updateHighlight = async ( }) const libraryItemId = updatedHighlight.libraryItem.id - await pubsub.entityUpdated( - EntityType.HIGHLIGHT, - { - id: libraryItemId, - slug: updatedHighlight.libraryItem.slug, - highlights: [highlight], - } as ItemEvent, - userId - ) + // await pubsub.entityUpdated( + // EntityType.HIGHLIGHT, + // { + // id: libraryItemId, + // slug: updatedHighlight.libraryItem.slug, + // highlights: [highlight], + // } as ItemEvent, + // userId + // ) await enqueueUpdateHighlight({ libraryItemId, diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 0358d6732..5b15639b7 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -63,10 +63,7 @@ interface NotionPage { 'Original URL'?: { url: string } - 'Omnivore ID': { - url: string - } - 'Omnivore URL'?: { + 'Omnivore URL': { url: string } 'Saved At'?: { @@ -230,14 +227,9 @@ export class NotionClient implements IntegrationClient { url: item.originalUrl, } : undefined, - 'Omnivore ID': { - url: item.id, + 'Omnivore URL': { + url: `${env.client.url}/me/${item.id}`, }, - 'Omnivore URL': item.slug - ? { - url: `${env.client.url}/me/${item.slug}`, - } - : undefined, 'Saved At': item.savedAt ? { date: { @@ -312,14 +304,14 @@ export class NotionClient implements IntegrationClient { await this.client.pages.create(page) } - private findPage = async (id: string, databaseId: string) => { + private findPage = async (url: string, databaseId: string) => { const response = await this.client.databases.query({ database_id: databaseId, page_size: 1, filter: { - property: 'Omnivore ID', + property: 'Omnivore URL', url: { - equals: id, + equals: url, }, }, }) diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index dc17f3e87..001820837 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -101,16 +101,14 @@ export class ReadwiseClient implements IntegrationClient { text: highlight.quote, title: item.title, author: item.author || undefined, - highlight_url: item.slug - ? getHighlightUrl(item.slug, highlight.id) - : undefined, + highlight_url: getHighlightUrl(item.slug as string, highlight.id), highlighted_at: highlight.createdAt as string | undefined, category, image_url: item.thumbnail || undefined, location_type: 'order', note: highlight.annotation || undefined, source_type: 'omnivore', - source_url: item.originalUrl, + source_url: item.originalUrl as string, } }) : [] diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index c2f5c110e..f4987a4b8 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -13,8 +13,6 @@ import { CreateLabelInput, labelRepository } from '../repository/label' import { Merge } from '../util' import { bulkEnqueueUpdateLabels } from '../utils/createTask' import { deepDelete } from '../utils/helpers' -import { logger } from '../utils/logger' -import { findHighlightById } from './highlights' import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item' const columnToDelete = ['description', 'createdAt'] as const @@ -213,30 +211,30 @@ export const saveLabelsInHighlight = async ( ) }) - const highlight = await findHighlightById(highlightId, userId) - if (!highlight) { - logger.error('Highlight not found', { highlightId, userId }) - return - } + // const highlight = await findHighlightById(highlightId, userId) + // if (!highlight) { + // logger.error('Highlight not found', { highlightId, userId }) + // return + // } - const libraryItemId = highlight.libraryItemId - // create pubsub event - await pubsub.entityCreated( - EntityType.LABEL, - { - id: libraryItemId, - highlights: [ - { - id: highlightId, - labels: labels.map((l) => deepDelete(l, columnToDelete)), - }, - ], - }, - userId - ) + // const libraryItemId = highlight.libraryItemId + // // create pubsub event + // await pubsub.entityCreated( + // EntityType.LABEL, + // { + // id: libraryItemId, + // highlights: [ + // { + // id: highlightId, + // labels: labels.map((l) => deepDelete(l, columnToDelete)), + // }, + // ], + // }, + // userId + // ) - // update labels in library item - await bulkEnqueueUpdateLabels([{ libraryItemId, userId }]) + // // update labels in library item + // await bulkEnqueueUpdateLabels([{ libraryItemId, userId }]) } export const findLabelsByIds = async ( diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4cc0806dd..4169ddc76 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -895,7 +895,10 @@ export const updateLibraryItem = async ( await pubsub.entityUpdated( EntityType.ITEM, - { ...libraryItem, id } as ItemEvent, + { + ...libraryItem, + id, + } as ItemEvent, userId )