Merge pull request #3947 from omnivore-app/fix/content-api

fix/content api

packages/api/src/jobs/upload_content.ts

@@ -1,3 +1,4 @@
+import { Highlight } from '../entity/highlight'
 import { findLibraryItemById } from '../services/library_item'
 import { logger } from '../utils/logger'
 import { htmlToHighlightedMarkdown, htmlToMarkdown } from '../utils/parser'
@@ -14,12 +15,16 @@ export interface UploadContentJobData {
   filePath: string
 }
 
-const convertContent = (content: string, format: ContentFormat): string => {
+const convertContent = (
+  content: string,
+  format: ContentFormat,
+  highlights?: Highlight[]
+): string => {
   switch (format) {
     case 'markdown':
       return htmlToMarkdown(content)
     case 'highlightedMarkdown':
-      return htmlToHighlightedMarkdown(content)
+      return htmlToHighlightedMarkdown(content, highlights)
     case 'original':
       return content
     default:
@@ -33,29 +38,62 @@ const CONTENT_TYPES = {
   original: 'text/html',
 }
 
+const getSelectOptions = (
+  format: ContentFormat
+): { column: 'readableContent' | 'originalContent'; highlights?: boolean } => {
+  switch (format) {
+    case 'markdown':
+      return {
+        column: 'readableContent',
+      }
+    case 'highlightedMarkdown':
+      return {
+        column: 'readableContent',
+        highlights: true,
+      }
+    case 'original':
+      return {
+        column: 'originalContent',
+      }
+    default:
+      throw new Error('Unsupported format')
+  }
+}
+
 export const uploadContentJob = async (data: UploadContentJobData) => {
   logger.info('Uploading content to bucket', data)
 
   const { libraryItemId, userId, format, filePath } = data
+
+  const { column, highlights } = getSelectOptions(format)
   const libraryItem = await findLibraryItemById(libraryItemId, userId, {
-    select: ['originalContent'],
+    select: ['id', column], // id is required for relations
+    relations: {
+      highlights,
+    },
   })
   if (!libraryItem) {
     logger.error('Library item not found', data)
     throw new Error('Library item not found')
   }
 
-  if (!libraryItem.originalContent) {
-    logger.error('Original content not found', data)
-    throw new Error('Original content not found')
+  const content = libraryItem[column]
+
+  if (!content) {
+    logger.error(`${column} not found`, data)
+    throw new Error('Content not found')
   }
 
   logger.info('Converting content', data)
-  const content = convertContent(libraryItem.originalContent, format)
+  const convertedContent = convertContent(
+    content,
+    format,
+    libraryItem.highlights
+  )
 
   console.time('uploadToBucket')
   logger.info('Uploading content', data)
-  await uploadToBucket(filePath, Buffer.from(content), {
+  await uploadToBucket(filePath, Buffer.from(convertedContent), {
     contentType: CONTENT_TYPES[format],
     timeout: 60000, // 1 minute
   })

packages/content-fetch/src/analytics.ts

@@ -1,14 +1,12 @@
 import { PostHog } from 'posthog-node'
 
 interface AnalyticEvent {
-  distinctId: string
-  event: string
+  result: 'success' | 'failure'
   properties?: Record<string | number, any>
 }
 
 interface AnalyticClient {
-  capture: (event: AnalyticEvent) => void
-  shutdownAsync?: () => Promise<void>
+  capture: (userIds: string[], event: AnalyticEvent) => void
 }
 
 class PostHogClient implements AnalyticClient {
@@ -18,21 +16,23 @@ class PostHogClient implements AnalyticClient {
     this.client = new PostHog(apiKey)
   }
 
-  capture({ distinctId, event, properties }: AnalyticEvent) {
-    // get client from request context
+  capture(userIds: string[], { properties, result }: AnalyticEvent) {
+    if (process.env.SEND_ANALYTICS) {
+      userIds.forEach((userId) => {
+        this.client.capture({
+          distinctId: userId,
+          event: `content_fetch_${result}`,
+          properties: {
+            ...properties,
+            env: process.env.API_ENV,
+          },
+        })
+      })
 
-    this.client.capture({
-      distinctId,
-      event,
-      properties: {
-        ...properties,
-        env: process.env.API_ENV || 'demo',
-      },
-    })
-  }
+      return
+    }
 
-  async shutdownAsync() {
-    return this.client.shutdownAsync()
+    console.log('analytics', { userIds, result, properties })
   }
 }
 

packages/content-fetch/src/request_handler.ts

@@ -148,33 +148,23 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => {
       logRecord.error = 'unknown error'
     }
 
-    // capture error event
-    users.forEach((user) => {
-      analytics.capture({
-        distinctId: user.id,
-        event: 'content_fetch_failure',
-        properties: {
-          url,
-        },
-      })
-    })
-
     return res.sendStatus(500)
   } finally {
     logRecord.totalTime = Date.now() - functionStartTime
     console.log(`parse-page result`, logRecord)
-  }
 
-  // capture success event
-  users.forEach((user) => {
-    analytics.capture({
-      distinctId: user.id,
-      event: 'content_fetch_success',
-      properties: {
-        url,
-      },
-    })
-  })
+    // capture events
+    analytics.capture(
+      users.map((user) => user.id),
+      {
+        result: logRecord.error ? 'failure' : 'success',
+        properties: {
+          url,
+          totalTime: logRecord.totalTime,
+        },
+      }
+    )
+  }
 
   res.sendStatus(200)
 }