use private bucket to upload page events (#244)

* use private bucket to upload page events

* fix tests

* add GCS_UPLOAD_PRIVATE_BUCKET in test env

* allow GCS_UPLOAD_PRIVATE_BUCKET to be empty

packages/api/.env.test

@@ -21,6 +21,7 @@ JAEGER_HOST=
 SAMPLE_METRICS_LOCALLY=FALSE
 GCS_UPLOAD_BUCKET=
 GCS_UPLOAD_SA_KEY_FILE_PATH=
+GCS_UPLOAD_PRIVATE_BUCKET=
 TWITTER_BEARER_TOKEN=
 PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
 SEGMENT_WRITE_KEY='test'

packages/api/src/datalayer/pubsub.ts

@@ -43,8 +43,8 @@ export const createPubSubClient = (): PubsubClient => {
     pageCreated: (page: Page): Promise<void> => {
       return publish('pageCreated', Buffer.from(JSON.stringify(page)))
     },
-    pageDeleted: (id: string): Promise<void> => {
-      return publish('pageDeleted', Buffer.from(JSON.stringify({ id })))
+    pageDeleted: (id: string, userId: string): Promise<void> => {
+      return publish('pageDeleted', Buffer.from(JSON.stringify({ id, userId })))
     },
     reportSubmitted: (
       submitterId: string,
@@ -71,7 +71,7 @@ export interface PubsubClient {
   ) => Promise<void>
   pageCreated: (page: Page) => Promise<void>
   pageSaved: (page: Partial<Page>) => Promise<void>
-  pageDeleted: (id: string) => Promise<void>
+  pageDeleted: (id: string, userId: string) => Promise<void>
   reportSubmitted(
     submitterId: string | undefined,
     itemUrl: string,

packages/api/src/elastic/index.ts

@@ -202,6 +202,7 @@ export const updatePage = async (
 
 export const deletePage = async (
   id: string,
+  userId: string,
   ctx: PageContext
 ): Promise<boolean> => {
   try {
@@ -213,7 +214,7 @@ export const deletePage = async (
 
     if (body.deleted === 0) return false
 
-    await ctx.pubsub.pageDeleted(id)
+    await ctx.pubsub.pageDeleted(id, userId)
 
     return true
   } catch (e) {

packages/api/src/resolvers/article/index.ts

@@ -626,7 +626,7 @@ export const setBookmarkArticleResolver = authorized<
         return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] }
       }
 
-      await deletePage(userArticleRemoved.id, { pubsub })
+      await deletePage(userArticleRemoved.id, uid, { pubsub })
 
       const highlightsUnshared = await authTrx(async (tx) => {
         return models.highlight.unshareAllHighlights(articleID, uid, tx)

packages/api/src/routers/svc/pages.ts

@@ -5,6 +5,8 @@ import express from 'express'
 import { readPushSubscription } from '../../datalayer/pubsub'
 import { generateUploadSignedUrl, uploadToSignedUrl } from '../../utils/uploads'
 import { v4 as uuidv4 } from 'uuid'
+import { env } from '../../env'
+import { Page } from '../../elastic/types'
 
 export function pageServiceRouter() {
   const router = express.Router()
@@ -25,10 +27,21 @@ export function pageServiceRouter() {
     }
 
     try {
+      const data: Partial<Page> = JSON.parse(msgStr)
+      if (!data.userId) {
+        console.log('No userId found in message')
+        res.status(400).send('Bad Request')
+        return
+      }
+
       const contentType = 'application/json'
+      const bucketName = env.fileUpload.gcsUploadPrivateBucket
       const uploadUrl = await generateUploadSignedUrl(
-        `${req.params.folder}/${new Date().toDateString()}/${uuidv4()}.json`,
-        contentType
+        `${req.params.folder}/${
+          data.userId
+        }/${new Date().toDateString()}/${uuidv4()}.json`,
+        contentType,
+        bucketName
       )
       await uploadToSignedUrl(
         uploadUrl,

packages/api/src/util.ts

@@ -65,6 +65,7 @@ interface BackendEnv {
   fileUpload: {
     gcsUploadBucket: string
     gcsUploadSAKeyFilePath: string
+    gcsUploadPrivateBucket: string
   }
   elastic: {
     url: string
@@ -110,6 +111,7 @@ const nullableEnvVars = [
   'TWITTER_BEARER_TOKEN',
   'ELASTIC_USERNAME',
   'ELASTIC_PASSWORD',
+  'GCS_UPLOAD_PRIVATE_BUCKET',
 ] // Allow some vars to be null/empty
 
 /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
@@ -202,6 +204,7 @@ export function getEnv(): BackendEnv {
   const fileUpload = {
     gcsUploadBucket: parse('GCS_UPLOAD_BUCKET'),
     gcsUploadSAKeyFilePath: parse('GCS_UPLOAD_SA_KEY_FILE_PATH'),
+    gcsUploadPrivateBucket: parse('GCS_UPLOAD_PRIVATE_BUCKET'),
   }
   const elastic = {
     url: parse('ELASTIC_URL'),

packages/api/src/utils/uploads.ts

@@ -17,7 +17,8 @@ const bucketName = env.fileUpload.gcsUploadBucket
 
 export const generateUploadSignedUrl = async (
   filePathName: string,
-  contentType: string
+  contentType: string,
+  selectedBucket?: string
 ): Promise<string> => {
   if (env.dev.isLocal) {
     return 'http://localhost:3000/uploads/' + filePathName
@@ -33,7 +34,7 @@ export const generateUploadSignedUrl = async (
 
   // Get a v4 signed URL for uploading file
   const [url] = await storage
-    .bucket(bucketName)
+    .bucket(selectedBucket || bucketName)
     .file(filePathName)
     .getSignedUrl(options)
   return url

packages/api/test/elastic/index.test.ts

@@ -15,6 +15,7 @@ import { createPubSubClient } from '../../src/datalayer/pubsub'
 
 describe('elastic api', () => {
   const ctx: PageContext = { pubsub: createPubSubClient(), refresh: true }
+  const userId = 'userId'
 
   let page: Page
 
@@ -23,7 +24,7 @@ describe('elastic api', () => {
     page = {
       id: '',
       hash: 'test hash',
-      userId: 'test userId',
+      userId: userId,
       pageType: PageType.Article,
       title: 'test title',
       content: '<p>test</p>',
@@ -58,7 +59,7 @@ describe('elastic api', () => {
 
   after(async () => {
     // delete the testing page
-    await deletePage(page.id, ctx)
+    await deletePage(page.id, userId, ctx)
   })
 
   describe('createPage', () => {
@@ -66,7 +67,7 @@ describe('elastic api', () => {
 
     after(async () => {
       if (newPageId) {
-        await deletePage(newPageId, ctx)
+        await deletePage(newPageId, userId, ctx)
       }
     })
 

packages/api/test/resolvers/article.test.ts

@@ -281,7 +281,7 @@ describe('Article API', () => {
       })
 
       after(async () => {
-        await deletePage(pageId, ctx)
+        await deletePage(pageId, user.id, ctx)
       })
 
       it('should create an article', async () => {
@@ -320,7 +320,7 @@ describe('Article API', () => {
 
     after(async () => {
       if (pageId) {
-        await deletePage(pageId, ctx)
+        await deletePage(pageId, user.id, ctx)
       }
     })
 
@@ -564,7 +564,7 @@ describe('Article API', () => {
 
     after(async () => {
       if (pageId) {
-        await deletePage(pageId, ctx)
+        await deletePage(pageId, user.id, ctx)
       }
     })
 
@@ -612,7 +612,7 @@ describe('Article API', () => {
 
     after(async () => {
       if (pageId) {
-        await deletePage(pageId, ctx)
+        await deletePage(pageId, user.id, ctx)
       }
     })
 

packages/api/test/resolvers/highlight.test.ts

@@ -72,7 +72,7 @@ describe('Highlights API', () => {
   after(async () => {
     await deleteTestUser(username)
     if (pageId) {
-      await deletePage(pageId, ctx)
+      await deletePage(pageId, user.id, ctx)
     }
   })