From e47415cf690f9eb9d4200f338510ec3017bc57d7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Jun 2022 22:19:41 +0800 Subject: [PATCH 1/6] Add new environment variable for content-fetch url in gcf --- packages/api/src/util.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index ead205a52..e7aa61519 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -59,7 +59,8 @@ interface BackendEnv { queue: { location: string name: string - puppeteerTaskHanderUrl: string + contentFetchUrl: string + contentFetchGCFUrl: string reminderTaskHanderUrl: string } fileUpload: { @@ -191,7 +192,8 @@ export function getEnv(): BackendEnv { const queue = { location: parse('PUPPETEER_QUEUE_LOCATION'), name: parse('PUPPETEER_QUEUE_NAME'), - puppeteerTaskHanderUrl: parse('PUPPETEER_TASK_HANDLER_URL'), + contentFetchUrl: parse('CONTENT_FETCH_URL'), + contentFetchGCFUrl: parse('CONTENT_FETCH_GCF_URL'), reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'), } const imageProxy = { From 75d3372426be27d3d413ac4d69cade9a395b5513 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Jun 2022 22:22:12 +0800 Subject: [PATCH 2/6] Use gcf for low priority task --- packages/api/src/utils/createTask.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 5b37f604b..a0cab7909 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -19,7 +19,7 @@ const createHttpTaskWithToken = async ({ project, queue = env.queue.name, location = env.queue.location, - taskHandlerUrl = env.queue.puppeteerTaskHanderUrl, + taskHandlerUrl = env.queue.contentFetchUrl, serviceAccountEmail = `${process.env.GOOGLE_CLOUD_PROJECT}@appspot.gserviceaccount.com`, payload, priority = 'high', @@ -43,6 +43,8 @@ const createHttpTaskWithToken = async ({ // Construct the fully qualified queue name. if (priority === 'low') { queue = `${queue}-low` + // use GCF url for low priority tasks + taskHandlerUrl = env.queue.contentFetchGCFUrl } const parent = client.queuePath(project, location, queue) @@ -224,7 +226,7 @@ export const enqueueParseRequest = async ( if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { // Calling the handler function directly. setTimeout(() => { - axios.post(env.queue.puppeteerTaskHanderUrl, payload).catch((error) => { + axios.post(env.queue.contentFetchUrl, payload).catch((error) => { console.error(error) logger.warning( `Error occurred while requesting local puppeteer-parse function\nPlease, ensure your function is set up properly and running using "yarn start" from the "/pkg/gcf/puppeteer-parse" folder` From aae9d7e382bea3fe9c013ac283192a6533ffb442 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 23 Jun 2022 22:31:17 +0800 Subject: [PATCH 3/6] Fix test --- packages/api/test/routers/article_router.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/test/routers/article_router.test.ts b/packages/api/test/routers/article_router.test.ts index 06fba8bb1..cf62e1407 100644 --- a/packages/api/test/routers/article_router.test.ts +++ b/packages/api/test/routers/article_router.test.ts @@ -13,7 +13,7 @@ describe('/article/save API', () => { // We need to mock the pupeeteer-parse // service here because in dev mode the task gets // called immediately. - nock(env.queue.puppeteerTaskHanderUrl).post('/').reply(200) + nock(env.queue.contentFetchUrl).post('/').reply(200) before(async () => { // create test user and login From 67bf6d6f5d4f778e2c37031f02518696c7ddd494 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 24 Jun 2022 11:49:07 +0800 Subject: [PATCH 4/6] Update .env.test --- packages/api/.env.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/.env.test b/packages/api/.env.test index 4d10b8316..16fd0aaf8 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -27,5 +27,5 @@ PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper' SEGMENT_WRITE_KEY='test' REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger PUBSUB_VERIFICATION_TOKEN='123456' -PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/ +CONTENT_FETCH_URL=http://localhost:9090/ ELASTIC_URL=http://localhost:9200 From 92c8047a665dcd5079ae44d521d11125a1933a6f Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 24 Jun 2022 10:52:13 -0700 Subject: [PATCH 5/6] Update environment variable names in docker --- docker-compose-test.yml | 2 +- docker-compose.yml | 2 +- packages/api/src/util.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 7b51fb797..341803d46 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -56,7 +56,7 @@ services: - SSO_JWT_SECRET=some_sso_secret - CLIENT_URL=http://localhost:3000 - GATEWAY_URL=http://localhost:8080/api - - PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/ + - CONTENT_FETCH_URL=http://localhost:9090/ - REMINDER_TASK_HANDLER_URL=/svc/reminders/trigger - BOOKMARKLET_JWT_SECRET=some_bookmarklet_secret - BOOKMARKLET_VERSION=1.0.0 diff --git a/docker-compose.yml b/docker-compose.yml index 7ff53a373..2d11f7a57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: - SSO_JWT_SECRET=some_sso_secret - CLIENT_URL=http://localhost:3000 - GATEWAY_URL=http://localhost:8080/api - - PUPPETEER_TASK_HANDLER_URL=http://content-fetch:8080/?token=some_token + - CONTENT_FETCH_URL=http://content-fetch:8080/?token=some_token - REMINDER_TASK_HANDLER_URL=/svc/reminders/trigger depends_on: migrate: diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index e7aa61519..96ef41a2a 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -101,7 +101,7 @@ const nullableEnvVars = [ 'SAMPLE_METRICS_LOCALLY', 'PUPPETEER_QUEUE_LOCATION', 'PUPPETEER_QUEUE_NAME', - 'PUPPETEER_TASK_HANDLER_URL', + 'CONTENT_FETCH_URL', 'PREVIEW_IMAGE_WRAPPER_ID', 'PREVIEW_GENERATION_SERVICE_URL', 'GCS_UPLOAD_SA_KEY_FILE_PATH', From 4c89323f0bff569dcfb676f552417296c300e647 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 24 Jun 2022 11:30:52 -0700 Subject: [PATCH 6/6] Make the content fetch GCF URL nullable --- packages/api/src/util.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 96ef41a2a..bda3ae1ae 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -102,6 +102,7 @@ const nullableEnvVars = [ 'PUPPETEER_QUEUE_LOCATION', 'PUPPETEER_QUEUE_NAME', 'CONTENT_FETCH_URL', + 'CONTENT_FETCH_GCF_URL', 'PREVIEW_IMAGE_WRAPPER_ID', 'PREVIEW_GENERATION_SERVICE_URL', 'GCS_UPLOAD_SA_KEY_FILE_PATH',