Merge pull request #859 from omnivore-app/use-gcf-for-low-priority-queue

use gcf for low priority queue

docker-compose-test.yml

@@ -56,7 +56,7 @@ services:
       - SSO_JWT_SECRET=some_sso_secret
       - CLIENT_URL=http://localhost:3000
       - GATEWAY_URL=http://localhost:8080/api
-      - PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/
+      - CONTENT_FETCH_URL=http://localhost:9090/
       - REMINDER_TASK_HANDLER_URL=/svc/reminders/trigger
       - BOOKMARKLET_JWT_SECRET=some_bookmarklet_secret
       - BOOKMARKLET_VERSION=1.0.0

docker-compose.yml

@@ -93,7 +93,7 @@ services:
       - SSO_JWT_SECRET=some_sso_secret
       - CLIENT_URL=http://localhost:3000
       - GATEWAY_URL=http://localhost:8080/api
-      - PUPPETEER_TASK_HANDLER_URL=http://content-fetch:8080/?token=some_token
+      - CONTENT_FETCH_URL=http://content-fetch:8080/?token=some_token
       - REMINDER_TASK_HANDLER_URL=/svc/reminders/trigger
     depends_on:
       migrate:

packages/api/.env.test

@@ -27,5 +27,5 @@ PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
 SEGMENT_WRITE_KEY='test'
 REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger
 PUBSUB_VERIFICATION_TOKEN='123456'
-PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/
+CONTENT_FETCH_URL=http://localhost:9090/
 ELASTIC_URL=http://localhost:9200

packages/api/src/util.ts

@@ -59,7 +59,8 @@ interface BackendEnv {
   queue: {
     location: string
     name: string
-    puppeteerTaskHanderUrl: string
+    contentFetchUrl: string
+    contentFetchGCFUrl: string
     reminderTaskHanderUrl: string
   }
   fileUpload: {
@@ -100,7 +101,8 @@ const nullableEnvVars = [
   'SAMPLE_METRICS_LOCALLY',
   'PUPPETEER_QUEUE_LOCATION',
   'PUPPETEER_QUEUE_NAME',
-  'PUPPETEER_TASK_HANDLER_URL',
+  'CONTENT_FETCH_URL',
+  'CONTENT_FETCH_GCF_URL',
   'PREVIEW_IMAGE_WRAPPER_ID',
   'PREVIEW_GENERATION_SERVICE_URL',
   'GCS_UPLOAD_SA_KEY_FILE_PATH',
@@ -191,7 +193,8 @@ export function getEnv(): BackendEnv {
   const queue = {
     location: parse('PUPPETEER_QUEUE_LOCATION'),
     name: parse('PUPPETEER_QUEUE_NAME'),
-    puppeteerTaskHanderUrl: parse('PUPPETEER_TASK_HANDLER_URL'),
+    contentFetchUrl: parse('CONTENT_FETCH_URL'),
+    contentFetchGCFUrl: parse('CONTENT_FETCH_GCF_URL'),
     reminderTaskHanderUrl: parse('REMINDER_TASK_HANDLER_URL'),
   }
   const imageProxy = {

packages/api/src/utils/createTask.ts

@@ -19,7 +19,7 @@ const createHttpTaskWithToken = async ({
   project,
   queue = env.queue.name,
   location = env.queue.location,
-  taskHandlerUrl = env.queue.puppeteerTaskHanderUrl,
+  taskHandlerUrl = env.queue.contentFetchUrl,
   serviceAccountEmail = `${process.env.GOOGLE_CLOUD_PROJECT}@appspot.gserviceaccount.com`,
   payload,
   priority = 'high',
@@ -43,6 +43,8 @@ const createHttpTaskWithToken = async ({
   // Construct the fully qualified queue name.
   if (priority === 'low') {
     queue = `${queue}-low`
+    // use GCF url for low priority tasks
+    taskHandlerUrl = env.queue.contentFetchGCFUrl
   }
 
   const parent = client.queuePath(project, location, queue)
@@ -224,7 +226,7 @@ export const enqueueParseRequest = async (
   if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) {
     // Calling the handler function directly.
     setTimeout(() => {
-      axios.post(env.queue.puppeteerTaskHanderUrl, payload).catch((error) => {
+      axios.post(env.queue.contentFetchUrl, payload).catch((error) => {
         console.error(error)
         logger.warning(
           `Error occurred while requesting local puppeteer-parse function\nPlease, ensure your function is set up properly and running using "yarn start" from the "/pkg/gcf/puppeteer-parse" folder`

packages/api/test/routers/article_router.test.ts

@@ -13,7 +13,7 @@ describe('/article/save API', () => {
   // We need to mock the pupeeteer-parse
   // service here because in dev mode the task gets
   // called immediately.
-  nock(env.queue.puppeteerTaskHanderUrl).post('/').reply(200)
+  nock(env.queue.contentFetchUrl).post('/').reply(200)
 
   before(async () => {
     // create test user and login