split into two SQS queues, one for glue and the other for the traditi…

…onal loader.

iam.tf

@@ -27,11 +27,12 @@ data "aws_iam_policy_document" "receiver_execution_policy" {
     resources = ["arn:aws:logs:*:*:*"]
   }
 
-  # allow the lambda to enqueue work
+  # allow the receiver lambda to enqueue work
   statement {
     effect = "Allow"
     actions = ["sqs:SendMessage"]
-    resources = ["arn:aws:sqs:${var.aws_region}:*:${aws_sqs_queue.receiver_queue.name}"]
+    resources = ["arn:aws:sqs:${var.aws_region}:*:${aws_sqs_queue.receiver_queue.name}",
+                 "arn:aws:sqs:${var.aws_region}:*:${aws_sqs_queue.receiver_queue_glue.name}"]
   }
 }
 
@@ -221,11 +222,11 @@ data "aws_iam_policy_document" "run_glue_crawler_execution_policy" {
     resources = [ aws_glue_crawler.signatures_crawler.arn ]
   }
 
-  # allow lambda to be wired up to the queue. These are the minimum permissions for the SQS Lambda Executor.
+  # allow glue lambda to be wired up to the queue. These are the minimum permissions for the SQS Lambda Executor.
   statement {
     effect = "Allow"
     actions = ["sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"]
-    resources = ["arn:aws:sqs:${var.aws_region}:*:${aws_sqs_queue.receiver_queue.name}"]
+    resources = ["arn:aws:sqs:${var.aws_region}:*:${aws_sqs_queue.receiver_queue_glue.name}"]
   }
 }
 

lambdas/receiver.js

@@ -21,16 +21,24 @@ function enqueueTask(receivedData, kind) {
 
   messageBody['kind'] =  kind;
 
-  const params = {
-    MessageBody: JSON.stringify(messageBody),
+  const jsonMessageBody = JSON.stringify(messageBody)
+
+  const loaderQueueParams = {
+    MessageBody: jsonMessageBody,
     QueueUrl: process.env.SQS_QUEUE_URL
   };
 
+  const glueQueueParams = {
+    MessageBody: jsonMessageBody,
+    QueueUrl: process.env.GLUE_SQS_QUEUE_URL
+  };
+
   console.log('Enqueueing ' + JSON.stringify(messageBody) + ' on ' +  process.env.SQS_QUEUE_URL);
 
-  let resp = sqs.sendMessage(params).promise();
+  let loaderResp = sqs.sendMessage(loaderQueueParams).promise();
+  let glueResp = sqs.sendMessage(glueQueueParams).promise();
 
-  resp.then(
+  Promise.all([loaderResp, glueResp]).then(
     function(data) {
       console.log("Success " + data.MessageId);
     },

receiver.tf

@@ -16,6 +16,7 @@ resource "aws_lambda_function" "receiver_lambda" {
   environment {
     variables = {
       SQS_QUEUE_URL = aws_sqs_queue.receiver_queue.id
+      GLUE_SQS_QUEUE_URL = aws_sqs_queue.receiver_queue_glue.id
     }
   }
 }
@@ -97,6 +98,11 @@ resource "aws_sqs_queue" "receiver_queue" {
   visibility_timeout_seconds = 900
 }
 
+resource "aws_sqs_queue" "receiver_queue_glue" {
+  name = "controlshift-received-webhooks-glue"
+  visibility_timeout_seconds = 900
+}
+
 # there is no formal way to associate this resource, beyond ensuring the name matches.
 resource "aws_cloudwatch_log_group" "api_gateway_log_retention" {
   name              = "API-Gateway-Execution-Logs_${aws_api_gateway_rest_api.receiver.id}/${aws_api_gateway_deployment.deployment.stage_name}"

run_glue_crawler.tf

@@ -22,7 +22,7 @@ resource "aws_lambda_function" "glue_crawler_lambda" {
 }
 
 resource "aws_lambda_event_source_mapping" "run_crawler_on_new_data_export_task" {
-  event_source_arn = aws_sqs_queue.receiver_queue.arn
+  event_source_arn = aws_sqs_queue.receiver_queue_glue.arn
   function_name    = aws_lambda_function.glue_crawler_lambda.arn
   batch_size = 1
 }