Merge branch 'develop' into gchqgh-50-graph-separation-by-namespace

Conflicts:
	infrastructure/lib/rest-api/lambdas/graph/__init__.py
	infrastructure/lib/workers/lambdas/add_graph.py
	infrastructure/lib/workers/lambdas/kubernetes/__init__.py
	infrastructure/test/e2e/integration.test.ts

infrastructure/lib/rest-api/lambdas/add_graph_request.py

@@ -105,7 +105,8 @@ def handler(event, context):
         "releaseName": release_name,
         "namespaceName": namespace_name,
         "schema": schema,
-        "expectedStatus": initial_status
+        "expectedStatus": initial_status,
+        "endpoints":{}
     }
 
     sqs = boto3.client("sqs")

infrastructure/lib/rest-api/lambdas/graph/__init__.py

@@ -66,7 +66,8 @@ def create_graph(self, release_name, graph_name, status, administrators, namespa
                 "releaseName": release_name,
                 "namespaceName": namespace_name,
                 "currentState": status,
-                "administrators": administrators
+                "administrators": administrators,
+                "endpoints": {}
             },
             ConditionExpression=Attr("releaseName").not_exists() | Attr("namespaceName").not_exists()
         )

infrastructure/lib/workers/lambdas/add_graph.py

@@ -16,6 +16,7 @@
 cluster_name = os.getenv("cluster_name")
 graph_table_name = os.getenv("table_name")
 
+
 def generate_password(length=8):
     """
     Generates a random password of a given length
@@ -45,7 +46,7 @@ def create_values(graph_name, schema, security_groups):
         },
         "pathPrefix": "/*"
     }
-    
+
     return {
         "graph": {
             "config": {
@@ -95,6 +96,15 @@ def create_values(graph_name, schema, security_groups):
         }
     }
 
+def update_endpoints(namespace_name, release_name, graph):
+    kubernetes_client = kubernetes.KubernetesClient(cluster_name)
+    alb_endpoint_output = kubernetes_client.get_alb_endpoints(namespace_name, release_name)
+    if (alb_endpoint_output):
+        for line in alb_endpoint_output.splitlines():
+            resource_details = line.split()
+            resource_name = resource_details[0]
+            resource_address = resource_details[1]
+            graph.update_endpoints(resource_name, resource_address)
 
 def deploy_graph(helm_client, body, security_groups):
     """
@@ -124,15 +134,17 @@ def deploy_graph(helm_client, body, security_groups):
 
     # Create values file
     values = create_values(graph_name, schema, security_groups)
-    
+
     values_file = "/tmp/" + graph_name + ".json"
     with open(values_file, "w") as f:
         f.write(json.dumps(values, indent=2))
-        
+
     # Deploy Graph
     success = helm_client.install_chart(release_name, namespace_name, values=values_file)
 
     if success:
+        # capture endpoints
+        update_endpoints(namespace_name, release_name, graph)
         logger.info("Deployment of %s Succeeded", graph_name)
         graph.update_status("DEPLOYED")
     else:
@@ -152,7 +164,7 @@ def handler(event, context):
     cluster = eks.describe_cluster(name=cluster_name)
     security_groups = cluster["cluster"]["resourcesVpcConfig"]["clusterSecurityGroupId"]
     extra_security_groups = os.getenv("extra_security_groups")
-    
+
     if extra_security_groups is not None:
         security_groups = security_groups + ", " + extra_security_groups
 

infrastructure/lib/workers/lambdas/kubernetes/__init__.py

@@ -11,14 +11,19 @@ class CommandHelper:
     @staticmethod
     def run_command(cmd, release_name):
         succeeded=False
+        output=None
         try:
-            subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, cwd="/tmp")
+            cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, text=True, cwd="/tmp")
             succeeded=True
+            output=cp.stdout
         except subprocess.CalledProcessError as err:
             logger.error("Error during excution of command: %s against release name: %s", cmd, release_name)
             logger.error(err.output)
 
-        return succeeded
+        return {
+            "success": succeeded,
+            "output": output
+        }
 
 
 class KubeConfigurator:
@@ -59,7 +64,7 @@ def __run(self, instruction, release_name, namespace_name, values=None, chart=No
             cmd.extend(["--values", values])
         cmd.extend(["--kubeconfig", self.kubeconfig])
 
-        return CommandHelper.run_command(cmd, release_name)
+        return CommandHelper.run_command(cmd, release_name)["success"]
 
     def install_chart(self, release_name, namespace_name, values=None, chart="gaffer", repo="https://gchq.github.io/gaffer-docker"):
         """
@@ -101,8 +106,15 @@ def __delete_volumes(self, release_name, namespace_name, selectors):
 
     def create_namespace(self, namespace_name):
         cmd = [ self.__KUBECTL_CMD, "create", "namespace", namespace_name, "--kubeconfig", self.kubeconfig ]
-        return CommandHelper.run_command(cmd, namespace_name)
+        return CommandHelper.run_command(cmd, namespace_name)["success"]
 
     def delete_namespace(self, namespace_name):
         cmd = [ self.__KUBECTL_CMD, "delete", "namespace", namespace_name, "--kubeconfig", self.kubeconfig ]
-        return CommandHelper.run_command(cmd, namespace_name)
+        return CommandHelper.run_command(cmd, namespace_name)["success"]
+
+    def get_alb_endpoints(self, namespace_name, release_name):
+        selector = "app.kubernetes.io/instance=" + release_name
+        output_format = "custom-columns=NAME:.metadata.name,ADDRESS:.status.loadBalancer.ingress[0].hostname"
+        cmd = [ self.__KUBECTL_CMD, "get", "ing", "--kubeconfig", self.kubeconfig, "--namespace", namespace_name, "--selector", selector, "--output", output_format, "--no-headers"]
+        result = CommandHelper.run_command(cmd, release_name)
+        return result["output"] if result["success"] else None

infrastructure/lib/workers/lambdas/table/__init__.py

@@ -31,6 +31,24 @@ def check_status(self, expected_status):
 
         return status == expected_status
 
+
+    def update_endpoints(self, resource_name, resource_address):
+        """
+        Update graph with endpoints that get created by the application load balancer
+        """
+        self.table.update_item(
+            Key=self.index_key,
+            UpdateExpression = "SET endpoints.#resourceName = :resourceAddress",
+            ExpressionAttributeNames = { 
+                "#resourceName" : resource_name 
+            },
+            ExpressionAttributeValues = {
+                ":resourceAddress": resource_address
+            },
+            ConditionExpression = "attribute_not_exists(endpoints.#resourceName)"
+        )
+
+
     def update_status(self, status):
         """
         Updates the status of the item