Merge pull request #39 from deepmind/update-tests

Update tests

kapitan/resources.py

@@ -125,7 +125,11 @@ def search_imports(cwd, import_str, search_path):
     logger.debug("cwd:%s import_str:%s basename:%s -> norm:%s",
                  cwd, import_str, basename, normalised_path)
 
-    return normalised_path, open(normalised_path).read()
+    normalised_path_content = ""
+    with open(normalised_path) as f:
+        normalised_path_content = f.read()
+
+    return normalised_path, normalised_path_content
 
 
 def inventory(search_path, target, inventory_path="inventory/"):

kapitan/utils.py

@@ -212,3 +212,30 @@ def deep_get(x, keys):
                 fd.close()
     for i in output:
         print('{0!s:{l}} {1!s}'.format(*i, l=maxlenght + 2))
+
+
+def get_directory_hash(directory):
+    '''
+    Compute a sha256 hash for the file contents of a directory
+    '''
+    if not os.path.exists(directory):
+        logger.error("utils.get_directory_hash failed, %s dir doesn't exist", directory)
+        return -1
+
+    try:
+        hash = sha256()
+        for root, _, files in os.walk(directory):
+            for names in files:
+                file_path = os.path.join(root, names)
+                try:
+                    with open(file_path, 'r') as f:
+                        hash.update(sha256(f.read().encode("UTF-8")).hexdigest().encode("UTF-8"))
+                except Exception as e:
+                    logger.error("utils.get_directory_hash failed to open %s: %s", file_path, str(e))
+                    raise
+
+    except Exception as e:
+        logger.error("utils.get_directory_hash failed: %s", str(e))
+        raise
+
+    return hash.hexdigest()

tests/test_compile.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Kapitan Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"compile tests"
+
+import unittest
+import os
+import sys
+import shutil
+from kapitan.cli import main
+from kapitan.utils import get_directory_hash
+
+class CompileTest(unittest.TestCase):
+    def setUp(self):
+        os.chdir(os.getcwd() + '/examples/kubernetes/')
+
+    def test_compile(self):
+        sys.argv[1] = "compile"
+        main()
+        compiled_dir_hash = get_directory_hash(os.getcwd() + '/compiled')
+        test_compiled_dir_hash = get_directory_hash(os.getcwd() + '/../../tests/test_kubernetes_compiled')
+        self.assertEqual(compiled_dir_hash, test_compiled_dir_hash)
+
+    def tearDown(self):
+        compile_path = os.getcwd() + '/compiled'
+        if os.path.exists(compile_path):
+            shutil.rmtree(compile_path)
+        os.chdir(os.getcwd() + '/../../')

tests/test_jinja2.py

@@ -29,6 +29,14 @@ def test_sha256(self):
             digest = 'e863c1ac42619a2b429a08775a6acd89ff4c2c6b8dae12e3461a5fa63b2f92f5'
             self.assertEqual(render_jinja2_file(f.name, context), digest)
 
+    def test_yaml(self):
+        with tempfile.NamedTemporaryFile() as f:
+            f.write("{{text|yaml}}".encode("UTF-8"))
+            f.seek(0)
+            context = {"text":["this", "that"]}
+            yaml = '- this\n- that\n'
+            self.assertEqual(render_jinja2_file(f.name, context), yaml)
+
 class Jinja2ContextVars(unittest.TestCase):
     def test_inventory_context(self):
         with tempfile.NamedTemporaryFile() as f:

tests/test_kubernetes_compiled/minikube-es-2/README.md

@@ -0,0 +1,61 @@
+# Elasticsearch Minikube
+
+This is a specific version of Elasticsearch to run on a minikube instalation.
+
+## Prerequisites
+
+Elasticsearch is a resource hungry application, for this setup we require
+that minikube is running with the above options:
+
+```
+$ minikube start --insecure-registry https://quay.io --memory=4096 --cpus=2
+```
+
+_If_ you have created the minikube VM previously, you will most likely need to 
+delete the vm and recreate it with more memory/cpu. (i.e. 
+```$ minikube delete```)
+
+## Setting up
+
+Assuming you're already running Minikube, setup for this target:
+
+```
+$ scripts/setup.sh
+```
+
+This will create a context in your minikube cluster called minikube-es-2.
+
+
+Apply the compiled manifests:
+
+```
+$ scripts/kubectl.sh apply -f manifests/
+```
+
+If the commands above did not error, you should be good to go. 
+
+Let's confirm everything is up:
+
+```
+$ scripts/kubectl.sh get pods -w
+```
+
+## Connecting to Elasticsearch
+
+List the elasticsearch service endpoints running in the cluster:
+
+```
+$ minikube service -n minikube-es  elasticsearch --url
+```
+
+and curl the health endpoint, i.e.: 
+```$ curl http://192.168.99.100:32130/_cluster/health?pretty```
+
+
+## Deleting Elasticsearch
+
+Deleting is easy (warning, this will remove _everything_):
+
+```
+$ scripts/kubectl.sh delete -f manifests/
+```

tests/test_kubernetes_compiled/minikube-es-2/manifests/es-client.yml

@@ -0,0 +1,67 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  labels:
+    name: cluster-client
+    role: client
+  name: cluster-client
+  namespace: minikube-es-2
+spec:
+  replicas: 2
+  serviceName: cluster-client
+  template:
+    metadata:
+      labels:
+        name: cluster-client
+        role: client
+    spec:
+      containers:
+        - env:
+            - name: CLUSTER_NAME
+              value: cluster
+            - name: ES_JAVA_OPTS
+              value: -Xms512m -Xmx512m
+            - name: HTTP_ENABLE
+              value: 'true'
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: NODE_DATA
+              value: 'false'
+            - name: NODE_INGEST
+              value: 'false'
+            - name: NODE_MASTER
+              value: 'false'
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NUMBER_OF_MASTERS
+              value: '1'
+          image: quay.io/pires/docker-elasticsearch-kubernetes:5.5.0
+          imagePullPolicy: Always
+          name: client
+          ports:
+            - containerPort: 9200
+              name: client
+              protocol: TCP
+            - containerPort: 9300
+              name: transport
+              protocol: TCP
+          securityContext:
+            capabilities:
+              add:
+                - IPC_LOCK
+                - SYS_RESOURCE
+            privileged: false
+      initContainers:
+        - command:
+            - sysctl
+            - -w
+            - vm.max_map_count=262144
+          image: busybox
+          imagePullPolicy: IfNotPresent
+          name: sysctl
+          securityContext:
+            privileged: true

tests/test_kubernetes_compiled/minikube-es-2/manifests/es-data.yml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  labels:
+    name: cluster-data
+    role: data
+  name: cluster-data
+  namespace: minikube-es-2
+spec:
+  replicas: 2
+  serviceName: cluster-data
+  template:
+    metadata:
+      labels:
+        name: cluster-data
+        role: data
+    spec:
+      containers:
+        - env:
+            - name: CLUSTER_NAME
+              value: cluster
+            - name: ES_JAVA_OPTS
+              value: -Xms512m -Xmx512m
+            - name: HTTP_ENABLE
+              value: 'false'
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: NODE_DATA
+              value: 'true'
+            - name: NODE_INGEST
+              value: 'false'
+            - name: NODE_MASTER
+              value: 'false'
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NUMBER_OF_MASTERS
+              value: '1'
+          image: quay.io/pires/docker-elasticsearch-kubernetes:5.5.0
+          imagePullPolicy: Always
+          name: data
+          ports:
+            - containerPort: 9300
+              name: transport
+              protocol: TCP
+          securityContext:
+            capabilities:
+              add:
+                - IPC_LOCK
+                - SYS_RESOURCE
+            privileged: false
+      initContainers:
+        - command:
+            - sysctl
+            - -w
+            - vm.max_map_count=262144
+          image: busybox
+          imagePullPolicy: IfNotPresent
+          name: sysctl
+          securityContext:
+            privileged: true

tests/test_kubernetes_compiled/minikube-es-2/manifests/es-discovery-svc.yml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: elasticsearch-discovery
+  name: elasticsearch-discovery
+  namespace: minikube-es-2
+spec:
+  ports:
+    - name: transport
+      port: 9300
+      targetPort: transport
+  selector:
+    name: cluster-master
+    role: master
+  type: ClusterIP

tests/test_kubernetes_compiled/minikube-es-2/manifests/es-elasticsearch-svc.yml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: elasticsearch
+  name: elasticsearch
+  namespace: minikube-es-2
+spec:
+  ports:
+    - name: client
+      port: 9200
+      targetPort: client
+    - name: transport
+      port: 9300
+      targetPort: transport
+  selector:
+    name: cluster-client
+    role: client
+  type: NodePort

tests/test_kubernetes_compiled/minikube-es-2/manifests/es-master.yml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  labels:
+    name: cluster-master
+    role: master
+  name: cluster-master
+  namespace: minikube-es-2
+spec:
+  replicas: 2
+  serviceName: cluster-master
+  template:
+    metadata:
+      labels:
+        name: cluster-master
+        role: master
+    spec:
+      containers:
+        - env:
+            - name: CLUSTER_NAME
+              value: cluster
+            - name: ES_JAVA_OPTS
+              value: -Xms512m -Xmx512m
+            - name: HTTP_ENABLE
+              value: 'false'
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: NODE_DATA
+              value: 'false'
+            - name: NODE_INGEST
+              value: 'false'
+            - name: NODE_MASTER
+              value: 'true'
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NUMBER_OF_MASTERS
+              value: '1'
+          image: quay.io/pires/docker-elasticsearch-kubernetes:5.5.0
+          imagePullPolicy: Always
+          name: master
+          ports:
+            - containerPort: 9300
+              name: transport
+              protocol: TCP
+          securityContext:
+            capabilities:
+              add:
+                - IPC_LOCK
+                - SYS_RESOURCE
+            privileged: false
+      initContainers:
+        - command:
+            - sysctl
+            - -w
+            - vm.max_map_count=262144
+          image: busybox
+          imagePullPolicy: IfNotPresent
+          name: sysctl
+          securityContext:
+            privileged: true