Parallelise the request to the vcenters

.gitignore

@@ -0,0 +1,3 @@
+.*
+*.pyc
+*.egg-info

vcenter_operator/configurator.py

@@ -1,12 +1,12 @@
 import atexit
+from concurrent.futures import ThreadPoolExecutor as Executor
 import http.client
 import json
 import logging
 import re
 import ssl
 import time
 
-from collections import deque
 from contextlib import contextmanager
 from keystoneauth1.session import Session
 from keystoneauth1.identity.v3 import Password
@@ -17,12 +17,9 @@
 from kubernetes import client
 from pyVim.connect import SmartConnect, Disconnect
 from pyVmomi import vim
-from jinja2.exceptions import TemplateError
-from yaml.error import YAMLError
 
 from .masterpassword import MasterPassword
 from .phelm import DeploymentState
-from .templates import env, TemplateLoadingFailed
 import vcenter_operator.vcenter_util as vcu
 
 LOG = logging.getLogger(__name__)
@@ -68,7 +65,7 @@ def __init__(self, domain, global_options={}):
         self.domain = domain
         self.os_session = None
         self.vcenters = dict()
-        self.states = deque()
+        self.states = dict()
         self.poll_config()
         self.global_options['cells'] = {}
         self.global_options['domain'] = domain
@@ -174,7 +171,8 @@ def _reconnect_vcenter_if_necessary(self, host):
         if needs_reconnect:
             self._connect_vcenter(host)
 
-    def _poll(self, host):
+    def _poll_host_with_exceptions(self, host):
+        self._reconnect_vcenter_if_necessary(host)
         vcenter_options = self.vcenters[host]
         values = {'clusters': {}, 'datacenters': {}}
         service_instance = vcenter_options['service_instance']
@@ -252,7 +250,6 @@ def _poll(self, host):
                     continue
 
                 values['clusters'][cluster_name] = cluster_options
-                self._add_code('vcenter_cluster', cluster_options)
 
             for availability_zone in availability_zones:
                 cluster_options = self.global_options.copy()
@@ -261,22 +258,8 @@ def _poll(self, host):
                 cluster_options.update(availability_zone=availability_zone)
                 values['datacenters'][availability_zone] = cluster_options
 
-            if cluster_options:
-                self._add_code('vcenter_datacenter', cluster_options)
         return values
 
-    def _add_code(self, scope, options):
-        template_names = env.list_templates(
-            filter_func=lambda x: (x.startswith(scope)
-                                   and x.endswith('.yaml.j2')))
-        for template_name in template_names:
-            try:
-                template = env.get_template(template_name)
-                result = template.render(options)
-                self.states[-1].add(result)
-            except (TemplateError, YAMLError):
-                LOG.exception("Failed to render %s", template_name)
-
     @property
     def _client(self):
         return client
@@ -337,42 +320,60 @@ def poll_nova(self):
         except (HttpError, ConnectionError) as e:
             LOG.error("Failed to get cells: {}".format(e))
 
-    def poll(self):
-        self.poll_config()
-        self.poll_nova()
-        self.states.append(DeploymentState(
+    def _values_from_host(self, host):
+        try:
+            return self._poll_host_with_exceptions(host)
+        except VcConnectionFailed:
+            LOG.error(
+                "Reconnecting to %s failed. Ignoring VC for this run.", host
+            )
+        except VcConnectSkipped:
+            LOG.info("Ignoring disconnected %s for this run.", host)
+        except http.client.HTTPException as e:
+            LOG.warning("%s: %r", host, e)
+
+    def _state_for_values(self, values):
+        if not values:
+            return
+
+        state = DeploymentState(
             namespace=self.global_options['namespace'],
-            dry_run=(self.global_options.get('dry_run', 'False') == 'True')))
+            dry_run=(self.global_options.get('dry_run', 'False')
+                     == 'True'))
 
-        hosts = {}
-        for host in self.vcenters:
-            try:
-                self._reconnect_vcenter_if_necessary(host)
-            except VcConnectionFailed:
-                LOG.error('Reconnecting to %s failed. Ignoring VC for this '
-                          'run.', host)
-                continue
-            except VcConnectSkipped:
-                LOG.info('Ignoring disconnected %s for this run.', host)
-                continue
+        for options in values['clusters'].values():
+            state.render('vcenter_cluster', options)
 
-            try:
-                hosts[host] = self._poll(host)
-            except http.client.HTTPException as e:
-                LOG.warning("%s: %r", host, e)
-                continue
-            except TemplateLoadingFailed as e:
-                LOG.warning("Loading of templates failed: %r", e)
-                return
-
-        all_values = {'hosts': hosts}
-        all_values.update(self.global_options)
-        all_values.pop('service_instance', None)
-        self._add_code('vcenter_global', all_values)
-
-        if len(self.states) > 1:
-            last = self.states.popleft()
-            delta = last.delta(self.states[-1])
-            delta.apply()
-        else:
-            self.states[-1].apply()
+        for options in values['datacenters'].values():
+            state.render('vcenter_datacenter', options)
+
+        return state
+
+    def _apply_state_for_host(self, host, state):
+        if not state:
+            return
+
+        last = self.states.get(host)
+
+        try:
+            if last:
+                delta = last.delta(state)
+                delta.apply()
+            else:
+                state.apply()
+
+            self.states[host] = state
+        except http.client.HTTPException as e:
+            LOG.warning("%s: %r", host, e)
+
+    def poll(self):
+        self.poll_config()
+        self.poll_nova()
+        with Executor() as executor:
+            results = executor.map(
+                lambda host: (host, self._values_from_host(host)),
+                self.vcenters)
+
+            for host, values in results:
+                state = self._state_for_values(values)
+                self._apply_state_for_host(host, state)

vcenter_operator/phelm.py

@@ -9,8 +9,12 @@
 import attr
 import jsonpatch
 import yaml
+from jinja2.exceptions import TemplateError
 from jsonpointer import resolve_pointer
 from kubernetes import client
+from yaml.error import YAMLError
+
+from .templates import TemplateLoadingFailed, env
 
 LOG = logging.getLogger(__name__)
 
@@ -48,6 +52,21 @@ class DeploymentState(object):
     items = attr.ib(default=attr.Factory(OrderedDict))
     actions = attr.ib(default=attr.Factory(OrderedDict))
 
+    def render(self, scope, options):
+        try:
+            template_names = env.list_templates(
+                filter_func=lambda x: (x.startswith(scope)
+                                       and x.endswith('.yaml.j2')))
+            for template_name in template_names:
+                try:
+                    template = env.get_template(template_name)
+                    result = template.render(options)
+                    self.add(result)
+                except (TemplateError, YAMLError):
+                    LOG.exception("Failed to render %s", template_name)
+        except TemplateLoadingFailed:
+            LOG.exception("Failed to load templates")
+
     def add(self, result):
         stream = io.StringIO(result)
         for item in yaml.safe_load_all(stream):