Add service user password rotation feature

This patch adds the service user rotation feature, which provides two
actions:

 - list-service-usernames
 - rotate-service-user-password

The first lists the possible usernames that can be rotated.  The
second action rotates the service, and is tested via the func-test-pr.

Change-Id: Ia94ab3d54cd8a59e9ba5005b88d3ec1ff87019b1
func-test-pr: openstack-charmers/zaza-openstack-tests#1029

actions.yaml

@@ -79,3 +79,20 @@ run-deferred-hooks:
 show-deferred-events:
     descrpition: |
         Show the outstanding restarts
+rotate-service-user-password:
+  description: |
+    Rotate the specified rabbitmq-server user's password. The current password
+    is replaced with a randomly generated password. The password is changed on
+    the relation to the user's units. This may result in a control plane outage
+    for the duration of the password changing process.
+  params:
+    service-user:
+      type: string
+      description: |
+        The username of the rabbitmq-server service as specified in
+        list-service-usernames.
+list-service-usernames:
+  description: |
+    List the usernames of the passwords that have been provided on the
+    amqp relations. The service username passed to
+    'rotate-service-user-password' needs to be on this list.

actions/actions.py

@@ -19,6 +19,7 @@
 from collections import OrderedDict
 from subprocess import check_output, CalledProcessError, PIPE
 import sys
+import traceback
 
 
 _path = os.path.dirname(os.path.realpath(__file__))
@@ -71,6 +72,9 @@ def _add_path(path):
     list_vhosts,
     vhost_queue_info,
     rabbitmq_version_newer_or_equal,
+    get_usernames_for_passwords,
+    NotLeaderError,
+    InvalidServiceUserError,
 )
 
 
@@ -299,6 +303,39 @@ def show_deferred_events(args):
     os_utils.show_deferred_events_action_helper()
 
 
+def list_service_usernames(args):
+    """List the service usernames known in this model that can be rotated."""
+    usernames = get_usernames_for_passwords()
+    action_set({'usernames': usernames or []})
+
+
+def rotate_service_user_password(args):
+    """Rotate the service user's password.
+
+    The parameter must be passed in the service-user parameter.
+    """
+    service_user = action_get("service-user")
+    if service_user is None:
+        action_fail(
+            "The 'service-user' parameter was not passed and is required.")
+        return
+    try:
+        rabbitmq_server_relations.rotate_service_user_password(service_user)
+    except NotLeaderError:
+        action_fail(
+            "This unit either isn't the leader or is not ready to do "
+            "leader actions.  The rotate-service-user-password action. "
+            "can't be run at the moment. Please verify that the unit is the "
+            "leader and that the cluster is ready.")
+    except InvalidServiceUserError:
+        action_fail(
+            "Service username {} is not valid for password rotation.  Please "
+            "check the action 'list-service-users' for the correct username."
+            .format(service_user))
+    except Exception:
+        raise
+
+
 # A dictionary of all the defined actions to callables (which take
 # parsed arguments).
 ACTIONS = {
@@ -313,6 +350,8 @@ def show_deferred_events(args):
     "restart-services": restart,
     "run-deferred-hooks": run_deferred_hooks,
     "show-deferred-events": show_deferred_events,
+    "list-service-usernames": list_service_usernames,
+    "rotate-service-user-password": rotate_service_user_password,
 }
 
 
@@ -332,7 +371,10 @@ def main(args):
         try:
             action(args)
         except Exception as e:
+            log("Action {} failed: {}\nTrackback:\n{}"
+                .format(action_name, str(e), traceback.format_exc()), ERROR)
             action_fail("Action {} failed: {}".format(action_name, str(e)))
+
     _run_atexit()
 
 

actions/list-service-usernames

@@ -0,0 +1 @@
+actions.py

actions/rotate-service-user-password

@@ -0,0 +1 @@
+actions.py

hooks/rabbit_utils.py

@@ -94,7 +94,7 @@
 
 from charmhelpers.contrib.peerstorage import (
     peer_store,
-    peer_retrieve
+    peer_retrieve,
 )
 
 from charmhelpers.fetch import (
@@ -136,6 +136,7 @@
 
 _named_passwd = '/var/lib/charm/{}/{}.passwd'
 _local_named_passwd = '/var/lib/charm/{}/{}.local_passwd'
+_service_password_glob = '/var/lib/charm/{}/*.passwd'
 
 
 # hook_contexts are used as a convenient mechanism to render templates
@@ -179,6 +180,16 @@ def CONFIG_FILES():
     return _cfiles
 
 
+class NotLeaderError(Exception):
+    """Exception raised if not the leader."""
+    pass
+
+
+class InvalidServiceUserError(Exception):
+    """Exception raised if an invalid Service User is detected."""
+    pass
+
+
 class ConfigRenderer(object):
     """
     This class is a generic configuration renderer for
@@ -590,6 +601,24 @@ def create_user(user, password, tags=[]):
     apply_tags(user, tags)
 
 
+def change_user_password(user, new_password):
+    """Change the password of the rabbitmq user.
+
+    :param user: the user to change; must exist in the rabbitmq instance.
+    :type user: str
+    :param new_password: the password to change to.
+    :type new_password: str
+    :raises KeyError: if the user doesn't exist.
+    """
+    exists = user_exists(user)
+    if not exists:
+        msg = "change_user_password: user '{}' doesn't exist.".format(user)
+        log(msg, ERROR)
+        raise KeyError(msg)
+    rabbitmqctl('change_password', user, new_password)
+    log("Changed password on rabbitmq for user: {}".format(user), INFO)
+
+
 def grant_permissions(user, vhost):
     """Grant all permissions on a vhost to a user.
 
@@ -1178,7 +1207,7 @@ def get_rabbit_password_on_disk(username, password=None, local=False):
 
 def migrate_passwords_to_peer_relation():
     '''Migrate any passwords storage on disk to cluster peer relation'''
-    for f in glob.glob('/var/lib/charm/{}/*.passwd'.format(service_name())):
+    for f in glob.glob(_service_password_glob.format(service_name())):
         _key = os.path.basename(f)
         with open(f, 'r') as passwd:
             _value = passwd.read().strip()
@@ -1190,6 +1219,48 @@ def migrate_passwords_to_peer_relation():
             pass
 
 
+def get_usernames_for_passwords_on_disk():
+    """Return a list of usernames that have passwords on the disk.
+
+    Note this is only for non local passwords (i.e. that end in .passwd)
+
+    :returns: the list of usernames with passwords on the disk.
+    :rtype: List[str]
+    """
+    return [
+        os.path.splitext(os.path.basename(f))[0]
+        for f in glob.glob(_service_password_glob.format(service_name()))]
+
+
+def get_usernames_for_passwords():
+    """Return a list of usernames that have passwords.
+
+    This checks BOTH the peer relationship (leader-storage, or the fallback to
+    the 'cluster' relation) and on disk.  If the peer storage has usernames,
+    ignore the ones on disk (as they have already been migrated), otherwise
+    return the ones on disk.
+
+    The keys that have passwords in peer storage end with .passwd.
+
+    :returns: the list of usernames that have had passwords set.
+    :rtype: List[str]
+    """
+    # first get from leader settings/peer relation, if available
+    peer_keys = None
+    try:
+        peer_keys = peer_retrieve(None)
+    except ValueError:
+        pass
+    if peer_keys is None:
+        peer_keys = {}
+    usernames = set(u[:-7] for u in peer_keys.keys() if u.endswith(".passwd"))
+    # if usernames were found in peer storage, return them.
+    if usernames:
+        return sorted(usernames)
+    # otherwise, return the ones on disk, if any
+    return sorted(get_usernames_for_passwords_on_disk())
+
+
 def get_rabbit_password(username, password=None, local=False):
     ''' Retrieve, generate or store a rabbit password for
     the provided username using peer relation cluster'''

hooks/rabbitmq_server_relations.py

@@ -100,6 +100,7 @@ def _add_path(path):
 )
 from charmhelpers.core.host import (
     cmp_pkgrevno,
+    pwgen,
     service_stop,
     service_restart,
 )
@@ -315,6 +316,77 @@ def configure_amqp(username, vhost, relation_id, admin=False,
     return password
 
 
+def rotate_service_user_password(service_username):
+    """Rotate the service username and update the relation.
+
+    This only works on the leader unit due to how peer storage is overlayed on
+    leader storage.
+
+    :param service_username: the username to rotate the password for.
+    :type service_username: str
+    :raises NotLeaderError: if the unit is not the leader.
+    :raises InvalidServiceUserError: if the service_username doesn't exist.
+    """
+    if not rabbit.leader_node_is_ready():
+        raise rabbit.NotLeaderError(
+            "This unit can't perform leadership actions and so the password "
+            "cannot be rotated.")
+    if service_username not in rabbit.get_usernames_for_passwords():
+        raise rabbit.InvalidServiceUserError(
+            "Username {} is not valid for password rotation."
+            .format(service_username))
+
+    # pick a new password.
+    new_passwd = pwgen(length=64)
+
+    # Update the password in rabbitmq
+    rabbit.change_user_password(service_username, new_passwd)
+
+    # Update the setting either locally on disk, leader settings or peer
+    # storage.
+    try:
+        peer_store("{}.passwd".format(service_username), new_passwd)
+    except ValueError:
+        # if there was no cluster, just push to leader settings.
+        leader_set({"{}.passwd".format(service_username): new_passwd})
+
+    # Note that the password is not stored in the local cache, so the kv()
+    # won't need updating for this.  The related unit with the username does
+    # need finding, though.  Note that the username may have a '_' in it if
+    # multiple prefixes are used, but that was specified as service_username
+    pattern = re.compile(r"(\S+)_username")
+    for rid in relation_ids('amqp'):
+        key = None
+        for unit in related_units(rid):
+            current = relation_get(rid=rid, unit=unit) or {}
+            # the username is either as 'username' or '{previx}_username'
+            if 'username' in current:
+                key = 'password'
+                break
+            for key in current.keys():
+                match = pattern.match(key)
+                if match:
+                    key = '_'.join((match[1], 'password'))
+                    break
+            else:
+                continue
+            break
+        if key is not None:
+            log("Updating password on key {} on relation_id: {}"
+                .format(key, rid),
+                INFO)
+            relation_set(relation_id=rid,
+                         relation_settings={key: new_passwd})
+            # set the password for the peer as well for update_client to work
+            # on the non-leader units
+            peer_key = "{}_{}".format(rid, key)
+            try:
+                peer_store(peer_key, new_passwd)
+            except ValueError:
+                # if there was no cluster, just push to leader settings.
+                leader_set({peer_key: new_passwd})
+
+
 def update_clients(check_deferred_restarts=True):
     """Update amqp client relation hooks
 
@@ -326,7 +398,7 @@ def update_clients(check_deferred_restarts=True):
     :type check_deferred_events: bool
     """
     if check_deferred_restarts and get_deferred_restarts():
-        log("Not sendinfg client update as a restart is pending.", INFO)
+        log("Not sending client update as a restart is pending.", INFO)
         return
     _leader_node_is_ready = rabbit.leader_node_is_ready()
     _client_node_is_ready = rabbit.client_node_is_ready()

tests/tests.yaml

@@ -12,6 +12,7 @@ dev_bundles:
 tests:
   - zaza.openstack.charm_tests.rabbitmq_server.tests.RabbitMQDeferredRestartTest
   - zaza.openstack.charm_tests.rabbitmq_server.tests.RmqTests
+  - zaza.openstack.charm_tests.rabbitmq_server.tests.RmqRotateServiceUserPasswordTests
 
 tests_options:
   force_deploy: