From 5bb728358e32bb793ea27f618fcb49b8bc9787ce Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 8 Jun 2018 09:01:27 -0500
Subject: [PATCH 01/10] mgr/devicehealth: rename old smart module to
 devicehealth

Let's avoid "SMART" since it's misleading (it refers specifically to ATA).

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/pybind/mgr/devicehealth/__init__.py |  2 ++
 src/pybind/mgr/devicehealth/module.py   | 37 +++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 src/pybind/mgr/devicehealth/__init__.py
 create mode 100644 src/pybind/mgr/devicehealth/module.py

diff --git a/src/pybind/mgr/devicehealth/__init__.py b/src/pybind/mgr/devicehealth/__init__.py
new file mode 100644
index 0000000000000..4c5b97ce82b27
--- /dev/null
+++ b/src/pybind/mgr/devicehealth/__init__.py
@@ -0,0 +1,2 @@
+
+from .module import Module
diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
new file mode 100644
index 0000000000000..2e6755e617b73
--- /dev/null
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -0,0 +1,37 @@
+
+"""
+Device health monitoring
+"""
+
+import json
+from mgr_module import MgrModule, CommandResult
+import rados
+from threading import Event
+from datetime import datetime, timedelta, date, time
+
+class Module(MgrModule):
+    COMMANDS = [
+        {
+            "cmd": "device query-daemon-health-metrics "
+                   "name=osd_id,type=CephOsdName,req=true",
+            "desc": "Get device health metrics for a given daemon (OSD)",
+            "perm": "r"
+        },
+    ]
+
+    def handle_command(self, cmd):
+        self.log.error("handle_command")
+
+        if cmd['prefix'] == 'device query-daemon-health-metrics':
+            result = CommandResult('')
+            self.send_command(result, 'osd', str(cmd['osd_id']), json.dumps({
+                'prefix': 'smart',
+                'format': 'json',
+            }), '')
+            r, outb, outs = result.wait()
+            return (r, outb, outs)
+
+        else:
+            # mgr should respect our self.COMMANDS and not call us for
+            # any prefix we don't advertise
+            raise NotImplementedError(cmd['prefix'])

From 45b019feed1a8091cb5f1b44d43e611e61d14e20 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 8 Jun 2018 09:01:34 -0500
Subject: [PATCH 02/10] mgr: enable devicehealth by default

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/common/options.cc | 2 +-
 src/vstart.sh         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 83753de44102f..31b43b98e57e3 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4749,7 +4749,7 @@ std::vector<Option> get_global_options() {
     .set_description("Filesystem path to manager modules."),
 
     Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)
-    .set_default("restful status balancer iostat")
+    .set_default("restful status balancer iostat devicehealth")
     .set_flag(Option::FLAG_NO_MON_UPDATE)
     .set_flag(Option::FLAG_CLUSTER_CREATE)
     .add_service("mon")
diff --git a/src/vstart.sh b/src/vstart.sh
index 97d17bbbdd436..7afc9c88b477c 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -434,7 +434,7 @@ prepare_conf() {
         heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
 "
 
-    local mgr_modules="restful status balancer iostat"
+    local mgr_modules="restful status balancer iostat devicehealth"
     if $with_mgr_dashboard; then
       mgr_modules="dashboard $mgr_modules"
     fi

From c72568da39a7c52ab0fbf31d98e398e14f876c09 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 8 Jun 2018 09:21:36 -0500
Subject: [PATCH 03/10] osd: key smart results by unique device id

This is more useful than the current local device name.

Clean up some formatting.

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/osd/OSD.cc | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 591a1b6090252..56c0389e685ce 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -6196,10 +6196,12 @@ void OSD::probe_smart(ostream& ss)
 {
   set<string> devnames;
   store->get_devices(&devnames);
-  uint64_t smart_timeout = cct->_conf->get_val<uint64_t>("osd_smart_report_timeout");
+  uint64_t smart_timeout = cct->_conf->get_val<uint64_t>(
+    "osd_smart_report_timeout");
   std::string result;
 
-  json_spirit::mObject json_map; // == typedef std::map<std::string, mValue> mObject;
+  // == typedef std::map<std::string, mValue> mObject;
+  json_spirit::mObject json_map;
   json_spirit::mValue smart_json;
 
   for (auto dev : devnames) {
@@ -6208,6 +6210,12 @@ void OSD::probe_smart(ostream& ss)
       continue;
     }
 
+    string devid = get_device_id(dev);
+    if (devid.size() == 0) {
+      dout(10) << __func__ << " no unique id for dev " << dev << ", skipping"
+	       << dendl;
+      continue;
+    }
     if (probe_smart_device(("/dev/" + dev).c_str(), smart_timeout, &result)) {
       dout(10) << "probe_smart_device failed for /dev/" << dev << dendl;
       continue;
@@ -6217,7 +6225,7 @@ void OSD::probe_smart(ostream& ss)
     if (!json_spirit::read(result, smart_json)) {
       derr << "smartctl JSON output of /dev/" + dev + " is invalid" << dendl;
     } else { //json is valid, assigning
-      json_map[dev] = smart_json;
+      json_map[devid] = smart_json;
     }
     // no need to result.clear() or clear smart_json
   }
@@ -6227,8 +6235,9 @@ void OSD::probe_smart(ostream& ss)
 int OSD::probe_smart_device(const char *device, int timeout, std::string *result)
 {
   // when using --json, smartctl will report its errors in JSON format to stdout 
-  SubProcessTimed smartctl("sudo", SubProcess::CLOSE, SubProcess::PIPE, SubProcess::CLOSE,
-			   timeout);
+  SubProcessTimed smartctl(
+    "sudo", SubProcess::CLOSE, SubProcess::PIPE, SubProcess::CLOSE,
+    timeout);
   smartctl.add_cmd_args(
     "smartctl",
     "-a",

From 78916b8355cc38d80791237335fdf25437aba7d2 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 8 Jun 2018 17:45:25 -0500
Subject: [PATCH 04/10] mgr/ActivePyModules: add get for 'device <devid>'

This is slightly different than the usual pattern because it is
parameterized.  I want to avoid fetching *all* devices if we don't need
it.

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/mgr/ActivePyModules.cc | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 707a6333c9288..0c411435b79d8 100644
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -281,10 +281,10 @@ PyObject *ActivePyModules::get_python(const std::string &what)
     return f.get();
   } else if (what == "pg_dump") {
     PyFormatter f;
-        cluster_state.with_pgmap(
-        [&f](const PGMap &pg_map) {
-	  pg_map.dump(&f);
-        }
+    cluster_state.with_pgmap(
+      [&f](const PGMap &pg_map) {
+	pg_map.dump(&f);
+      }
     );
     return f.get();
   } else if (what == "devices") {
@@ -295,6 +295,14 @@ PyObject *ActivePyModules::get_python(const std::string &what)
       });
     f.close_section();
     return f.get();
+  } else if (what.size() > 7 &&
+	     what.substr(0, 7) == "device ") {
+    string devid = what.substr(7);
+    PyFormatter f;
+    daemon_state.with_device(devid, [&f] (const DeviceState& dev) {
+	f.dump_object("device", dev);
+      });
+    return f.get();
   } else if (what == "io_rate") {
     PyFormatter f;
     cluster_state.with_pgmap(

From c1eeee0ba079fc9d7f727a0294f13bc8fa65e1ce Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 8 Jun 2018 17:45:43 -0500
Subject: [PATCH 05/10] osd: optionally query smart for a single devid

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/osd/OSD.cc | 19 ++++++++++++++-----
 src/osd/OSD.h  |  2 +-
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 56c0389e685ce..bba5c5af47334 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2243,7 +2243,9 @@ will start to track new ops received afterwards.";
     }
     f->close_section();
   } else if (admin_command == "smart") {
-    probe_smart(ss);
+    string devid;
+    cmd_getval(cct, cmdmap, "devid", devid);
+    probe_smart(devid, ss);
   } else if (admin_command == "list_devices") {
     set<string> devnames;
     store->get_devices(&devnames);
@@ -6171,7 +6173,9 @@ void OSD::do_command(Connection *con, ceph_tid_t tid, vector<string>& cmd, buffe
   }
 
   else if (prefix == "smart") {
-    probe_smart(ds);
+    string devid;
+    cmd_getval(cct, cmdmap, "devid", devid);
+    probe_smart(devid, ds);
   }
 
   else {
@@ -6192,13 +6196,12 @@ void OSD::do_command(Connection *con, ceph_tid_t tid, vector<string>& cmd, buffe
   }
 }
 
-void OSD::probe_smart(ostream& ss)
+void OSD::probe_smart(const string& only_devid, ostream& ss)
 {
   set<string> devnames;
   store->get_devices(&devnames);
   uint64_t smart_timeout = cct->_conf->get_val<uint64_t>(
     "osd_smart_report_timeout");
-  std::string result;
 
   // == typedef std::map<std::string, mValue> mObject;
   json_spirit::mObject json_map;
@@ -6216,9 +6219,15 @@ void OSD::probe_smart(ostream& ss)
 	       << dendl;
       continue;
     }
+    if (only_devid.size() && devid != only_devid) {
+      continue;
+    }
+
+    std::string result;
     if (probe_smart_device(("/dev/" + dev).c_str(), smart_timeout, &result)) {
       dout(10) << "probe_smart_device failed for /dev/" << dev << dendl;
-      continue;
+      //continue;
+      result = "{\"error\": \"smartctl failed\", \"dev\": \"" + dev + "\"}";
     }
 
     // TODO: change to read_or_throw?
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 7f0afb96eb544..c4cf9fcea9555 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -2208,7 +2208,7 @@ class OSD : public Dispatcher,
 
   float get_osd_recovery_sleep();
 
-  void probe_smart(ostream& ss);
+  void probe_smart(const string& devid, ostream& ss);
   int probe_smart_device(const char *device, int timeout, std::string *result);
 
 public:

From 97b79a597df34c04f6ab962a26a4c4e9937b8499 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 8 Jun 2018 14:59:34 -0500
Subject: [PATCH 06/10] mgr/devicehealth: add some scraping infrastructure

- command to fetch smart info
- command to scrape a device and store the metrics
- command to scrape all devices (and store)
- command to dump stored metrics
- purging of old metrics

This is based on code originally written by Yaarit.

Signed-off-by: Yaarit Hatuka yaarithatuka@gmail.com
Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/pybind/mgr/devicehealth/module.py | 216 +++++++++++++++++++++++++-
 1 file changed, 214 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index 2e6755e617b73..960842c3054b9 100644
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -3,35 +3,247 @@
 Device health monitoring
 """
 
+import errno
 import json
 from mgr_module import MgrModule, CommandResult
 import rados
 from threading import Event
 from datetime import datetime, timedelta, date, time
 
+TIME_FORMAT = '%Y%m%d-%H%M%S'
+
 class Module(MgrModule):
+    OPTIONS = [
+        { 'name': 'active' },
+        { 'name': 'scrape_frequency' },
+        { 'name': 'pool_name' },
+        { 'name': 'retention_period' },
+    ]
+    DEFAULTS = {
+        'active': True,
+        'scrape_frequency': str(86400),
+        'retention_period': str(86400*14),
+        'pool_name': 'device_health_metrics',
+    }
+    active = DEFAULTS['active']
+    scrape_frequency = DEFAULTS['scrape_frequency']
+    retention_period = DEFAULTS['retention_period']
+    pool_name = DEFAULTS['pool_name']
+
     COMMANDS = [
         {
             "cmd": "device query-daemon-health-metrics "
-                   "name=osd_id,type=CephOsdName,req=true",
+                   "name=who,type=CephString",
             "desc": "Get device health metrics for a given daemon (OSD)",
             "perm": "r"
         },
+        {
+            "cmd": "device scrape-daemon-health-metrics "
+                   "name=who,type=CephString",
+            "desc": "Scrape and store device health metrics for a given daemon",
+            "perm": "r"
+        },
+        {
+            "cmd": "device scrape-health-metrics name=devid,type=CephString,req=False",
+            "desc": "Scrape and store health metrics",
+            "perm": "r"
+        },
+        {
+            "cmd": "device show-health-metrics name=devid,type=CephString name=sample,type=CephString,req=False",
+            "desc": "Show stored device metrics for the device",
+            "perm": "r"
+        },
     ]
 
+    run = True
+    event = Event()
+    last_scrape_time = ""
+
     def handle_command(self, cmd):
         self.log.error("handle_command")
 
         if cmd['prefix'] == 'device query-daemon-health-metrics':
+            who = cmd.get('who', '')
+            if who[0:4] != 'osd.':
+                return (-errno.EINVAL, '', 'not a valid <osd.NNN> id')
+            osd_id = who[4:]
             result = CommandResult('')
-            self.send_command(result, 'osd', str(cmd['osd_id']), json.dumps({
+            self.send_command(result, 'osd', osd_id, json.dumps({
                 'prefix': 'smart',
                 'format': 'json',
             }), '')
             r, outb, outs = result.wait()
             return (r, outb, outs)
+        elif cmd['prefix'] == 'device scrape-daemon-health-metrics':
+            who = cmd.get('who', '')
+            if who[0:4] != 'osd.':
+                return (-errno.EINVAL, '', 'not a valid <osd.NNN> id')
+            id = int(who[4:])
+            return self.scrape_osd(id)
+        elif cmd['prefix'] == 'device scrape-health-metrics':
+            if 'devid' in cmd:
+                return self.scrape_device(cmd['devid'])
+            return self.scrape_all();
+        elif cmd['prefix'] == 'device show-health-metrics':
+            return self.show_device_metrics(cmd['devid'], cmd.get('sample'))
 
         else:
             # mgr should respect our self.COMMANDS and not call us for
             # any prefix we don't advertise
             raise NotImplementedError(cmd['prefix'])
+
+    def refresh_config(self):
+        self.active = self.get_config('active', '') is not '' or 'false'
+        for opt, value in self.DEFAULTS.iteritems():
+            setattr(self, opt, self.get_config(opt) or value)
+
+    def serve(self):
+        self.log.info("Starting")
+        while self.run:
+            self.refresh_config()
+
+            # TODO normalize/align sleep interval
+            sleep_interval = int(self.scrape_frequency)
+
+            self.log.debug('Sleeping for %d seconds', sleep_interval)
+            ret = self.event.wait(sleep_interval)
+            self.event.clear()
+
+            # in case 'wait' was interrupted, it could mean config was changed
+            # by the user; go back and read config vars
+            if ret:
+                continue
+
+            self.log.debug('Waking up [%s]',
+                           "active" if self.active else "inactive")
+            if not self.active:
+                continue
+            self.log.debug('Running')
+
+    def open_connection(self):
+        pools = self.rados.list_pools()
+        is_pool = False
+        for pool in pools:
+            if pool == self.pool_name:
+                is_pool = True
+                break
+        if not is_pool:
+            self.rados.create_pool(self.pool_name)
+            self.log.debug('create %s pool' % self.pool_name)
+        ioctx = self.rados.open_ioctx(self.pool_name)
+        return (ioctx)
+
+    def scrape_osd(self, osd_id):
+        ioctx = self.open_connection()
+        raw_smart_data = self.do_scrape_osd(osd_id, ioctx)
+        if raw_smart_data:
+            for device, raw_data in raw_smart_data.items():
+                data = self.extract_smart_features(raw_data)
+                self.put_device_metrics(ioctx, device, data)
+        ioctx.close()
+        return (0, "", "")
+
+    def scrape_all(self):
+        osdmap = self.get("osd_map")
+        assert osdmap is not None
+        ioctx = self.open_connection()
+        did_device = {}
+        for osd in osdmap['osds']:
+            osd_id = osd['osd']
+            raw_smart_data = self.do_scrape_osd(osd_id, ioctx)
+            if not raw_smart_data:
+                continue
+            for device, raw_data in raw_smart_data.items():
+                if device in did_device:
+                    self.log.debug('skipping duplicate %s' % device)
+                    continue
+                did_device[device] = 1
+                data = self.extract_smart_features(raw_data)
+                self.put_device_metrics(ioctx, device, data)
+
+        ioctx.close()
+        return (0, "", "")
+
+    def scrape_device(self, devid):
+        r = self.get("device " + devid)
+        if not r or 'device' not in r.keys():
+            return (-errno.ENOENT, '', 'device ' + devid + ' not found')
+        daemons = r['device'].get('daemons', [])
+        osds = [int(r[4:]) for r in daemons if r.startswith('osd.')]
+        if not osds:
+            return (-errno.EAGAIN, '',
+                    'device ' + devid + ' not claimed by any active OSD daemons')
+        osd_id = osds[0]
+        ioctx = self.open_connection()
+        raw_smart_data = self.do_scrape_osd(osd_id, ioctx, devid=devid)
+        if raw_smart_data:
+            for device, raw_data in raw_smart_data.items():
+                data = self.extract_smart_features(raw_data)
+                self.put_device_metrics(ioctx, device, data)
+        ioctx.close()
+        return (0, "", "")
+
+    def do_scrape_osd(self, osd_id, ioctx, devid=''):
+        self.log.debug('do_scrape_osd osd.%d' % osd_id)
+
+        # scrape from osd
+        result = CommandResult('')
+        self.send_command(result, 'osd', str(osd_id), json.dumps({
+            'prefix': 'smart',
+            'format': 'json',
+            'devid': devid,
+        }), '')
+        r, outb, outs = result.wait()
+
+        try:
+            return json.loads(outb)
+        except:
+            self.log.debug('Fail to parse JSON result from "%s"' % outb)
+
+    def put_device_metrics(self, ioctx, devid, data):
+        old_key = datetime.now() - timedelta(
+            seconds=int(self.retention_period))
+        prune = old_key.strftime(TIME_FORMAT)
+        self.log.debug('put_device_metrics device %s prune %s' %
+                       (devid, prune))
+        erase = []
+        with rados.ReadOpCtx() as op:
+            iter, ret = ioctx.get_omap_keys(op, "", 500) # fixme
+            assert ret == 0
+            ioctx.operate_read_op(op, devid)
+            for key, _ in list(iter):
+                if key >= prune:
+                    break
+                erase.append(key)
+        key = datetime.now().strftime(TIME_FORMAT)
+        self.log.debug('put_device_metrics device %s key %s = %s, erase %s' %
+                       (devid, key, data, erase))
+        with rados.WriteOpCtx() as op:
+            ioctx.set_omap(op, (key,), (str(json.dumps(data)),))
+            if len(erase):
+                ioctx.remove_omap_keys(op, tuple(erase))
+            ioctx.operate_write_op(op, devid)
+
+    def show_device_metrics(self, devid, sample):
+        ioctx = self.open_connection()
+        res = {}
+        with rados.ReadOpCtx() as op:
+            iter, ret = ioctx.get_omap_vals(op, "", sample or '', 500) # fixme
+            assert ret == 0
+            ioctx.operate_read_op(op, devid)
+            for key, value in list(iter):
+                if sample and key != sample:
+                    break
+                try:
+                    v = json.loads(value)
+                except:
+                    self.log.debug('unable to parse value for %s: "%s"' %
+                                   (key, value))
+                    pass
+                res[key] = v
+        return (0, json.dumps(res, indent=4), '')
+
+    def extract_smart_features(self, raw):
+        # FIXME: extract and normalize raw smartctl --json output and
+        # generate a dict of the fields we care about.
+        return raw

From 03fde46f02a729ea77c7ea840b49699af27aa050 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Mon, 11 Jun 2018 07:10:08 -0500
Subject: [PATCH 07/10] mgr/devicehealth: fix 'device get-health-metrics' when
 no metrics are stored

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/pybind/mgr/devicehealth/module.py | 30 +++++++++++++++++----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index 960842c3054b9..eb7796ab9b54a 100644
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -225,22 +225,30 @@ def put_device_metrics(self, ioctx, devid, data):
             ioctx.operate_write_op(op, devid)
 
     def show_device_metrics(self, devid, sample):
+        # verify device exists
+        r = self.get("device " + devid)
+        if not r or 'device' not in r.keys():
+            return (-errno.ENOENT, '', 'device ' + devid + ' not found')
+        # fetch metrics
         ioctx = self.open_connection()
         res = {}
         with rados.ReadOpCtx() as op:
             iter, ret = ioctx.get_omap_vals(op, "", sample or '', 500) # fixme
             assert ret == 0
-            ioctx.operate_read_op(op, devid)
-            for key, value in list(iter):
-                if sample and key != sample:
-                    break
-                try:
-                    v = json.loads(value)
-                except:
-                    self.log.debug('unable to parse value for %s: "%s"' %
-                                   (key, value))
-                    pass
-                res[key] = v
+            try:
+                ioctx.operate_read_op(op, devid)
+                for key, value in list(iter):
+                    if sample and key != sample:
+                        break
+                    try:
+                        v = json.loads(value)
+                    except:
+                        self.log.debug('unable to parse value for %s: "%s"' %
+                                       (key, value))
+                        pass
+                    res[key] = v
+            except:
+                pass
         return (0, json.dumps(res, indent=4), '')
 
     def extract_smart_features(self, raw):

From 5a44d4f4097b45c5a984be9fe53e3072dc9fba4d Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Mon, 11 Jun 2018 13:34:54 -0500
Subject: [PATCH 08/10] mgr/mgr_module: update docstring for get()

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/pybind/mgr/mgr_module.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 1db2e71a425e3..84f0bcddac8af 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -312,7 +312,8 @@ def get(self, data_name):
 
         :param str data_name: Valid things to fetch are osd_crush_map_text, 
                 osd_map, osd_map_tree, osd_map_crush, config, mon_map, fs_map,
-                osd_metadata, pg_summary, io_rate, pg_dump, df, osd_stats, health, mon_status.
+                osd_metadata, pg_summary, io_rate, pg_dump, df, osd_stats,
+                health, mon_status, devices, device <devid>.
 
         Note:
             All these structures have their own JSON representations: experiment

From 32c80f51086a03070e5c6f7628e4a6a17ad1b6a6 Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Tue, 12 Jun 2018 14:11:09 -0400
Subject: [PATCH 09/10] mgr/devicehealth: add helpers to
 life_expectancy_response()

- if mark_out_threshold is met we write to log.warn instead of raising a
  health warning.
- check that OSD is 'in' before calling mark_out().
- raise a health warning in case OSD is marked 'out' but still has PGs
  attached to it.
- cast thresholds default values to string.
- add SCSI multipath support to health warning message.
- change health warning message.

Signed-off-by: Yaarit Hatuka yaarithatuka@gmail.com
---
 src/pybind/mgr/devicehealth/module.py | 98 +++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index eb7796ab9b54a..7aa99e6e043e8 100644
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -24,6 +24,9 @@ class Module(MgrModule):
         'scrape_frequency': str(86400),
         'retention_period': str(86400*14),
         'pool_name': 'device_health_metrics',
+        'mark_out_threshold': str(86400*14),
+        'warn_threshold': str(86400*14*2),
+        'self_heal': True,
     }
     active = DEFAULTS['active']
     scrape_frequency = DEFAULTS['scrape_frequency']
@@ -251,6 +254,101 @@ def show_device_metrics(self, devid, sample):
                 pass
         return (0, json.dumps(res, indent=4), '')
 
+    def life_expectancy_response(self):
+        mark_out_threshold_td = timedelta(seconds=int(self.mark_out_threshold))
+        warn_threshold_td = timedelta(seconds=int(self.warn_threshold))
+        health_warnings = []
+        devs = self.get("devices")
+        for dev in devs['devices']:
+            if 'life_expectancy_min' not in dev:
+                continue
+            # life_expectancy_(min/max) is in the format of:
+            # '%Y-%m-%d %H:%M:%S.%f', e.g.:
+            # '2019-01-20 21:12:12.000000'
+            life_expectancy_min = datetime.strptime(dev['life_expectancy_min'], '%Y-%m-%d %H:%M:%S.%f')
+            now = datetime.now()
+            if life_expectancy_min - now <= mark_out_threshold_td:
+                if self.self_heal:
+                    # dev['daemons'] == ["osd.0","osd.1","osd.2"]
+                    if dev['daemons']:
+                        osd_ids = map(lambda x: x[4:], dev['daemons'])
+                        osds_in = []
+                        osds_out = []
+                        for _id in osd_ids:
+                            if self.is_osd_in(_id):
+                                osds_in.append(_id)
+                            else:
+                                osds_out.append(_id)
+                        if osds_in:
+                            self.mark_out(osds_in)
+                        # OSD might be marked 'out' (which means it has no
+                        # data), however PGs are still attached to it.
+                        for _id in osds_out:
+                            num_pgs = self.get_osd_num_pgs(_id)
+                            if num_pgs > 0:
+                                health_warnings.append('osd.%s is marked out, '
+                                                       'but still has %s PG(s)'
+                                                       ' attached' %
+                                                       (_id, num_pgs))
+                        # TODO: set_primary_affinity
+                self.log.warn(self.create_warning_message(dev))
+            elif life_expectancy_min - now <= warn_threshold_td:
+                health_warnings.append(self.create_warning_message(dev))
+        if health_warnings:
+            self.set_health_checks({
+                'MGR_DEVICE_HEALTH': {
+                    'severity': 'warning',
+                    'summary': 'Imminent failure anticipated for device(s)',
+                    'detail': health_warnings
+                }
+            })
+        else:
+            self.set_health_checks({}) # clearing health checks
+        return (0,"","")
+
+    def is_osd_in(self, osd_id):
+        osdmap = self.get("osd_map")
+        assert osdmap is not None
+        for osd in osdmap['osds']:
+            if str(osd_id) == str(osd['osd']):
+                return osd['in']
+        # return False
+
+    def get_osd_num_pgs(self, osd_id):
+        stats = self.get('osd_stats')
+        assert stats is not None
+        for stat in stats['osd_stats']:
+            if str(osd_id) == str(stat['osd']):
+                return stat['num_pgs']
+        return -1
+
+    def create_warning_message(self, dev):
+        # device can appear in more than one location in case of SCSI multipath
+        device_locations = map(lambda x: x['host'] + ':' + x['dev'], dev['location'])
+        return ('%s at %s;'
+               ' Affected OSDs: %s;'
+               ' Life expectancy: between %s and %s'
+               % (dev['devid'],
+                device_locations,
+                dev.get('daemons', 'none'),
+                dev['life_expectancy_min'],
+                dev.get('life_expectancy_max', 'unknown')))
+                # TODO: by default, dev['life_expectancy_max'] == '0.000000',
+                # so dev.get('life_expectancy_max', 'unknown')
+                # above should be altered.
+
+    def mark_out(self, osd_ids):
+        self.log.info('Marking out OSDs: %s' % osd_ids)
+        result = CommandResult('')
+        self.send_command(result, 'mon', '', json.dumps({
+            'prefix': 'osd out',
+            'format': 'json',
+            'ids': osd_ids,
+        }), '')
+        r, outb, outs = result.wait()
+        if r != 0:
+            self.log.warn('Could not mark OSD %s out. r: [%s], outb: [%s], outs: [%s]' % (osd_ids, r, outb, outs))
+
     def extract_smart_features(self, raw):
         # FIXME: extract and normalize raw smartctl --json output and
         # generate a dict of the fields we care about.

From 547ba067e606cde449270a6422fb2fdaa7159d50 Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Wed, 13 Jun 2018 10:19:40 -0400
Subject: [PATCH 10/10] common/blkdev remove debug statements

Signed-off-by: Yaarit Hatuka yaarithatuka@gmail.com
---
 src/common/blkdev.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/common/blkdev.cc b/src/common/blkdev.cc
index 40deb9e341496..c206c0360fd54 100644
--- a/src/common/blkdev.cc
+++ b/src/common/blkdev.cc
@@ -384,12 +384,10 @@ std::string get_device_id(const std::string& devname)
 
   udev = udev_new();
   if (!udev) {
-    //derr << "failed to run udev_new(), when calling for device " << devname << dendl;
     return {};
   }
   dev = udev_device_new_from_subsystem_sysname(udev, "block", devname.c_str());
   if (!dev) {
-    //derr << "failed to run udev_device_new_from_subsystem_sysname() for " << devname << dendl;
     udev_unref(udev);
     return {};
   }
@@ -405,7 +403,6 @@ std::string get_device_id(const std::string& devname)
   udev_unref(udev);
 
   if (!device_id.empty()) {
-    //dout << devname << " serial number: " << data << dendl;
     std::replace(device_id.begin(), device_id.end(), ' ', '_');
     return device_id;
   }
@@ -413,7 +410,6 @@ std::string get_device_id(const std::string& devname)
   // either udev_device_get_property_value() failed, or succeeded but
   // returned nothing; trying to read from files.  note that the 'vendor'
   // file rarely contains the actual vendor; it's usually 'ATA'.
-  //derr << "udev could not retrieve serial number of " << devname << dendl;
   std::string model, serial;
   model = get_block_device_string_property_wrap(devname, "device/model");
   serial = get_block_device_string_property_wrap(devname, "device/serial");
@@ -434,7 +430,6 @@ std::string get_block_device_string_property_wrap(const std::string &devname,
   std::string prop_val;
   int ret = get_block_device_string_property(devname.c_str(), property.c_str(), buff, sizeof(buff));
   if (ret < 0) {
-    //derr << "Could not retrieve content of " << property << " file of " << devname << dendl;
     return {};
   }
   prop_val = buff;