json-progress.patch

From 87011c3c02fda19e67144ea7bdef4db40ec7e7e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= <mkg20001@gmail.com>
Date: Tue, 14 Jul 2020 17:50:44 +0200
Subject: [PATCH] feat: add a JSON progress bar to fd 3

---
 src/libutil/logging.cc  | 12 ++++++
 src/libutil/logging.hh  |  1 +
 src/nix/progress-bar.cc | 90 ++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 101 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index b379306f..27d7c92f 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -88,6 +88,18 @@ void writeToStderr(const string & s)
     }
 }
 
+void writeToAlien(const string & s)
+{
+    try {
+        writeFull(3, s, false);
+    } catch (SysError & e) {
+        /* Ignore failing writes to stderr.  We need to ignore write
+           errors to ensure that cleanup code that logs to stderr runs
+           to completion if the other side of stderr has been closed
+           unexpectedly. */
+    }
+}
+
 Logger * makeDefaultLogger()
 {
     return new SimpleLogger();
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5df03da7..865f95e1 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -168,5 +168,6 @@ inline void warn(const std::string & fs, Args... args)
 void warnOnce(bool & haveWarned, const FormatOrString & fs);
 
 void writeToStderr(const string & s);
+void writeToAlien(const string & s);
 
 }
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 98049b5d..1ed693a1 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -7,6 +7,7 @@
 #include <atomic>
 #include <map>
 #include <thread>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -80,13 +81,14 @@ public:
         : printBuildLogs(printBuildLogs)
         , isTTY(isTTY)
     {
-        state_.lock()->active = isTTY;
+        state_.lock()->active = true;
         updateThread = std::thread([&]() {
             auto state(state_.lock());
             while (state->active) {
                 if (!state->haveUpdate)
                     state.wait(updateCV);
                 draw(*state);
+                dumpState(*state);
                 state.wait_for(quitCV, std::chrono::milliseconds(50));
             }
         });
@@ -120,6 +122,10 @@ public:
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         if (state.active) {
+            dumpState(state);
+        }
+
+        if (state.active && isTTY) {
             writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n");
             draw(state);
         } else {
@@ -305,10 +311,46 @@ public:
         updateCV.notify_one();
     }
 
+    void dumpState(State & state)
+    {
+        state.haveUpdate = false;
+        if (!state.active || !std::getenv("DUMP_JSON")) return;
+
+        auto s = nlohmann::json::object();
+
+        s["status"] = getStatusJSON(state);
+
+        s["activities"] = nlohmann::json::array();
+
+        auto I = 0;
+
+        if (!state.activities.empty()) {
+            auto i = state.activities.rbegin();
+
+            while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty())))
+                ++i;
+
+            if (i != state.activities.rend()) {
+                auto act = nlohmann::json::object();
+                s["activites"][I] = act;
+                I++;
+                act["s"] = i->s;
+                if (!i->phase.empty()) {
+                    act["phase"] = i->phase;
+                }
+                if (!i->lastLine.empty()) {
+                    act["lastLine"] = i->lastLine;
+                }
+            }
+        }
+
+        writeToAlien(s.dump() + '\n');
+    }
+
     void draw(State & state)
     {
         state.haveUpdate = false;
-        if (!state.active) return;
+        if (!state.active || !isTTY) return;
 
         std::string line;
 
@@ -346,6 +388,50 @@ public:
         writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
     }
 
+    nlohmann::json getStatusJSON(State & state) {
+        auto status = nlohmann::json::object();
+
+        auto MiB = 1024.0 * 1024.0;
+
+        auto renderActivity = [&](ActivityType type, const std::string & key, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
+            auto & act = state.activitiesByType[type];
+            uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
+            for (auto & j : act.its) {
+                done += j.second->done;
+                expected += j.second->expected;
+                running += j.second->running;
+                failed += j.second->failed;
+            }
+
+            expected = std::max(expected, act.expected);
+
+            status[key] = nlohmann::json::object();
+
+            status[key]["unit"] = unit;
+            status[key]["running"] = running;
+            status[key]["done"] = done;
+            status[key]["expected"] = expected;
+            status[key]["failed"] = failed;
+
+            status[key]["itemFmt"] = itemFmt;
+            status[key]["numberFmt"] = numberFmt;
+        };
+
+        renderActivity(actCopyPaths, "copyPaths", "%s copied");
+        renderActivity(actCopyPath, "copyPath", "%s MiB", "%.1f", MiB);
+        renderActivity(actDownload, "download", "%s MiB DL", "%.1f", MiB);
+        renderActivity(actBuilds, "builds", "%s built");
+        renderActivity(actOptimiseStore, "optimiseStore", "%s paths optimised");
+        renderActivity(actVerifyPaths, "verifyPaths", "%s paths verified");
+
+        status["corruptedPaths"] = state.corruptedPaths;
+        status["untrustedPaths"] = state.untrustedPaths;
+        status["bytesLinked"] = state.bytesLinked;
+        status["filesLinked"] = state.filesLinked;
+
+        return status;
+    }
+
     std::string getStatus(State & state)
     {
         auto MiB = 1024.0 * 1024.0;
-- 
2.27.0