WIP: Preliminary update of the audit-tool for cryptographic commitments

Signed-off-by: Sam Stuewe <[email protected]>

src/uhs/atomizer/shard/controller.cpp

@@ -235,15 +235,18 @@ namespace cbdc::shard {
             m_audit_thread.join();
         }
         m_audit_thread = std::thread([this, s = std::move(snp), height]() {
-            auto maybe_total = m_shard.audit(s);
-            if(!maybe_total.has_value()) {
-                m_logger->fatal("Error running audit at height", height);
+            auto range_summaries = m_shard.audit(s);
+            auto buf = cbdc::buffer();
+            buf.extend(sizeof(commitment_t));
+            for(const auto& [bucket, summary] : range_summaries) {
+                buf.clear();
+                buf.append(summary.data(), summary.size());
+                m_audit_log
+                    << height << " "
+                    << static_cast<int>(bucket) << " "
+                    << buf.to_hex() << std::endl;
             }
-            m_audit_log << height << " " << maybe_total.value() << std::endl;
-            m_logger->info("Audit completed for",
-                           height,
-                           maybe_total.value(),
-                           "coins total");
+            m_logger->info("Audit completed for", height);
         });
     }
 }

src/uhs/atomizer/shard/shard.cpp

@@ -4,6 +4,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "shard.hpp"
+#include "uhs/transaction/messages.hpp"
 
 #include <utility>
 
@@ -205,34 +206,44 @@ namespace cbdc::shard {
     }
 
     auto shard::audit(const std::shared_ptr<const leveldb::Snapshot>& snp)
-        -> std::optional<uint64_t> {
+        -> std::unordered_map<unsigned char, commitment_t> {
+        std::unordered_map<unsigned char, std::vector<commitment_t>> comms{};
         auto opts = leveldb::ReadOptions();
         opts.snapshot = snp.get();
         auto it = std::shared_ptr<leveldb::Iterator>(m_db->NewIterator(opts));
         it->SeekToFirst();
         // Skip best block height key
         it->Next();
-        uint64_t tot{};
         for(; it->Valid(); it->Next()) {
             auto key = it->key();
-            auto buf = cbdc::buffer();
-            buf.extend(key.size());
-            std::memcpy(buf.data(), key.data(), key.size());
-            auto maybe_uhs_element
-                = cbdc::from_buffer<transaction::uhs_element>(buf);
-            if(!maybe_uhs_element.has_value()) {
-                return std::nullopt;
+            auto val = it->value();
+
+            transaction::compact_output outp{};
+            std::memcpy(outp.m_id.data(), key.data(), key.size());
+            std::memcpy(outp.m_auxiliary.data(), val.data(), outp.m_auxiliary.size());
+            std::memcpy(outp.m_range.data(), val.data() + sizeof(outp.m_auxiliary), outp.m_range.size());
+            std::memcpy(outp.m_provenance.data(), val.data() + sizeof(outp.m_auxiliary) + sizeof(outp.m_range), outp.m_provenance.size());
+            if(!transaction::validate_uhs_id(outp)) {
+                continue;
+            }
+            auto bucket = outp.m_id[0];
+            if(comms.find(bucket) == comms.end()) {
+                std::vector<commitment_t> commits{};
+                commits.reserve(1);
+                comms.emplace(bucket, std::move(commits));
             }
-            auto& uhs_element = maybe_uhs_element.value();
-            if(transaction::calculate_uhs_id(uhs_element.m_data,
-                                             uhs_element.m_value)
-               != uhs_element.m_id) {
-                return std::nullopt;
+            comms[bucket].emplace_back(std::move(outp.m_auxiliary));
+        }
+
+        std::unordered_map<unsigned char, commitment_t> summaries{};
+        for(auto& [k, v] : comms) {
+            auto summary = sum_commitments(m_secp.get(), v);
+            if (summary.has_value()) {
+                summaries[k] = summary.value();
             }
-            tot += uhs_element.m_value;
         }
 
-        return tot;
+        return summaries;
     }
 
     auto shard::get_snapshot() -> std::shared_ptr<const leveldb::Snapshot> {

src/uhs/atomizer/shard/shard.hpp

@@ -72,11 +72,9 @@ namespace cbdc::shard {
         /// Audit the supply of coins in this shard's UHS and check UHS IDs
         /// match the nested data and value stored in the UHS.
         /// \param snp LevelDB snapshot upon which to calculate the audit.
-        /// \return total value of all UHS elements in the snapshot or
-        ///         std::nullopt if any of the UHS elements do not match their
-        ///         UHS ID.
+        /// \return per-range summary commitments to value
         auto audit(const std::shared_ptr<const leveldb::Snapshot>& snp)
-            -> std::optional<uint64_t>;
+            -> std::unordered_map<unsigned char, commitment_t>;
 
       private:
         [[nodiscard]] auto is_output_on_shard(const hash_t& uhs_hash) const
@@ -96,6 +94,13 @@ namespace cbdc::shard {
 
         const std::string m_best_block_height_key;
 
+        static const inline auto m_secp
+            = std::unique_ptr<secp256k1_context,
+                              decltype(&secp256k1_context_destroy)>(
+                secp256k1_context_create(SECP256K1_CONTEXT_SIGN
+                                         | SECP256K1_CONTEXT_VERIFY),
+                &secp256k1_context_destroy);
+
         std::pair<uint8_t, uint8_t> m_prefix_range;
     };
 }

src/uhs/transaction/transaction.cpp

@@ -182,6 +182,17 @@ namespace cbdc::transaction {
         return id;
     }
 
+    auto validate_uhs_id(const compact_output& output) -> bool {
+        CSHA256 sha;
+        sha.Write(output.m_provenance.data(), output.m_provenance.size());
+        sha.Write(output.m_auxiliary.data(), output.m_auxiliary.size());
+
+        hash_t check{};
+        sha.Finalize(check.data());
+
+        return output.m_id == check;
+    }
+
     auto roll_auxiliaries(secp256k1_context* ctx,
                           random_source& rng,
                           const std::vector<hash_t>& blinds,

src/uhs/transaction/transaction.hpp

@@ -84,10 +84,6 @@ namespace cbdc::transaction {
                           const out_point& point,
                           const output& put) -> std::pair<hash_t, hash_t>;
 
-    auto calculate_uhs_id(const out_point& point,
-                          const output& put,
-                          const commitment_t& value) -> hash_t;
-
     /// \brief Additional information a spender needs to spend an input
     struct spend_data {
         /// The blinding factor for the auxiliary commitment
@@ -179,6 +175,15 @@ namespace cbdc::transaction {
         auto operator!=(const compact_output& rhs) const -> bool;
     };
 
+    /// \brief Validates that a compact_output's UHS ID matches its contents
+    ///
+    /// Simply rehashes the provenance nested-hash and value commitment,
+    /// and checks to see the result is identical to the ID.
+    ///
+    /// \param output the compact_output itself
+    /// \returns true if the rehash matches the included ID, false otherwise
+    auto validate_uhs_id(const compact_output& output) -> bool;
+
     /// \brief A condensed, hash-only transaction representation
     ///
     /// The minimum amount of data necessary for the transaction processor to

src/util/common/commitment.cpp

@@ -61,4 +61,41 @@ namespace cbdc {
 
         return commitment;
     }
+
+    auto sum_commitments(const secp256k1_context* ctx,
+                         std::vector<commitment_t> commitments)
+        -> std::optional<commitment_t> {
+        if(commitments.size() == 0) {
+            return std::nullopt;
+        } else if(commitments.size() == 1) {
+            return {commitments[0]};
+        }
+
+        std::vector<secp256k1_pubkey> as_keys{};
+        for(auto& c : commitments) {
+            auto maybe_pc = deserialize_commitment(ctx, c);
+            if(!maybe_pc.has_value()) {
+                return std::nullopt;
+            }
+
+            auto pc = maybe_pc.value();
+            secp256k1_pubkey k{};
+            secp256k1_pedersen_commitment_as_key(&pc, &k);
+            as_keys.emplace_back(k);
+        }
+
+        secp256k1_pubkey k{};
+        auto res =
+            secp256k1_ec_pubkey_combine(ctx, &k,
+                reinterpret_cast<const secp256k1_pubkey* const*>(as_keys.data()),
+                as_keys.size());
+        if(res != 1) {
+            return std::nullopt;
+        }
+
+        secp256k1_pedersen_commitment summary{};
+        secp256k1_pubkey_as_pedersen_commitment(ctx, &k, &summary);
+
+        return serialize_commitment(ctx, summary);
+    }
 }

src/util/common/commitment.hpp

@@ -55,6 +55,16 @@ namespace cbdc {
     auto deserialize_commitment(const secp256k1_context* ctx,
                                 commitment_t comm)
         -> std::optional<secp256k1_pedersen_commitment>;
+
+    /// Attempts to sum a list of Pedersen commitments
+    ///
+    /// \param ctx secp256k1 context initialized for signing and commitment
+    /// \param commitments the vector of commitments to sum
+    /// \return std::nullopt if conversion or summing failed; the summed
+    ///         commitment otherwise
+    auto sum_commitments(const secp256k1_context* ctx,
+                         std::vector<commitment_t> commitments)
+        -> std::optional<commitment_t>;
 }
 
 #endif // OPENCBDC_TX_SRC_COMMON_COMMITMENT_H_

tests/integration/atomizer_end_to_end_test.cpp

@@ -88,7 +88,8 @@ class atomizer_end_to_end_test : public ::testing::Test {
             audit_entries.emplace_back(epoch, value);
         }
         ASSERT_FALSE(audit_entries.empty());
-        ASSERT_EQ(audit_entries.back().second, 100ul);
+        //ASSERT_EQ(audit_entries.back().second, 100ul);
+        // todo: sum_commitments with circulation-commitment
 
         std::filesystem::remove_all("archiver0_db");
         std::filesystem::remove_all("atomizer_raft_log_0");

tools/audit/CMakeLists.txt

@@ -4,4 +4,6 @@ include_directories(../../src)
 
 add_executable(audit audit.cpp)
 target_link_libraries(audit common
+                            util
+                            serialization
                             crypto)

tools/audit/audit.cpp

@@ -3,7 +3,10 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-#include "common/config.hpp"
+#include "util/common/config.hpp"
+#include "util/serialization/util.hpp"
+#include "util/serialization/format.hpp"
+#include "util/common/commitment.hpp"
 
 #include <unordered_map>
 
@@ -24,13 +27,9 @@ auto main(int argc, char** argv) -> int {
     }
     auto cfg = std::get<cbdc::config::options>(cfg_or_err);
 
-    struct total {
-        uint64_t m_total_value{};
-        size_t m_shard_count{};
-    };
-
-    auto totals = std::unordered_map<uint64_t, total>();
+    auto audits = std::unordered_map<uint64_t, std::unordered_map<unsigned char, cbdc::commitment_t>>();
 
+    // todo: ensure/detect whether or not the most recent audit has finished
     for(auto& audit_file : cfg.m_shard_audit_logs) {
         auto f = std::ifstream(audit_file);
         if(!f.good()) {
@@ -39,22 +38,39 @@ auto main(int argc, char** argv) -> int {
         }
 
         uint64_t epoch{};
-        uint64_t total_value{};
-        while(f >> epoch >> total_value) {
-            auto it = totals.find(epoch);
-            if(it != totals.end()) {
-                it->second.m_total_value += total_value;
-                it->second.m_shard_count++;
+        std::string bucket_str{};
+        std::string commit_hex{};
+        while(f >> epoch >> bucket_str >> commit_hex) {
+            auto bucket = static_cast<unsigned char>(std::stoul(bucket_str));
+
+            auto commitbuf = cbdc::buffer::from_hex(commit_hex);
+            auto commit = cbdc::from_buffer<cbdc::commitment_t>(commitbuf.value()).value();
+
+            auto it = audits.find(epoch);
+            if(it != audits.end()) {
+                auto& audit = it->second;
+                auto entry = audit.find(bucket);
+                if(entry != audit.end()) {
+                    if(entry->second != commit) {
+                        std::cerr << "Audit failed at epoch " << epoch
+                                  << "; inconsistency in range "
+                                  << bucket_str << std::endl;
+                        return 1;
+                    }
+                } else {
+                    audit[bucket] = commit;
+                }
             } else {
-                totals[epoch] = total{total_value, 1};
+                auto entries = std::unordered_map<unsigned char, cbdc::commitment_t>();
+                entries.emplace(bucket, std::move(commit));
+                audits[epoch] = std::move(entries);
             }
         }
     }
 
-    for(auto& [epoch, tot] : totals) {
-        std::cout << "epoch: " << epoch
-                  << ", total_value: " << tot.m_total_value
-                  << ", shard_count: " << tot.m_shard_count << std::endl;
+    // todo: per-epoch: get a vector of all commitments, push_back circulation_commitment, check sum = 1
+    for(auto& [epoch, entries] : audits) {
+        std::cout << "epoch: " << epoch << std::endl;
     }
 
     return 0;