Merge branch 'develop' into sventimir/genesis-ledger-export

MinaProtocol · Oct 9, 2023 · 101627b · 101627b
2 parents ba12db3 + 0970329
commit 101627b
Show file tree

Hide file tree

Showing 15 changed files with 100 additions and 29 deletions.
diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall
@@ -4,10 +4,10 @@
 -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI
 {
   toolchainBase = "codaprotocol/ci-toolchain-base:v3",
-  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:563fd7adda282fb3b6765c1811a3566e0fa0560f5d1c5270003483030d82d394",
-  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:49891eb46089f937f054afa464ce9868529981b92b30740cce32ef60957a1098",
-  minaToolchainBookworm = "gcr.io/o1labs-192920/mina-toolchain@sha256:49891eb46089f937f054afa464ce9868529981b92b30740cce32ef60957a1098",
-  minaToolchain = "gcr.io/o1labs-192920/mina-toolchain@sha256:49891eb46089f937f054afa464ce9868529981b92b30740cce32ef60957a1098",
+  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:71173ebccf6af3e24d27262a5071f3dd0bd2c40b9de1c258422fdb9419507d3c",
+  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:9c4062e76fcd910ad60d3f1f58e2395f6a5e70f16fbef422442aedb70112ac73",
+  minaToolchainBookworm = "gcr.io/o1labs-192920/mina-toolchain@sha256:9c4062e76fcd910ad60d3f1f58e2395f6a5e70f16fbef422442aedb70112ac73",
+  minaToolchain = "gcr.io/o1labs-192920/mina-toolchain@sha256:9c4062e76fcd910ad60d3f1f58e2395f6a5e70f16fbef422442aedb70112ac73",
   delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:12ffd0a9016819c720687f440c7a46b8815f8d3ad06d306d342ee5f8dd4375f5",
   elixirToolchain = "elixir:1.10-alpine",
   nodeToolchain = "node:14.13.1-stretch-slim",

diff --git a/dockerfiles/stages/1-build-deps b/dockerfiles/stages/1-build-deps
@@ -20,11 +20,15 @@ ARG GO_VERSION=1.19.11
 ARG GO_CAPNP_VERSION=v3.0.0-alpha.5
 
 # Rust Version passed into rustup-init, can also be "stable", "nightly" or similar
-ARG RUST_VERSION=1.63.0
+# This should stay in line with:
+# - src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml
+# - src/lib/crypto/proof-systems/rust-toolchain.toml
+ARG RUST_VERSION=1.72
 # Nightly Rust Version used for WebAssembly builds
-ARG RUST_NIGHTLY=2022-09-12
+# - src/lib/snarkyjs/src/bindings/kimchi/wasm/rust-toolchain.toml
+ARG RUST_NIGHTLY=2023-09-01
 # wasm-pack version
-ARG WASM_PACK_VERSION=v0.10.3
+ARG WASM_PACK_VERSION=v0.12.1
 
 # Rocksdb commit tag/branch to clone
 ARG ROCKSDB_VERSION=v5.18.4

diff --git a/helm/cron_jobs/README.md b/helm/cron_jobs/README.md
@@ -0,0 +1,18 @@
+Replayer cron jobs
+==================
+
+There are replayer cron jobs for Mainnet, Devnet, and Berkeley. These
+jobs are run daily, to replay a day's worth of transactions.
+
+Each cron job downloads the most recent archive dump corresponding to
+a network, and loads the data into PostgreSQL. That results in an
+archive database. The most recent replayer checkpoint file is
+downloaded, which provides the starting point for the replayer. When
+the replayer runs, it creates new checkpoint files every 50
+blocks. When the replayer finishes, it uploads the most recent
+checkpoint file, so it can be used in the following day's run. If
+there are any errors, the replayer logs are also uploaded.
+
+There is a separate checkpoint file bucket for each network.  Both the
+checkpoint files and error files for a given network are uploaded to
+the same bucket.
diff --git a/nix/rust.nix b/nix/rust.nix
@@ -8,10 +8,16 @@ let
       # override stdenv.targetPlatform here, if neccesary
     };
   toolchainHashes = {
-    "1.67.0" = "sha256-riZUc+R9V35c/9e8KJUE+8pzpXyl0lRXt3ZkKlxoY0g=";
-    "nightly-2023-02-05" =
-      "sha256-MM8fdvveBEWzpwjH7u6C0F7qSWGPIMpfZWLgVxSqtxY=";
-    # copy this line with the correct toolchain name
+    "1.72" = "sha256-dxE7lmCFWlq0nl/wKcmYvpP9zqQbBitAQgZ1zx9Ooik=";
+    "nightly-2023-09-01" = "sha256-nfYc8EgbYl75yIIHmEEmpux4ZpwaIyuC+g6Hf4y1Hyk=";
+    # copy the placeholder line with the correct toolchain name when adding a new toolchain
+    # That is,
+    # 1. Put the correct version name;
+    #
+    # 2. Put the hash you get in line "got" from the error you obtain, which looks like
+    #    error: hash mismatch in fixed-output derivation '/nix/store/XXXXX'
+    #          specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+    #             got:    sha256-Q9UgzzvxLi4x9aWUJTn+/5EXekC98ODRU1TwhUs9RnY=
     "placeholder" = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
   };
   # rust-toolchain.toml -> { rustc, cargo, rust-analyzer, ... }
@@ -150,8 +156,9 @@ in
 
         checkInputs = [ final.nodejs ];
 
-        # other tests require it to be ran in the wasm-bindgen monorepo
-        cargoTestFlags = [ "--test=interface-types" ];
+        # other tests, like --test=wasm-bindgen, require it to be ran in the
+        # wasm-bindgen monorepo
+        cargoTestFlags = [ "--test=interface-types --test=reference" ];
       };
     in
     rustPlatform.buildRustPackage {
@@ -200,4 +207,3 @@ in
     cargoLock.lockFile = ../src/app/trace-tool/Cargo.lock;
   };
 }
-
diff --git a/src/app/archive_blocks/README.md b/src/app/archive_blocks/README.md
@@ -0,0 +1,20 @@
+archive_blocks
+==============
+
+The `archive_blocks` app adds blocks in either "precomputed" or
+"extensional" format to the archive database.
+
+Precomputed blocks are stored in the bucket `mina_network_block_data`
+on Google Cloud Storage. Blocks are named NETWORK-HEIGHT-STATEHASH.json.
+Example: mainnet-100000-3NKLvMCimUjX1zjjiC3XPMT34D1bVQGzkKW58XDwFJgQ5wDQ9Tki.json.
+
+Extensional blocks are extracted from other archive databases using
+the `extract_blocks` app.
+
+As many blocks as are available can be added at a time, but all blocks
+must be in the same format.
+
+Except for blocks from the original mainnet, both precomputed and
+extensional blocks have a version in their JSON representation. That
+version must match the corresponding OCaml type in the code when this
+app was built.
diff --git a/src/app/archive_blocks/archive_blocks.ml b/src/app/archive_blocks/archive_blocks.ml
@@ -1,4 +1,4 @@
-(* archive_blocks.ml -- archive precomputed or extensional blocks to Postgresql *)
+(* archive_blocks.ml *)
 
 open Core_kernel
 open Async

diff --git a/src/app/extract_blocks/README.md b/src/app/extract_blocks/README.md
@@ -0,0 +1,12 @@
+extract_blocks
+==============
+
+The `extract_blocks` app pulls out individual blocks from an archive
+database in "extensional" format. Such blocks can be added to other
+archive databases using the `archive_blocks` app.
+
+Blocks are extracted into files with name <state-hash>.json.
+
+The app offers the choice to extract all canonical blocks, or a
+subchain specified with starting state hash, or a subchain specified
+with starting and ending state hashes.
diff --git a/src/app/extract_blocks/extract_blocks.ml b/src/app/extract_blocks/extract_blocks.ml
@@ -1,4 +1,5 @@
 (* extract_blocks.ml -- dump extensional blocks from archive db *)
+
 [@@@coverage exclude_file]
 
 open Core_kernel

diff --git a/src/app/missing_blocks_auditor/README.md b/src/app/missing_blocks_auditor/README.md
@@ -0,0 +1,10 @@
+missing_blocks_auditor
+======================
+
+The `missing_blocks_auditor` app looks for blocks without parent
+blocks in an archive database.
+
+The app also looks for blocks marked as pending that are lower (have a
+lesser height) than the highest (most recent) canonical block. There
+can be such blocks if blocks are added when there are missing blocks
+in the database.
diff --git a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml
@@ -1,4 +1,4 @@
-(* missing_blocks_auditor.ml -- report missing blocks from an archive db *)
+(* missing_blocks_auditor.ml *)
 
 open Core_kernel
 open Async

diff --git a/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml b/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml
@@ -11,4 +11,4 @@
 # 4. figure out the hashes of the (now obsolete) docker images used in CI rules that are failing, grep for these hashes and replace them with the new hashes
 
 [toolchain]
-channel = "1.67.0"
+channel = "1.72"
diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/arkworks/bigint_256.rs b/src/lib/crypto/kimchi_bindings/stubs/src/arkworks/bigint_256.rs
@@ -239,12 +239,12 @@ mod tests {
     #[test]
     fn biguint() {
         let x = 10000.to_biguint().unwrap();
-        println!("biguint.to_string: {}", x.to_string());
+        println!("biguint.to_string: {}", x);
         let y = CamlBigInteger256::try_from(x.clone()).unwrap();
         println!("camlbigint.to_string: {}", y.to_string());
         //assert!(&y.to_string() == "10000");
         let x2: BigUint = y.into();
         assert!(x2 == x);
-        println!("biguint.to_string: {}", x2.to_string());
+        println!("biguint.to_string: {}", x2);
     }
 }
diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems
diff --git a/src/lib/pickles/common.mli b/src/lib/pickles/common.mli
@@ -36,22 +36,22 @@ val bits_to_bytes : bool list -> string
 val finite_exn : 'a Kimchi_types.or_infinity -> 'a * 'a
 
 val ft_comm :
-     add:('a -> 'a -> 'a)
-  -> scale:('a -> 'b -> 'a)
-  -> endoscale:('a -> 'c -> 'a)
-  -> negate:('a -> 'a)
-  -> verification_key:'a Pickles_types.Plonk_verification_key_evals.t
+     add:('comm -> 'comm -> 'comm)
+  -> scale:('comm -> 'scalar -> 'comm)
+  -> endoscale:('comm -> 'c -> 'comm)
+  -> negate:('comm -> 'comm)
+  -> verification_key:'comm Pickles_types.Plonk_verification_key_evals.t
   -> alpha:'c
   -> plonk:
        ( 'd
        , 'e
-       , 'b
+       , 'scalar
        , 'g
        , 'f
        , 'bool )
        Import.Types.Wrap.Proof_state.Deferred_values.Plonk.In_circuit.t
-  -> t_comm:'a array
-  -> 'a
+  -> t_comm:'comm array
+  -> 'comm
 
 val dlog_pcs_batch :
      'total Pickles_types.Nat.t

diff --git a/src/lib/snarkyjs b/src/lib/snarkyjs