diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index f687849ca59..9c06bf956f3 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -4,10 +4,10 @@ -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI { toolchainBase = "codaprotocol/ci-toolchain-base:v3", - minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:563fd7adda282fb3b6765c1811a3566e0fa0560f5d1c5270003483030d82d394", - minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:49891eb46089f937f054afa464ce9868529981b92b30740cce32ef60957a1098", - minaToolchainBookworm = "gcr.io/o1labs-192920/mina-toolchain@sha256:49891eb46089f937f054afa464ce9868529981b92b30740cce32ef60957a1098", - minaToolchain = "gcr.io/o1labs-192920/mina-toolchain@sha256:49891eb46089f937f054afa464ce9868529981b92b30740cce32ef60957a1098", + minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:71173ebccf6af3e24d27262a5071f3dd0bd2c40b9de1c258422fdb9419507d3c", + minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:9c4062e76fcd910ad60d3f1f58e2395f6a5e70f16fbef422442aedb70112ac73", + minaToolchainBookworm = "gcr.io/o1labs-192920/mina-toolchain@sha256:9c4062e76fcd910ad60d3f1f58e2395f6a5e70f16fbef422442aedb70112ac73", + minaToolchain = "gcr.io/o1labs-192920/mina-toolchain@sha256:9c4062e76fcd910ad60d3f1f58e2395f6a5e70f16fbef422442aedb70112ac73", delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:12ffd0a9016819c720687f440c7a46b8815f8d3ad06d306d342ee5f8dd4375f5", elixirToolchain = "elixir:1.10-alpine", nodeToolchain = "node:14.13.1-stretch-slim", diff --git a/dockerfiles/stages/1-build-deps b/dockerfiles/stages/1-build-deps index 8cef1afb96f..5202a830a59 100644 --- a/dockerfiles/stages/1-build-deps +++ b/dockerfiles/stages/1-build-deps @@ -20,11 +20,15 @@ ARG GO_VERSION=1.19.11 ARG GO_CAPNP_VERSION=v3.0.0-alpha.5 # Rust Version passed into rustup-init, can also be "stable", "nightly" or similar -ARG RUST_VERSION=1.63.0 +# This should stay in line with: +# - src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml +# - src/lib/crypto/proof-systems/rust-toolchain.toml +ARG RUST_VERSION=1.72 # Nightly Rust Version used for WebAssembly builds -ARG RUST_NIGHTLY=2022-09-12 +# - src/lib/snarkyjs/src/bindings/kimchi/wasm/rust-toolchain.toml +ARG RUST_NIGHTLY=2023-09-01 # wasm-pack version -ARG WASM_PACK_VERSION=v0.10.3 +ARG WASM_PACK_VERSION=v0.12.1 # Rocksdb commit tag/branch to clone ARG ROCKSDB_VERSION=v5.18.4 diff --git a/helm/cron_jobs/README.md b/helm/cron_jobs/README.md new file mode 100644 index 00000000000..edd63378621 --- /dev/null +++ b/helm/cron_jobs/README.md @@ -0,0 +1,18 @@ +Replayer cron jobs +================== + +There are replayer cron jobs for Mainnet, Devnet, and Berkeley. These +jobs are run daily, to replay a day's worth of transactions. + +Each cron job downloads the most recent archive dump corresponding to +a network, and loads the data into PostgreSQL. That results in an +archive database. The most recent replayer checkpoint file is +downloaded, which provides the starting point for the replayer. When +the replayer runs, it creates new checkpoint files every 50 +blocks. When the replayer finishes, it uploads the most recent +checkpoint file, so it can be used in the following day's run. If +there are any errors, the replayer logs are also uploaded. + +There is a separate checkpoint file bucket for each network. Both the +checkpoint files and error files for a given network are uploaded to +the same bucket. diff --git a/nix/rust.nix b/nix/rust.nix index 51726d84501..4e8f9e24fad 100644 --- a/nix/rust.nix +++ b/nix/rust.nix @@ -8,10 +8,16 @@ let # override stdenv.targetPlatform here, if neccesary }; toolchainHashes = { - "1.67.0" = "sha256-riZUc+R9V35c/9e8KJUE+8pzpXyl0lRXt3ZkKlxoY0g="; - "nightly-2023-02-05" = - "sha256-MM8fdvveBEWzpwjH7u6C0F7qSWGPIMpfZWLgVxSqtxY="; - # copy this line with the correct toolchain name + "1.72" = "sha256-dxE7lmCFWlq0nl/wKcmYvpP9zqQbBitAQgZ1zx9Ooik="; + "nightly-2023-09-01" = "sha256-nfYc8EgbYl75yIIHmEEmpux4ZpwaIyuC+g6Hf4y1Hyk="; + # copy the placeholder line with the correct toolchain name when adding a new toolchain + # That is, + # 1. Put the correct version name; + # + # 2. Put the hash you get in line "got" from the error you obtain, which looks like + # error: hash mismatch in fixed-output derivation '/nix/store/XXXXX' + # specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + # got: sha256-Q9UgzzvxLi4x9aWUJTn+/5EXekC98ODRU1TwhUs9RnY= "placeholder" = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; }; # rust-toolchain.toml -> { rustc, cargo, rust-analyzer, ... } @@ -150,8 +156,9 @@ in checkInputs = [ final.nodejs ]; - # other tests require it to be ran in the wasm-bindgen monorepo - cargoTestFlags = [ "--test=interface-types" ]; + # other tests, like --test=wasm-bindgen, require it to be ran in the + # wasm-bindgen monorepo + cargoTestFlags = [ "--test=interface-types --test=reference" ]; }; in rustPlatform.buildRustPackage { @@ -200,4 +207,3 @@ in cargoLock.lockFile = ../src/app/trace-tool/Cargo.lock; }; } - diff --git a/src/app/archive_blocks/README.md b/src/app/archive_blocks/README.md new file mode 100644 index 00000000000..718e7a48467 --- /dev/null +++ b/src/app/archive_blocks/README.md @@ -0,0 +1,20 @@ +archive_blocks +============== + +The `archive_blocks` app adds blocks in either "precomputed" or +"extensional" format to the archive database. + +Precomputed blocks are stored in the bucket `mina_network_block_data` +on Google Cloud Storage. Blocks are named NETWORK-HEIGHT-STATEHASH.json. +Example: mainnet-100000-3NKLvMCimUjX1zjjiC3XPMT34D1bVQGzkKW58XDwFJgQ5wDQ9Tki.json. + +Extensional blocks are extracted from other archive databases using +the `extract_blocks` app. + +As many blocks as are available can be added at a time, but all blocks +must be in the same format. + +Except for blocks from the original mainnet, both precomputed and +extensional blocks have a version in their JSON representation. That +version must match the corresponding OCaml type in the code when this +app was built. diff --git a/src/app/archive_blocks/archive_blocks.ml b/src/app/archive_blocks/archive_blocks.ml index caa4203939e..7851ca04518 100644 --- a/src/app/archive_blocks/archive_blocks.ml +++ b/src/app/archive_blocks/archive_blocks.ml @@ -1,4 +1,4 @@ -(* archive_blocks.ml -- archive precomputed or extensional blocks to Postgresql *) +(* archive_blocks.ml *) open Core_kernel open Async diff --git a/src/app/extract_blocks/README.md b/src/app/extract_blocks/README.md new file mode 100644 index 00000000000..bbaecdbb29b --- /dev/null +++ b/src/app/extract_blocks/README.md @@ -0,0 +1,12 @@ +extract_blocks +============== + +The `extract_blocks` app pulls out individual blocks from an archive +database in "extensional" format. Such blocks can be added to other +archive databases using the `archive_blocks` app. + +Blocks are extracted into files with name .json. + +The app offers the choice to extract all canonical blocks, or a +subchain specified with starting state hash, or a subchain specified +with starting and ending state hashes. diff --git a/src/app/extract_blocks/extract_blocks.ml b/src/app/extract_blocks/extract_blocks.ml index f4fa0ac2e77..e7f58718cba 100644 --- a/src/app/extract_blocks/extract_blocks.ml +++ b/src/app/extract_blocks/extract_blocks.ml @@ -1,4 +1,5 @@ (* extract_blocks.ml -- dump extensional blocks from archive db *) + [@@@coverage exclude_file] open Core_kernel diff --git a/src/app/missing_blocks_auditor/README.md b/src/app/missing_blocks_auditor/README.md new file mode 100644 index 00000000000..e5db4ee64a3 --- /dev/null +++ b/src/app/missing_blocks_auditor/README.md @@ -0,0 +1,10 @@ +missing_blocks_auditor +====================== + +The `missing_blocks_auditor` app looks for blocks without parent +blocks in an archive database. + +The app also looks for blocks marked as pending that are lower (have a +lesser height) than the highest (most recent) canonical block. There +can be such blocks if blocks are added when there are missing blocks +in the database. diff --git a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml index 5bce995ea28..e6fd538f1b5 100644 --- a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml +++ b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml @@ -1,4 +1,4 @@ -(* missing_blocks_auditor.ml -- report missing blocks from an archive db *) +(* missing_blocks_auditor.ml *) open Core_kernel open Async diff --git a/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml b/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml index 8ecda82d680..5a82cfe5ab4 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml +++ b/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml @@ -11,4 +11,4 @@ # 4. figure out the hashes of the (now obsolete) docker images used in CI rules that are failing, grep for these hashes and replace them with the new hashes [toolchain] -channel = "1.67.0" +channel = "1.72" diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/arkworks/bigint_256.rs b/src/lib/crypto/kimchi_bindings/stubs/src/arkworks/bigint_256.rs index 90570304d3c..e9f54f65219 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/arkworks/bigint_256.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/arkworks/bigint_256.rs @@ -239,12 +239,12 @@ mod tests { #[test] fn biguint() { let x = 10000.to_biguint().unwrap(); - println!("biguint.to_string: {}", x.to_string()); + println!("biguint.to_string: {}", x); let y = CamlBigInteger256::try_from(x.clone()).unwrap(); println!("camlbigint.to_string: {}", y.to_string()); //assert!(&y.to_string() == "10000"); let x2: BigUint = y.into(); assert!(x2 == x); - println!("biguint.to_string: {}", x2.to_string()); + println!("biguint.to_string: {}", x2); } } diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index a2a1c8d3b9a..970c3ef8ef3 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit a2a1c8d3b9aee9e90438e15e6f652bcc30b0432c +Subproject commit 970c3ef8ef3b9a3f5d01aea51ed892ecd53b42b8 diff --git a/src/lib/pickles/common.mli b/src/lib/pickles/common.mli index 52ed6859c0c..1c7833c5ea8 100644 --- a/src/lib/pickles/common.mli +++ b/src/lib/pickles/common.mli @@ -36,22 +36,22 @@ val bits_to_bytes : bool list -> string val finite_exn : 'a Kimchi_types.or_infinity -> 'a * 'a val ft_comm : - add:('a -> 'a -> 'a) - -> scale:('a -> 'b -> 'a) - -> endoscale:('a -> 'c -> 'a) - -> negate:('a -> 'a) - -> verification_key:'a Pickles_types.Plonk_verification_key_evals.t + add:('comm -> 'comm -> 'comm) + -> scale:('comm -> 'scalar -> 'comm) + -> endoscale:('comm -> 'c -> 'comm) + -> negate:('comm -> 'comm) + -> verification_key:'comm Pickles_types.Plonk_verification_key_evals.t -> alpha:'c -> plonk: ( 'd , 'e - , 'b + , 'scalar , 'g , 'f , 'bool ) Import.Types.Wrap.Proof_state.Deferred_values.Plonk.In_circuit.t - -> t_comm:'a array - -> 'a + -> t_comm:'comm array + -> 'comm val dlog_pcs_batch : 'total Pickles_types.Nat.t diff --git a/src/lib/snarkyjs b/src/lib/snarkyjs index 8a5df31be90..a7e2cc77ad5 160000 --- a/src/lib/snarkyjs +++ b/src/lib/snarkyjs @@ -1 +1 @@ -Subproject commit 8a5df31be90e07eb8f5b3f5785e8ad9b9f534503 +Subproject commit a7e2cc77ad586d589629f4781f7d7b5303915142