From ac7ee9bf8122ec33b00267b447ccc18ebc0f46f3 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 12:28:41 +0200 Subject: [PATCH 01/22] feat: add scaffolding for running zombienet on Linux --- .envrc | 8 ++++ .gitignore | 2 + Cargo.toml | 2 +- DEVELOPMENT.md | 24 +++++++++++ flake.lock | 85 ++++++++++++++++++++++++++++++++++++++ flake.nix | 55 ++++++++++++++++++++++++ rust-toolchain.toml | 14 +++++++ scripts/local-testnet.toml | 22 ++++++++++ 8 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 .envrc create mode 100644 DEVELOPMENT.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 rust-toolchain.toml create mode 100644 scripts/local-testnet.toml diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..7b94f9cf0 --- /dev/null +++ b/.envrc @@ -0,0 +1,8 @@ +#!/bin/bash + +if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" +fi + +watch_file rust-toolchain.toml +use flake \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8ef4cbf13..fd5bfbd53 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ target/ # MSVC Windows builds of rustc generate these, which store debugging information *.pdb + +.direnv \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index ddd0aee99..1c7a01905 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,7 @@ cumulus-pallet-parachain-system = { git = "https://github.com/paritytech/polkado cumulus-pallet-session-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default-features = false } cumulus-pallet-xcm = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default-features = false } cumulus-pallet-xcmp-queue = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default-features = false } -cumulus-primitives-core = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default_features = false } +cumulus-primitives-core = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default-features = false } cumulus-primitives-parachain-inherent = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" } cumulus-primitives-storage-weight-reclaim = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default-features = false } cumulus-primitives-utility = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0", default-features = false } diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..b5884d5a6 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,24 @@ +# Development environment for Polka Storage Node + +- recommended OS: Linux. + +## Setup + +### Requirements +- [nix](https://nixos.org/download/) with [flakes](https://nixos.wiki/wiki/flakes) enabled (`echo 'experimental-features = nic-command flakes' >> ~/.config/nix/nix.conf`) + - reasoning: every developer has the same version of development tools (rust, protoc, zombienet), directed by `flake.nix`. + - how it works? fasterthanli.me has [a whole series on it](https://fasterthanli.me/series/building-a-rust-service-with-nix/part-10). +- [direnv](https://direnv.net/) with a [shell hook](https://direnv.net/docs/hook.html) + - installation: `nix profile install nixpkgs#direnv` + - *VS Code only* [direnv extension](https://marketplace.visualstudio.com/items?itemName=mkhl.direnv) (uses the same tooling as rust-toolchain.toml defined). + - reasoning: when you enter a directory it uses everything defined in `.envrc`, e.g. environment variables, `nix`, secrets. + +## Usage + +0. [Optional, if you don't have `direnv`] `nix develop` +1. `cargo build --release` +2. `zombienet -p native spawn scripts/local-testnet.toml` + +## Maintenance + +- Updating nix flakes (`flake.lock` file): `nix flake update`. \ No newline at end of file diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..d63e49ef6 --- /dev/null +++ b/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1715534503, + "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2057814051972fa1453ddfb0d98badbea9b83c06", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1715739484, + "narHash": "sha256-5zlSuCM54jH6tXi8OILZ7opT+lBYUkGU9eOMEvJh9HU=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "3d27c65641a61d36f1c7616d6150524cd9a2a5f7", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..8dadda81f --- /dev/null +++ b/flake.nix @@ -0,0 +1,55 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs = { + nixpkgs.follows = "nixpkgs"; + flake-utils.follows = "flake-utils"; + }; + }; + }; + outputs = { self, nixpkgs, flake-utils, rust-overlay }: + flake-utils.lib.eachDefaultSystem (system: + let + overlays = [ (import rust-overlay) ]; + pkgs = import nixpkgs { + inherit system overlays; + }; + # This is not pretty. I couldn't make it work with nix flake `github:paritytech/zombienet`. + zombienet = pkgs.stdenv.mkDerivation rec { + name = "zombienet"; + pname = name; + src = builtins.fetchurl { + url = "https://github.com/paritytech/zombienet/releases/download/v1.3.103/zombienet-linux-x64"; + sha256 = "sha256:1qlsvd3h4szcgzj2990qgig6vcrg5grzfxkzhdhg93378fmlz9lx"; + }; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin + cp $src $out/bin/${name} + chmod +x $out/bin/${name} + ''; + }; + rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + # build-time + nativeBuildInputs = with pkgs; [ pkg-config rustToolchain ]; + # runtime + buildInputs = with pkgs; [ clang polkadot zombienet ] ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + in + with pkgs; + { + devShells.default = mkShell { + inherit buildInputs nativeBuildInputs; + + LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib"; + PROTOC = "${protobuf}/bin/protoc"; + RUST_SRC_PATH = "${rustToolchain}/lib/rustlib/src/rust/library/"; + ROCKSDB_LIB_DIR = "${rocksdb}/lib"; + }; + } + ); +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..7033a1a41 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,14 @@ +[toolchain] +channel = "1.77.0" +components = [ + "cargo", + "clippy", + "rust-analyzer", + "rust-src", + "rust-std", + "rustc", + "rustc-dev", + "rustfmt", +] +targets = ["wasm32-unknown-unknown"] +profile = "minimal" \ No newline at end of file diff --git a/scripts/local-testnet.toml b/scripts/local-testnet.toml new file mode 100644 index 000000000..858e2380d --- /dev/null +++ b/scripts/local-testnet.toml @@ -0,0 +1,22 @@ +[relaychain] +default_command = "polkadot" +chain = "rococo-local" + + [[relaychain.nodes]] + name = "alice" + validator = true + + [[relaychain.nodes]] + name = "bob" + validator = true + +[[parachains]] +id = 2000 +cumulus_based = true +chain = "polkadot-storage-rococo-local" + + # run charlie as parachain collator + [[parachains.collators]] + name = "charlie" + validator = true + command = "target/release/polka-storage-node" \ No newline at end of file From 6f1aec26eabc174d47133315f37c341fcdfedcdc Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 12:37:46 +0200 Subject: [PATCH 02/22] fix: make zombienet not crash --- scripts/local-testnet.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/local-testnet.toml b/scripts/local-testnet.toml index 858e2380d..1b0251a21 100644 --- a/scripts/local-testnet.toml +++ b/scripts/local-testnet.toml @@ -13,7 +13,6 @@ chain = "rococo-local" [[parachains]] id = 2000 cumulus_based = true -chain = "polkadot-storage-rococo-local" # run charlie as parachain collator [[parachains.collators]] From 5c84aba133c02eea7a7ba1198dd0717eb020d6d0 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 12:48:43 +0200 Subject: [PATCH 03/22] doc: add docs about the version verification --- DEVELOPMENT.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b5884d5a6..7a2eeb2bc 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -16,8 +16,16 @@ ## Usage 0. [Optional, if you don't have `direnv`] `nix develop` -1. `cargo build --release` -2. `zombienet -p native spawn scripts/local-testnet.toml` +1. Verify: +``` +$ polkadot --version +polkadot 1.11.0-0bb6249 + +$ cargo --version +cargo 1.77.0 (3fe68eabf 2024-02-29) +``` +2. `cargo build --release` +3. `zombienet -p native spawn scripts/local-testnet.toml` ## Maintenance From c63c08b4a858addc9b16a1c0960c79fc16107a5c Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 13:33:45 +0200 Subject: [PATCH 04/22] build: add Justfile and support for macos zombienet --- Justfile | 2 ++ flake.nix | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 Justfile diff --git a/Justfile b/Justfile new file mode 100644 index 000000000..80438144d --- /dev/null +++ b/Justfile @@ -0,0 +1,2 @@ +testnet: + zombienet -p native spawn scripts/local-testnet.toml \ No newline at end of file diff --git a/flake.nix b/flake.nix index 8dadda81f..3a547df00 100644 --- a/flake.nix +++ b/flake.nix @@ -21,10 +21,18 @@ zombienet = pkgs.stdenv.mkDerivation rec { name = "zombienet"; pname = name; - src = builtins.fetchurl { - url = "https://github.com/paritytech/zombienet/releases/download/v1.3.103/zombienet-linux-x64"; - sha256 = "sha256:1qlsvd3h4szcgzj2990qgig6vcrg5grzfxkzhdhg93378fmlz9lx"; - }; + src = + if (system == "x86_64-linux") then + builtins.fetchurl + { + url = "https://github.com/paritytech/zombienet/releases/download/v1.3.103/zombienet-linux-x64"; + sha256 = "sha256:1qlsvd3h4szcgzj2990qgig6vcrg5grzfxkzhdhg93378fmlz9lx"; + } else + builtins.fetchurl { + url = "https://github.com/paritytech/zombienet/releases/download/v1.3.103/zombienet-macos"; + sha256 = "sha256:19rl1nzlzdf00kbs4s9k2m516basli3z0kyanb3xry4yqpxnr28z"; + }; + phases = [ "installPhase" ]; installPhase = '' mkdir -p $out/bin @@ -36,7 +44,7 @@ # build-time nativeBuildInputs = with pkgs; [ pkg-config rustToolchain ]; # runtime - buildInputs = with pkgs; [ clang polkadot zombienet ] ++ lib.optionals stdenv.isDarwin [ + buildInputs = with pkgs; [ clang just polkadot zombienet ] ++ lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.Security ]; in From 13db823bd316af05362514920cb93402fb3cde8a Mon Sep 17 00:00:00 2001 From: "konrad.stepniak" Date: Wed, 15 May 2024 15:03:29 +0200 Subject: [PATCH 05/22] docs: fix nix command --- DEVELOPMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7a2eeb2bc..378cdbf48 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -5,7 +5,7 @@ ## Setup ### Requirements -- [nix](https://nixos.org/download/) with [flakes](https://nixos.wiki/wiki/flakes) enabled (`echo 'experimental-features = nic-command flakes' >> ~/.config/nix/nix.conf`) +- [nix](https://nixos.org/download/) with [flakes](https://nixos.wiki/wiki/flakes) enabled (`echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf`) - reasoning: every developer has the same version of development tools (rust, protoc, zombienet), directed by `flake.nix`. - how it works? fasterthanli.me has [a whole series on it](https://fasterthanli.me/series/building-a-rust-service-with-nix/part-10). - [direnv](https://direnv.net/) with a [shell hook](https://direnv.net/docs/hook.html) From 65ac2e63720a44808647cd3156e91553240130ea Mon Sep 17 00:00:00 2001 From: "konrad.stepniak" Date: Wed, 15 May 2024 15:25:38 +0200 Subject: [PATCH 06/22] doc: add vs code extension for nix --- DEVELOPMENT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 378cdbf48..2238d6035 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -8,6 +8,7 @@ - [nix](https://nixos.org/download/) with [flakes](https://nixos.wiki/wiki/flakes) enabled (`echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf`) - reasoning: every developer has the same version of development tools (rust, protoc, zombienet), directed by `flake.nix`. - how it works? fasterthanli.me has [a whole series on it](https://fasterthanli.me/series/building-a-rust-service-with-nix/part-10). + - optional: [vscode extension](https://marketplace.visualstudio.com/items?itemName=jnoortheen.nix-ide) - [direnv](https://direnv.net/) with a [shell hook](https://direnv.net/docs/hook.html) - installation: `nix profile install nixpkgs#direnv` - *VS Code only* [direnv extension](https://marketplace.visualstudio.com/items?itemName=mkhl.direnv) (uses the same tooling as rust-toolchain.toml defined). From 8ea0334d60ec71f2a77c414290b99361e6ad42b1 Mon Sep 17 00:00:00 2001 From: "konrad.stepniak" Date: Wed, 15 May 2024 15:26:15 +0200 Subject: [PATCH 07/22] build: add macos arm build requirements --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 3a547df00..09c836041 100644 --- a/flake.nix +++ b/flake.nix @@ -46,6 +46,8 @@ # runtime buildInputs = with pkgs; [ clang just polkadot zombienet ] ++ lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.Security + darwin.apple_sdk.frameworks.CoreServices + darwin.apple_sdk.frameworks.SystemConfiguration ]; in with pkgs; From b04859f51cd87315ea260a3387d9d49469ce4666 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 15:58:58 +0200 Subject: [PATCH 08/22] build: add MacOS ARM support --- DEVELOPMENT.md | 19 +++++++------ Justfile | 12 +++++++- flake.lock | 76 +++++++++++++++++++++++++++++++++++++++++++++++++- flake.nix | 40 +++++++------------------- 4 files changed, 106 insertions(+), 41 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 2238d6035..faf149f14 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,18 +1,15 @@ # Development environment for Polka Storage Node -- recommended OS: Linux. - ## Setup ### Requirements - [nix](https://nixos.org/download/) with [flakes](https://nixos.wiki/wiki/flakes) enabled (`echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf`) - - reasoning: every developer has the same version of development tools (rust, protoc, zombienet), directed by `flake.nix`. + - reasoning: every developer has the same version of development tools (rust, protoc, zombienet), directed by [flake.nix](./flake.nix)`. - how it works? fasterthanli.me has [a whole series on it](https://fasterthanli.me/series/building-a-rust-service-with-nix/part-10). - - optional: [vscode extension](https://marketplace.visualstudio.com/items?itemName=jnoortheen.nix-ide) + - optional: [vscode extension for Nix](https://marketplace.visualstudio.com/items?itemName=jnoortheen.nix-ide) - [direnv](https://direnv.net/) with a [shell hook](https://direnv.net/docs/hook.html) - - installation: `nix profile install nixpkgs#direnv` - *VS Code only* [direnv extension](https://marketplace.visualstudio.com/items?itemName=mkhl.direnv) (uses the same tooling as rust-toolchain.toml defined). - - reasoning: when you enter a directory it uses everything defined in `.envrc`, e.g. environment variables, `nix`, secrets. + - reasoning: when you enter a directory it uses everything defined in [.envrc](./.envrc), e.g. environment variables, `nix`, secrets. ## Usage @@ -24,10 +21,14 @@ polkadot 1.11.0-0bb6249 $ cargo --version cargo 1.77.0 (3fe68eabf 2024-02-29) + +$ zombienet version +1.3.103 ``` -2. `cargo build --release` -3. `zombienet -p native spawn scripts/local-testnet.toml` +2. `just testnet` +3. Get into a direct link of `charlie` node to access parachain interface and see block production. + - testnet is defined via [zombienet configuration](https://paritytech.github.io/zombienet/guide.html) in [local-testnet.toml](./scripts//local-testnet.toml) ## Maintenance -- Updating nix flakes (`flake.lock` file): `nix flake update`. \ No newline at end of file +- Updating nix flakes (`flake.lock` file has frozen state of package): `nix flake update`. \ No newline at end of file diff --git a/Justfile b/Justfile index 80438144d..a10f12d79 100644 --- a/Justfile +++ b/Justfile @@ -1,2 +1,12 @@ -testnet: +alias b := build +alias r := release +alias t := testnet + +build: + cargo build + +release: + cargo build --release + +testnet: release zombienet -p native spawn scripts/local-testnet.toml \ No newline at end of file diff --git a/flake.lock b/flake.lock index d63e49ef6..0bb5ce4d4 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,23 @@ { "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1693611461, + "narHash": "sha256-aPODl8vAgGQ0ZYFIRisxYG5MOGSkIczvu2Cd8Gb9+1Y=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "7f53fdb7bdc5bb237da7fefef12d099e4fd611ca", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -34,11 +52,45 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1693471703, + "narHash": "sha256-0l03ZBL8P1P6z8MaSDS/MvuU8E75rVxe5eE1N6gxeTo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3e52e76b70d5508f3cec70b882a29199f4d1ee85", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "process-compose": { + "locked": { + "lastModified": 1693927910, + "narHash": "sha256-qPKHnWWzHS2bAi/SsFePQkGFeC2E1jklUjEidfQwYLc=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "5494afa0b6a7bc4ccf82ef1c36fe1fcdb4217255", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, "root": { "inputs": { "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay" + "rust-overlay": "rust-overlay", + "zombienet": "zombienet" } }, "rust-overlay": { @@ -78,6 +130,28 @@ "repo": "default", "type": "github" } + }, + "zombienet": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ], + "process-compose": "process-compose" + }, + "locked": { + "lastModified": 1715752734, + "narHash": "sha256-YPoOQa+SXG3KPDoP6bXdykmDhmnooFtlyEwDRpPJjog=", + "owner": "paritytech", + "repo": "zombienet", + "rev": "875a273502d81c29985eff3f78521650f26c2c28", + "type": "github" + }, + "original": { + "owner": "paritytech", + "repo": "zombienet", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 09c836041..a92ff2de8 100644 --- a/flake.nix +++ b/flake.nix @@ -9,42 +9,22 @@ flake-utils.follows = "flake-utils"; }; }; + zombienet = { + url = "github:paritytech/zombienet"; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; + }; }; - outputs = { self, nixpkgs, flake-utils, rust-overlay }: + outputs = { self, nixpkgs, flake-utils, rust-overlay, zombienet }: flake-utils.lib.eachDefaultSystem (system: let - overlays = [ (import rust-overlay) ]; + overlays = [ (import rust-overlay) zombienet.overlays.default ]; pkgs = import nixpkgs { inherit system overlays; }; - # This is not pretty. I couldn't make it work with nix flake `github:paritytech/zombienet`. - zombienet = pkgs.stdenv.mkDerivation rec { - name = "zombienet"; - pname = name; - src = - if (system == "x86_64-linux") then - builtins.fetchurl - { - url = "https://github.com/paritytech/zombienet/releases/download/v1.3.103/zombienet-linux-x64"; - sha256 = "sha256:1qlsvd3h4szcgzj2990qgig6vcrg5grzfxkzhdhg93378fmlz9lx"; - } else - builtins.fetchurl { - url = "https://github.com/paritytech/zombienet/releases/download/v1.3.103/zombienet-macos"; - sha256 = "sha256:19rl1nzlzdf00kbs4s9k2m516basli3z0kyanb3xry4yqpxnr28z"; - }; - - phases = [ "installPhase" ]; - installPhase = '' - mkdir -p $out/bin - cp $src $out/bin/${name} - chmod +x $out/bin/${name} - ''; - }; rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - # build-time - nativeBuildInputs = with pkgs; [ pkg-config rustToolchain ]; - # runtime - buildInputs = with pkgs; [ clang just polkadot zombienet ] ++ lib.optionals stdenv.isDarwin [ + buildInputs = with pkgs; [ clang pkg-config rustToolchain just polkadot pkgs.zombienet.default ] ++ lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.CoreServices darwin.apple_sdk.frameworks.SystemConfiguration @@ -53,7 +33,7 @@ with pkgs; { devShells.default = mkShell { - inherit buildInputs nativeBuildInputs; + inherit buildInputs; LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib"; PROTOC = "${protobuf}/bin/protoc"; From 03f25b826e4ef42c8eda561fb6d3937490ff6921 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:10:02 +0200 Subject: [PATCH 09/22] docs: add disclaimer about how long it takes --- DEVELOPMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index faf149f14..c22f0268f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -26,7 +26,7 @@ $ zombienet version 1.3.103 ``` 2. `just testnet` -3. Get into a direct link of `charlie` node to access parachain interface and see block production. +3. Get into a direct link of `charlie` node to access parachain interface and see block production (it takes ~30s). - testnet is defined via [zombienet configuration](https://paritytech.github.io/zombienet/guide.html) in [local-testnet.toml](./scripts//local-testnet.toml) ## Maintenance From 62a5c8d3ea1705545aec5db20de59292a3189925 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:11:46 +0200 Subject: [PATCH 10/22] chore: remove deprecated polkadot-launch --- polkadot-launch/config.json | 38 ------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 polkadot-launch/config.json diff --git a/polkadot-launch/config.json b/polkadot-launch/config.json deleted file mode 100644 index 98bc970b2..000000000 --- a/polkadot-launch/config.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "relaychain": { - "bin": "../../polkadot/target/release/polkadot", - "chain": "rococo-local", - "nodes": [ - { - "name": "alice", - "wsPort": 9944, - "port": 30444 - }, - { - "name": "bob", - "wsPort": 9955, - "port": 30555 - } - ] - }, - "parachains": [ - { - "bin": "../target/release/polkadot-parachain", - "id": "200", - "balance": "1000000000000000000000", - "nodes": [ - { - "wsPort": 9988, - "name": "alice", - "port": 31200, - "flags": [ - "--force-authoring", - "--", - "--execution=wasm" - ] - } - ] - } - ], - "types": {} -} From dadf4dfef3f232e6613a08dfa9e25ffc8817384e Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:13:47 +0200 Subject: [PATCH 11/22] style: format flake.nix --- flake.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a92ff2de8..21bfe4ce4 100644 --- a/flake.nix +++ b/flake.nix @@ -24,7 +24,15 @@ inherit system overlays; }; rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - buildInputs = with pkgs; [ clang pkg-config rustToolchain just polkadot pkgs.zombienet.default ] ++ lib.optionals stdenv.isDarwin [ + buildInputs = with pkgs; [ + clang + pkg-config + rustToolchain + just + polkadot + # Due to zombienet's flake.nix, needs to be prefixed with pkg.zombienet + pkgs.zombienet.default + ] ++ lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.CoreServices darwin.apple_sdk.frameworks.SystemConfiguration From c73bf7e0d92c19429a2441a382ec70643aec3c27 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:24:15 +0200 Subject: [PATCH 12/22] fix: taplo exlusion/inclusion policy --- .taplo.toml | 4 +++- rust-toolchain.toml | 13 ++----------- scripts/local-testnet.toml | 26 +++++++++++++------------- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/.taplo.toml b/.taplo.toml index a5e91e369..920bde191 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -1,6 +1,8 @@ -[[rule]] +exclude = [".direnv/*"] include = ["**/*.toml"] +[[rule]] + [rule.formatting] column_width = 150 reorder_arrays = true diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7033a1a41..47df6fabc 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,14 +1,5 @@ [toolchain] channel = "1.77.0" -components = [ - "cargo", - "clippy", - "rust-analyzer", - "rust-src", - "rust-std", - "rustc", - "rustc-dev", - "rustfmt", -] +components = ["cargo", "clippy", "rust-analyzer", "rust-src", "rust-std", "rustc", "rustc-dev", "rustfmt"] +profile = "minimal" targets = ["wasm32-unknown-unknown"] -profile = "minimal" \ No newline at end of file diff --git a/scripts/local-testnet.toml b/scripts/local-testnet.toml index 1b0251a21..8c0f821d4 100644 --- a/scripts/local-testnet.toml +++ b/scripts/local-testnet.toml @@ -1,21 +1,21 @@ [relaychain] -default_command = "polkadot" chain = "rococo-local" +default_command = "polkadot" - [[relaychain.nodes]] - name = "alice" - validator = true +[[relaychain.nodes]] +name = "alice" +validator = true - [[relaychain.nodes]] - name = "bob" - validator = true +[[relaychain.nodes]] +name = "bob" +validator = true [[parachains]] -id = 2000 cumulus_based = true +id = 2000 - # run charlie as parachain collator - [[parachains.collators]] - name = "charlie" - validator = true - command = "target/release/polka-storage-node" \ No newline at end of file +# run charlie as parachain collator +[[parachains.collators]] +command = "target/release/polka-storage-node" +name = "charlie" +validator = true From 80463bbfdec0af3699747bf83267f92ec8629da0 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:27:18 +0200 Subject: [PATCH 13/22] build: add taplo to build process --- Justfile | 7 +++++-- flake.nix | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index a10f12d79..91873c202 100644 --- a/Justfile +++ b/Justfile @@ -2,10 +2,13 @@ alias b := build alias r := release alias t := testnet -build: +lint: + taplo lint && taplo fmt --check + +build: lint cargo build -release: +release: lint cargo build --release testnet: release diff --git a/flake.nix b/flake.nix index 21bfe4ce4..280254ecf 100644 --- a/flake.nix +++ b/flake.nix @@ -29,6 +29,7 @@ pkg-config rustToolchain just + taplo polkadot # Due to zombienet's flake.nix, needs to be prefixed with pkg.zombienet pkgs.zombienet.default From f1f574ca24572233b1f0cec056d5a9f73b7e7e46 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:35:34 +0200 Subject: [PATCH 14/22] style: fix newlines at the end of files --- .envrc | 2 +- .gitignore | 3 ++- .taplo.toml | 2 +- Justfile | 5 +++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.envrc b/.envrc index 7b94f9cf0..22adc2d85 100644 --- a/.envrc +++ b/.envrc @@ -5,4 +5,4 @@ if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then fi watch_file rust-toolchain.toml -use flake \ No newline at end of file +use flake diff --git a/.gitignore b/.gitignore index fd5bfbd53..378be3110 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ target/ # MSVC Windows builds of rustc generate these, which store debugging information *.pdb -.direnv \ No newline at end of file +# reproducible local environment +.direnv diff --git a/.taplo.toml b/.taplo.toml index 920bde191..b44c3edd7 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -1,7 +1,7 @@ exclude = [".direnv/*"] -include = ["**/*.toml"] [[rule]] +include = ["**/*.toml"] [rule.formatting] column_width = 150 diff --git a/Justfile b/Justfile index 91873c202..80dfbaa97 100644 --- a/Justfile +++ b/Justfile @@ -4,7 +4,7 @@ alias t := testnet lint: taplo lint && taplo fmt --check - + build: lint cargo build @@ -12,4 +12,5 @@ release: lint cargo build --release testnet: release - zombienet -p native spawn scripts/local-testnet.toml \ No newline at end of file + zombienet -p native spawn scripts/local-testnet.toml + \ No newline at end of file From 35925ee09811ac4ce8722c498fbb75d11e93bfe8 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:38:42 +0200 Subject: [PATCH 15/22] docs: add nix-collect-garbage to the docs --- DEVELOPMENT.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c22f0268f..f09c3399e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -31,4 +31,5 @@ $ zombienet version ## Maintenance -- Updating nix flakes (`flake.lock` file has frozen state of package): `nix flake update`. \ No newline at end of file +- Updating nix flakes (`flake.lock` file has frozen state of package): `nix flake update`. +- Running out of the disk space? `nix-collect-garbage`. \ No newline at end of file From 227f6f4b380003ba1f872534a0ea66fedf1853c4 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 15 May 2024 16:44:09 +0200 Subject: [PATCH 16/22] style: add formatting to Justfile --- Justfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Justfile b/Justfile index 80dfbaa97..5197fb9c0 100644 --- a/Justfile +++ b/Justfile @@ -13,4 +13,3 @@ release: lint testnet: release zombienet -p native spawn scripts/local-testnet.toml - \ No newline at end of file From 1a5a3c892de40af3ae6e8337692ac5d2765168ad Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 16 May 2024 11:09:42 +0200 Subject: [PATCH 17/22] docs: add explanation and make direnv required --- DEVELOPMENT.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f09c3399e..96d68fb0a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -11,9 +11,16 @@ - *VS Code only* [direnv extension](https://marketplace.visualstudio.com/items?itemName=mkhl.direnv) (uses the same tooling as rust-toolchain.toml defined). - reasoning: when you enter a directory it uses everything defined in [.envrc](./.envrc), e.g. environment variables, `nix`, secrets. +## How it works? +Nix is a package manager, which sneakily downloads all of the dependencies and updates PATH when you launch it with `nix develop`. +You end up having all of the required dependencies in a configured shell (so you don't have to install a specific cargo version, just, polkadot on your own). +`nix develop` needs to be used in the workspace root, as it depends on [flake.nix](./flake.nix) file. +`direnv` is a shell hook, which configures your shell based on the [.envrc](./.envrc) file. +In our case it just launches `nix develop` shell for you and when you exit the folder, it disables it. + + ## Usage -0. [Optional, if you don't have `direnv`] `nix develop` 1. Verify: ``` $ polkadot --version From 9c28d7950ed3108aae1444d27b7d03df910a6613 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 16 May 2024 11:10:16 +0200 Subject: [PATCH 18/22] build: change scripts to zombienet --- Justfile | 2 +- {scripts => zombienet}/local-testnet.toml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {scripts => zombienet}/local-testnet.toml (100%) diff --git a/Justfile b/Justfile index 5197fb9c0..7d1fcf794 100644 --- a/Justfile +++ b/Justfile @@ -12,4 +12,4 @@ release: lint cargo build --release testnet: release - zombienet -p native spawn scripts/local-testnet.toml + zombienet -p native spawn zombienet/local-testnet.toml diff --git a/scripts/local-testnet.toml b/zombienet/local-testnet.toml similarity index 100% rename from scripts/local-testnet.toml rename to zombienet/local-testnet.toml From 1cdf453173dc674e943a3aa63ff941d47ff4ef90 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 16 May 2024 11:14:58 +0200 Subject: [PATCH 19/22] build: exclude target from taplo --- .taplo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.taplo.toml b/.taplo.toml index b44c3edd7..d2128764e 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -1,4 +1,4 @@ -exclude = [".direnv/*"] +exclude = [".direnv/*", "target/*"] [[rule]] include = ["**/*.toml"] From bc5edc395329b1a81efc193e92d2cd471cae95cb Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 16 May 2024 11:16:38 +0200 Subject: [PATCH 20/22] docs: fix charlie's instruction --- DEVELOPMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 96d68fb0a..78744027d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -33,7 +33,7 @@ $ zombienet version 1.3.103 ``` 2. `just testnet` -3. Get into a direct link of `charlie` node to access parachain interface and see block production (it takes ~30s). +3. Click `charlie`'s direct link (should look like `https://polkadot.js.org/apps/?rpc=ws://
#/explorer`). It will take you to the parachain interface where you can monitor block production. - testnet is defined via [zombienet configuration](https://paritytech.github.io/zombienet/guide.html) in [local-testnet.toml](./scripts//local-testnet.toml) ## Maintenance From c6d6a3abb4f96b35cb3ce134104595203c81466f Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 16 May 2024 11:18:59 +0200 Subject: [PATCH 21/22] build: add clippy to Justfile --- Justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Justfile b/Justfile index 7d1fcf794..6693c8b90 100644 --- a/Justfile +++ b/Justfile @@ -3,6 +3,7 @@ alias r := release alias t := testnet lint: + cargo clippy taplo lint && taplo fmt --check build: lint From d5324c629fa265f8de81e56b2780c218bb568619 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 16 May 2024 11:45:13 +0200 Subject: [PATCH 22/22] build: fix clippy command in Justfile --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 6693c8b90..69bdeea3d 100644 --- a/Justfile +++ b/Justfile @@ -3,7 +3,7 @@ alias r := release alias t := testnet lint: - cargo clippy + cargo clippy --locked --no-deps -- -D warnings taplo lint && taplo fmt --check build: lint