From 4230a98c8c93bf210850d21a6249e4e51878994a Mon Sep 17 00:00:00 2001 From: Maksim Dimitrov Date: Fri, 28 Jul 2023 10:39:19 +0300 Subject: [PATCH] feat(tests): Add kagome tests. Separate kagome and polkadot tests into subfolders Signed-off-by: Maksim Dimitrov --- ...1-parachains-pvf-compilation-time-bad.toml | 35 +++++++++++++++++++ ...-parachains-pvf-compilation-time-bad.zndsl | 19 ++++++++++ ...002-parachains-pvf-execution-time-bad.toml | 34 ++++++++++++++++++ ...02-parachains-pvf-execution-time-bad.zndsl | 27 ++++++++++++++ ...-parachains-pvf-memory-allocation-bad.toml | 34 ++++++++++++++++++ ...parachains-pvf-memory-allocation-bad.zndsl | 13 +++++++ ...4-parachains-pvf-stack-allocation-bad.toml | 34 ++++++++++++++++++ ...-parachains-pvf-stack-allocation-bad.zndsl | 13 +++++++ ...1-parachains-pvf-compilation-time-bad.toml | 0 ...-parachains-pvf-compilation-time-bad.zndsl | 0 ...-parachains-pvf-compilation-time-good.toml | 0 ...parachains-pvf-compilation-time-good.zndsl | 0 ...002-parachains-pvf-execution-time-bad.toml | 0 ...02-parachains-pvf-execution-time-bad.zndsl | 0 ...02-parachains-pvf-execution-time-good.toml | 0 ...2-parachains-pvf-execution-time-good.zndsl | 0 ...-parachains-pvf-memory-allocation-bad.toml | 0 ...parachains-pvf-memory-allocation-bad.zndsl | 0 ...parachains-pvf-memory-allocation-good.toml | 0 ...arachains-pvf-memory-allocation-good.zndsl | 0 ...4-parachains-pvf-stack-allocation-bad.toml | 0 ...-parachains-pvf-stack-allocation-bad.zndsl | 0 ...-parachains-pvf-stack-allocation-good.toml | 0 ...parachains-pvf-stack-allocation-good.zndsl | 0 24 files changed, 209 insertions(+) create mode 100644 tests/kagome/0001-parachains-pvf-compilation-time-bad.toml create mode 100644 tests/kagome/0001-parachains-pvf-compilation-time-bad.zndsl create mode 100644 tests/kagome/0002-parachains-pvf-execution-time-bad.toml create mode 100644 tests/kagome/0002-parachains-pvf-execution-time-bad.zndsl create mode 100644 tests/kagome/0003-parachains-pvf-memory-allocation-bad.toml create mode 100644 tests/kagome/0003-parachains-pvf-memory-allocation-bad.zndsl create mode 100644 tests/kagome/0004-parachains-pvf-stack-allocation-bad.toml create mode 100644 tests/kagome/0004-parachains-pvf-stack-allocation-bad.zndsl rename tests/{ => polkadot}/0001-parachains-pvf-compilation-time-bad.toml (100%) rename tests/{ => polkadot}/0001-parachains-pvf-compilation-time-bad.zndsl (100%) rename tests/{ => polkadot}/0001-parachains-pvf-compilation-time-good.toml (100%) rename tests/{ => polkadot}/0001-parachains-pvf-compilation-time-good.zndsl (100%) rename tests/{ => polkadot}/0002-parachains-pvf-execution-time-bad.toml (100%) rename tests/{ => polkadot}/0002-parachains-pvf-execution-time-bad.zndsl (100%) rename tests/{ => polkadot}/0002-parachains-pvf-execution-time-good.toml (100%) rename tests/{ => polkadot}/0002-parachains-pvf-execution-time-good.zndsl (100%) rename tests/{ => polkadot}/0003-parachains-pvf-memory-allocation-bad.toml (100%) rename tests/{ => polkadot}/0003-parachains-pvf-memory-allocation-bad.zndsl (100%) rename tests/{ => polkadot}/0003-parachains-pvf-memory-allocation-good.toml (100%) rename tests/{ => polkadot}/0003-parachains-pvf-memory-allocation-good.zndsl (100%) rename tests/{ => polkadot}/0004-parachains-pvf-stack-allocation-bad.toml (100%) rename tests/{ => polkadot}/0004-parachains-pvf-stack-allocation-bad.zndsl (100%) rename tests/{ => polkadot}/0004-parachains-pvf-stack-allocation-good.toml (100%) rename tests/{ => polkadot}/0004-parachains-pvf-stack-allocation-good.zndsl (100%) diff --git a/tests/kagome/0001-parachains-pvf-compilation-time-bad.toml b/tests/kagome/0001-parachains-pvf-compilation-time-bad.toml new file mode 100644 index 0000000..a47207c --- /dev/null +++ b/tests/kagome/0001-parachains-pvf-compilation-time-bad.toml @@ -0,0 +1,35 @@ +[settings] +timeout = 1000 + +[relaychain] +# default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +default_image = "docker.io/parity/polkadot:latest" +default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}" +default_args = [ "-lparachain=debug" ] + +chain = "rococo-local" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + image = "soramitsu/kagome:latest" + command = "$HOME/kagome/build/node/kagome" + args = [ "--alice", "-lparachain=debug,runtime=debug" ] + prometheus_prefix = "kagome" + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +genesis_wasm_path="./wasm/noops.wasm.hex" + + [parachains.collator] + name = "collator01" + image = "docker.io/parity/polkadot-parachain:latest" + command = "polkadot-parachain" + args = ["-lparachain=debug"] diff --git a/tests/kagome/0001-parachains-pvf-compilation-time-bad.zndsl b/tests/kagome/0001-parachains-pvf-compilation-time-bad.zndsl new file mode 100644 index 0000000..94f69a4 --- /dev/null +++ b/tests/kagome/0001-parachains-pvf-compilation-time-bad.zndsl @@ -0,0 +1,19 @@ +Description: PVF compilation time +Network: ./0001-parachains-pvf-compilation-time-kagome-bad.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds + +# Ensure parachains made progress. +alice: log line matches glob "*InvalidCandidate(HardTimeout)*" within 120 seconds +alice: parachain 2000 block height != 1 within 60 seconds + +# Check preparation time is under 10s. +# Check all buckets < 10. +alice: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["0.1", "0.5", "1", "2", "3"] within 20 seconds +# Check all buckets >= 10. +alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["10", "20", "30", "60", "120", "+Inf"] within 20 seconds \ No newline at end of file diff --git a/tests/kagome/0002-parachains-pvf-execution-time-bad.toml b/tests/kagome/0002-parachains-pvf-execution-time-bad.toml new file mode 100644 index 0000000..0862fa1 --- /dev/null +++ b/tests/kagome/0002-parachains-pvf-execution-time-bad.toml @@ -0,0 +1,34 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "docker.io/parity/polkadot:latest" +default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}" +default_args = [ "-lparachain=debug" ] + +chain = "rococo-local" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + image = "soramitsu/kagome:latest" + command = "$HOME/kagome/build/node/kagome" + args = [ "--alice", "-lparachain=debug,runtime=debug" ] + prometheus_prefix = "kagome" + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +genesis_wasm_path="./wasm/infinite-loop.wasm.hex" + + [parachains.collator] + name = "collator01" + image = "docker.io/parity/polkadot-parachain:latest" + command = "polkadot-parachain" + args = ["-lparachain=debug"] diff --git a/tests/kagome/0002-parachains-pvf-execution-time-bad.zndsl b/tests/kagome/0002-parachains-pvf-execution-time-bad.zndsl new file mode 100644 index 0000000..b7d53b5 --- /dev/null +++ b/tests/kagome/0002-parachains-pvf-execution-time-bad.zndsl @@ -0,0 +1,27 @@ +Description: PVF compilation time +Network: ./0002-parachains-pvf-execution-time-kagome-bad.toml +Creds: config + +# Check authority status. +alice: reports kagome_node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds + +# Ensure parachains made progress. +alice: parachain 2000 block height != 1 within 60 seconds + +# Check execution time. +# There are two different timeout conditions: BACKING_EXECUTION_TIMEOUT(2s) and +# APPROVAL_EXECUTION_TIMEOUT(6s). Currently these are not differentiated by metrics +# because the metrics are defined in `polkadot-node-core-pvf` which is a level below +# the relevant subsystems. +# That being said, we will take the simplifying assumption of testing only the +# 2s timeout. + +# We do this check by ensuring all executions fall into bucket le="2" or lower. +# First, check if we have at least 1 sample, but we should have many more. +alice: reports histogram kagome_block_verification_and_import_time_bucket has at least 0 samples in buckets ["0.005", "0.01", "0.025", "0.05", "0.1", "0.25", "0.5", "1", "2.5"] within 60 seconds + +# Check if we have no samples > 2s. +alice: reports histogram kagome_block_verification_and_import_time_bucket has at least 0 samples in buckets ["1", "2.5", "5", "10", "+Inf"] within 60 seconds diff --git a/tests/kagome/0003-parachains-pvf-memory-allocation-bad.toml b/tests/kagome/0003-parachains-pvf-memory-allocation-bad.toml new file mode 100644 index 0000000..c32b66c --- /dev/null +++ b/tests/kagome/0003-parachains-pvf-memory-allocation-bad.toml @@ -0,0 +1,34 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "docker.io/parity/polkadot:latest" +default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}" +default_args = [ "-lparachain=debug" ] + +chain = "rococo-local" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + image = "soramitsu/kagome:latest" + command = "$HOME/kagome/build/node/kagome" + args = [ "--alice", "-lparachain=debug,runtime=debug" ] + prometheus_prefix = "kagome" + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug" ] + +[[parachains]] +id = 2000 +genesis_wasm_path="./wasm/heap-overflow.wasm.hex" + + [parachains.collator] + name = "collator01" + image = "docker.io/parity/polkadot-parachain:latest" + command = "polkadot-parachain" + args = ["-lparachain=debug"] diff --git a/tests/kagome/0003-parachains-pvf-memory-allocation-bad.zndsl b/tests/kagome/0003-parachains-pvf-memory-allocation-bad.zndsl new file mode 100644 index 0000000..cdecf1c --- /dev/null +++ b/tests/kagome/0003-parachains-pvf-memory-allocation-bad.zndsl @@ -0,0 +1,13 @@ +Description: PVF memory allocation +Network: ./0003-parachains-pvf-memory-allocation-kagome-bad.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds + +# Ensure parachains made progress. +alice: log line matches glob "*Trap: final > memory*" within 60 seconds +alice: parachain 2000 block height != 1 within 60 seconds \ No newline at end of file diff --git a/tests/kagome/0004-parachains-pvf-stack-allocation-bad.toml b/tests/kagome/0004-parachains-pvf-stack-allocation-bad.toml new file mode 100644 index 0000000..18ecd52 --- /dev/null +++ b/tests/kagome/0004-parachains-pvf-stack-allocation-bad.toml @@ -0,0 +1,34 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "docker.io/parity/polkadot:latest" +default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}" +default_args = [ "-lparachain=debug" ] + +chain = "rococo-local" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + image = "soramitsu/kagome:latest" + command = "$HOME/kagome/build/node/kagome" + args = [ "--alice", "-lparachain=debug,runtime=debug" ] + prometheus_prefix = "kagome" + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +genesis_wasm_path="./wasm/stack-overflow.wasm.hex" + + [parachains.collator] + name = "collator01" + image = "docker.io/parity/polkadot-parachain:latest" + command = "polkadot-parachain" + args = ["-lparachain=debug"] diff --git a/tests/kagome/0004-parachains-pvf-stack-allocation-bad.zndsl b/tests/kagome/0004-parachains-pvf-stack-allocation-bad.zndsl new file mode 100644 index 0000000..106da87 --- /dev/null +++ b/tests/kagome/0004-parachains-pvf-stack-allocation-bad.zndsl @@ -0,0 +1,13 @@ +Description: PVF stack allocation +Network: ./0004-parachains-pvf-stack-allocation-kagome-bad.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds + +# Ensure parachains made progress. +alice: log line matches glob "*Trap: offset > memory:*" within 60 seconds +alice: parachain 2000 block height != 1 within 60 seconds \ No newline at end of file diff --git a/tests/0001-parachains-pvf-compilation-time-bad.toml b/tests/polkadot/0001-parachains-pvf-compilation-time-bad.toml similarity index 100% rename from tests/0001-parachains-pvf-compilation-time-bad.toml rename to tests/polkadot/0001-parachains-pvf-compilation-time-bad.toml diff --git a/tests/0001-parachains-pvf-compilation-time-bad.zndsl b/tests/polkadot/0001-parachains-pvf-compilation-time-bad.zndsl similarity index 100% rename from tests/0001-parachains-pvf-compilation-time-bad.zndsl rename to tests/polkadot/0001-parachains-pvf-compilation-time-bad.zndsl diff --git a/tests/0001-parachains-pvf-compilation-time-good.toml b/tests/polkadot/0001-parachains-pvf-compilation-time-good.toml similarity index 100% rename from tests/0001-parachains-pvf-compilation-time-good.toml rename to tests/polkadot/0001-parachains-pvf-compilation-time-good.toml diff --git a/tests/0001-parachains-pvf-compilation-time-good.zndsl b/tests/polkadot/0001-parachains-pvf-compilation-time-good.zndsl similarity index 100% rename from tests/0001-parachains-pvf-compilation-time-good.zndsl rename to tests/polkadot/0001-parachains-pvf-compilation-time-good.zndsl diff --git a/tests/0002-parachains-pvf-execution-time-bad.toml b/tests/polkadot/0002-parachains-pvf-execution-time-bad.toml similarity index 100% rename from tests/0002-parachains-pvf-execution-time-bad.toml rename to tests/polkadot/0002-parachains-pvf-execution-time-bad.toml diff --git a/tests/0002-parachains-pvf-execution-time-bad.zndsl b/tests/polkadot/0002-parachains-pvf-execution-time-bad.zndsl similarity index 100% rename from tests/0002-parachains-pvf-execution-time-bad.zndsl rename to tests/polkadot/0002-parachains-pvf-execution-time-bad.zndsl diff --git a/tests/0002-parachains-pvf-execution-time-good.toml b/tests/polkadot/0002-parachains-pvf-execution-time-good.toml similarity index 100% rename from tests/0002-parachains-pvf-execution-time-good.toml rename to tests/polkadot/0002-parachains-pvf-execution-time-good.toml diff --git a/tests/0002-parachains-pvf-execution-time-good.zndsl b/tests/polkadot/0002-parachains-pvf-execution-time-good.zndsl similarity index 100% rename from tests/0002-parachains-pvf-execution-time-good.zndsl rename to tests/polkadot/0002-parachains-pvf-execution-time-good.zndsl diff --git a/tests/0003-parachains-pvf-memory-allocation-bad.toml b/tests/polkadot/0003-parachains-pvf-memory-allocation-bad.toml similarity index 100% rename from tests/0003-parachains-pvf-memory-allocation-bad.toml rename to tests/polkadot/0003-parachains-pvf-memory-allocation-bad.toml diff --git a/tests/0003-parachains-pvf-memory-allocation-bad.zndsl b/tests/polkadot/0003-parachains-pvf-memory-allocation-bad.zndsl similarity index 100% rename from tests/0003-parachains-pvf-memory-allocation-bad.zndsl rename to tests/polkadot/0003-parachains-pvf-memory-allocation-bad.zndsl diff --git a/tests/0003-parachains-pvf-memory-allocation-good.toml b/tests/polkadot/0003-parachains-pvf-memory-allocation-good.toml similarity index 100% rename from tests/0003-parachains-pvf-memory-allocation-good.toml rename to tests/polkadot/0003-parachains-pvf-memory-allocation-good.toml diff --git a/tests/0003-parachains-pvf-memory-allocation-good.zndsl b/tests/polkadot/0003-parachains-pvf-memory-allocation-good.zndsl similarity index 100% rename from tests/0003-parachains-pvf-memory-allocation-good.zndsl rename to tests/polkadot/0003-parachains-pvf-memory-allocation-good.zndsl diff --git a/tests/0004-parachains-pvf-stack-allocation-bad.toml b/tests/polkadot/0004-parachains-pvf-stack-allocation-bad.toml similarity index 100% rename from tests/0004-parachains-pvf-stack-allocation-bad.toml rename to tests/polkadot/0004-parachains-pvf-stack-allocation-bad.toml diff --git a/tests/0004-parachains-pvf-stack-allocation-bad.zndsl b/tests/polkadot/0004-parachains-pvf-stack-allocation-bad.zndsl similarity index 100% rename from tests/0004-parachains-pvf-stack-allocation-bad.zndsl rename to tests/polkadot/0004-parachains-pvf-stack-allocation-bad.zndsl diff --git a/tests/0004-parachains-pvf-stack-allocation-good.toml b/tests/polkadot/0004-parachains-pvf-stack-allocation-good.toml similarity index 100% rename from tests/0004-parachains-pvf-stack-allocation-good.toml rename to tests/polkadot/0004-parachains-pvf-stack-allocation-good.toml diff --git a/tests/0004-parachains-pvf-stack-allocation-good.zndsl b/tests/polkadot/0004-parachains-pvf-stack-allocation-good.zndsl similarity index 100% rename from tests/0004-parachains-pvf-stack-allocation-good.zndsl rename to tests/polkadot/0004-parachains-pvf-stack-allocation-good.zndsl