feat(tests): Add kagome tests. Separate kagome and polkadot tests int…

…o subfolders

Signed-off-by: Maksim Dimitrov <[email protected]>

tests/kagome/0001-parachains-pvf-compilation-time-bad.toml

@@ -0,0 +1,35 @@
+[settings]
+timeout = 1000
+
+[relaychain]
+# default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_image = "docker.io/parity/polkadot:latest"
+default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+[relaychain.default_resources]
+limits = { memory = "4G", cpu = "2" }
+requests = { memory = "2G", cpu = "1" }
+
+  [[relaychain.nodes]]
+  name = "alice"
+  image = "soramitsu/kagome:latest"
+  command = "$HOME/kagome/build/node/kagome"
+  args = [ "--alice", "-lparachain=debug,runtime=debug" ]
+  prometheus_prefix = "kagome"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  args = [ "--bob", "-lparachain=debug,runtime=debug"]
+
+[[parachains]]
+id = 2000
+genesis_wasm_path="./wasm/noops.wasm.hex"
+
+  [parachains.collator]
+  name = "collator01"
+  image = "docker.io/parity/polkadot-parachain:latest"
+  command = "polkadot-parachain"
+  args = ["-lparachain=debug"]

tests/kagome/0001-parachains-pvf-compilation-time-bad.zndsl

@@ -0,0 +1,19 @@
+Description: PVF compilation time
+Network: ./0001-parachains-pvf-compilation-time-kagome-bad.toml
+Creds: config
+
+# Check authority status.
+alice: reports node_roles is 4
+
+# Ensure parachains are registered.
+alice: parachain 2000 is registered within 60 seconds
+
+# Ensure parachains made progress.
+alice: log line matches glob "*InvalidCandidate(HardTimeout)*" within 120 seconds
+alice: parachain 2000 block height != 1 within 60 seconds
+
+# Check preparation time is under 10s.
+# Check all buckets < 10.
+alice: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["0.1", "0.5", "1", "2", "3"] within 20 seconds
+# Check all buckets >= 10.
+alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["10", "20", "30", "60", "120", "+Inf"] within 20 seconds

tests/kagome/0002-parachains-pvf-execution-time-bad.toml

@@ -0,0 +1,34 @@
+[settings]
+timeout = 1000
+
+[relaychain]
+default_image = "docker.io/parity/polkadot:latest"
+default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+[relaychain.default_resources]
+limits = { memory = "4G", cpu = "2" }
+requests = { memory = "2G", cpu = "1" }
+
+  [[relaychain.nodes]]
+  name = "alice"
+  image = "soramitsu/kagome:latest"
+  command = "$HOME/kagome/build/node/kagome"
+  args = [ "--alice", "-lparachain=debug,runtime=debug" ]
+  prometheus_prefix = "kagome"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  args = [ "--bob", "-lparachain=debug,runtime=debug"]
+
+[[parachains]]
+id = 2000
+genesis_wasm_path="./wasm/infinite-loop.wasm.hex"
+
+  [parachains.collator]
+  name = "collator01"
+  image = "docker.io/parity/polkadot-parachain:latest"
+  command = "polkadot-parachain"
+  args = ["-lparachain=debug"]

tests/kagome/0002-parachains-pvf-execution-time-bad.zndsl

@@ -0,0 +1,27 @@
+Description: PVF compilation time
+Network: ./0002-parachains-pvf-execution-time-kagome-bad.toml
+Creds: config
+
+# Check authority status.
+alice: reports kagome_node_roles is 4
+
+# Ensure parachains are registered.
+alice: parachain 2000 is registered within 60 seconds
+
+# Ensure parachains made progress.
+alice: parachain 2000 block height != 1 within 60 seconds
+
+# Check execution time.
+# There are two different timeout conditions: BACKING_EXECUTION_TIMEOUT(2s) and
+# APPROVAL_EXECUTION_TIMEOUT(6s). Currently these are not differentiated by metrics
+# because the metrics are defined in `polkadot-node-core-pvf` which is a level below
+# the relevant subsystems.
+# That being said, we will take the simplifying assumption of testing only the
+# 2s timeout.
+
+# We do this check by ensuring all executions fall into bucket le="2" or lower.
+# First, check if we have at least 1 sample, but we should have many more.
+alice: reports histogram kagome_block_verification_and_import_time_bucket has at least 0 samples in buckets ["0.005", "0.01", "0.025", "0.05", "0.1", "0.25", "0.5", "1", "2.5"] within 60 seconds
+
+# Check if we have no samples > 2s.
+alice: reports histogram kagome_block_verification_and_import_time_bucket has at least 0 samples in buckets ["1", "2.5", "5", "10", "+Inf"] within 60 seconds

tests/kagome/0003-parachains-pvf-memory-allocation-bad.toml

@@ -0,0 +1,34 @@
+[settings]
+timeout = 1000
+
+[relaychain]
+default_image = "docker.io/parity/polkadot:latest"
+default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+[relaychain.default_resources]
+limits = { memory = "4G", cpu = "2" }
+requests = { memory = "2G", cpu = "1" }
+
+  [[relaychain.nodes]]
+  name = "alice"
+  image = "soramitsu/kagome:latest"
+  command = "$HOME/kagome/build/node/kagome"
+  args = [ "--alice", "-lparachain=debug,runtime=debug" ]
+  prometheus_prefix = "kagome"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  args = [ "--bob", "-lparachain=debug,runtime=debug" ]
+
+[[parachains]]
+id = 2000
+genesis_wasm_path="./wasm/heap-overflow.wasm.hex"
+
+  [parachains.collator]
+  name = "collator01"
+  image = "docker.io/parity/polkadot-parachain:latest"
+  command = "polkadot-parachain"
+  args = ["-lparachain=debug"]

tests/kagome/0003-parachains-pvf-memory-allocation-bad.zndsl

@@ -0,0 +1,13 @@
+Description: PVF memory allocation
+Network: ./0003-parachains-pvf-memory-allocation-kagome-bad.toml
+Creds: config
+
+# Check authority status.
+alice: reports node_roles is 4
+
+# Ensure parachains are registered.
+alice: parachain 2000 is registered within 60 seconds
+
+# Ensure parachains made progress.
+alice: log line matches glob "*Trap: final > memory*" within 60 seconds
+alice: parachain 2000 block height != 1 within 60 seconds

tests/kagome/0004-parachains-pvf-stack-allocation-bad.toml

@@ -0,0 +1,34 @@
+[settings]
+timeout = 1000
+
+[relaychain]
+default_image = "docker.io/parity/polkadot:latest"
+default_command = "{{ZOMBIENET_DEFAULT_START_COMMAND}}"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+[relaychain.default_resources]
+limits = { memory = "4G", cpu = "2" }
+requests = { memory = "2G", cpu = "1" }
+
+  [[relaychain.nodes]]
+  name = "alice"
+  image = "soramitsu/kagome:latest"
+  command = "$HOME/kagome/build/node/kagome"
+  args = [ "--alice", "-lparachain=debug,runtime=debug" ]
+  prometheus_prefix = "kagome"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  args = [ "--bob", "-lparachain=debug,runtime=debug"]
+
+[[parachains]]
+id = 2000
+genesis_wasm_path="./wasm/stack-overflow.wasm.hex"
+
+  [parachains.collator]
+  name = "collator01"
+  image = "docker.io/parity/polkadot-parachain:latest"
+  command = "polkadot-parachain"
+  args = ["-lparachain=debug"]

tests/kagome/0004-parachains-pvf-stack-allocation-bad.zndsl

@@ -0,0 +1,13 @@
+Description: PVF stack allocation
+Network: ./0004-parachains-pvf-stack-allocation-kagome-bad.toml
+Creds: config
+
+# Check authority status.
+alice: reports node_roles is 4
+
+# Ensure parachains are registered.
+alice: parachain 2000 is registered within 60 seconds
+
+# Ensure parachains made progress.
+alice: log line matches glob "*Trap: offset > memory:*" within 60 seconds
+alice: parachain 2000 block height != 1 within 60 seconds