Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: libpe_status: Don't fence a remote node due to failed migrate_from #3458

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
5 changes: 5 additions & 0 deletions cts/cts-scheduler.in
Original file line number Diff line number Diff line change
Expand Up @@ -1120,6 +1120,11 @@ TESTS = [
"Make sure partial migrations are handled before ops on the remote node" ],
[ "remote-partial-migrate2",
"Make sure partial migration target is prefered for remote connection" ],
[ "remote-failed-migrate1",
"Recover immediately without fencing after failed migrate_from"],
[ "remote-failed-migrate2",
"Recover immediately without fencing after failed migrate_from "
+ "(rsc active)"],
[ "remote-recover-fail", "Make sure start failure causes fencing if rsc are active on remote" ],
[ "remote-start-fail",
"Make sure a start failure does not result in fencing if no active resources are on remote" ],
Expand Down
47 changes: 23 additions & 24 deletions cts/scheduler/dot/bug-cl-5247.dot
Original file line number Diff line number Diff line change
Expand Up @@ -30,69 +30,67 @@
"msPostgresql_demoted_0" [ style=bold color="green" fontcolor="orange"]
"msPostgresql_post_notify_demoted_0" -> "msPostgresql_confirmed-post_notify_demoted_0" [ style = bold]
"msPostgresql_post_notify_demoted_0" -> "pgsql_post_notify_demoted_0 pgsr01" [ style = bold]
"msPostgresql_post_notify_demoted_0" -> "pgsql_post_notify_demoted_0 pgsr02" [ style = bold]
"msPostgresql_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"]
"msPostgresql_post_notify_stopped_0" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold]
"msPostgresql_post_notify_stopped_0" -> "pgsql_post_notify_stonith_0 pgsr01" [ style = bold]
"msPostgresql_post_notify_stopped_0" -> "pgsql_post_notify_stopped_0 pgsr01" [ style = bold]
"msPostgresql_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
"msPostgresql_pre_notify_demote_0" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold]
"msPostgresql_pre_notify_demote_0" -> "pgsql_pre_notify_demote_0 pgsr01" [ style = bold]
"msPostgresql_pre_notify_demote_0" -> "pgsql_pre_notify_demote_0 pgsr02" [ style = bold]
"msPostgresql_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"]
"msPostgresql_pre_notify_stop_0" -> "msPostgresql_confirmed-pre_notify_stop_0" [ style = bold]
"msPostgresql_pre_notify_stop_0" -> "pgsql_pre_notify_stop_0 pgsr01" [ style = bold]
"msPostgresql_pre_notify_stop_0" -> "pgsql_pre_notify_stop_0 pgsr02" [ style = bold]
"msPostgresql_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
"msPostgresql_stop_0" -> "msPostgresql_stopped_0" [ style = bold]
"msPostgresql_stop_0" -> "pgsql_stop_0 pgsr02" [ style = bold]
"msPostgresql_stop_0" [ style=bold color="green" fontcolor="orange"]
"msPostgresql_stopped_0" -> "msPostgresql_post_notify_stopped_0" [ style = bold]
"msPostgresql_stopped_0" [ style=bold color="green" fontcolor="orange"]
"pgsql_confirmed-post_notify_stonith_0" -> "pgsql_monitor_9000 pgsr01" [ style = bold]
"pgsql_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"pgsql_demote_0 pgsr02" -> "msPostgresql_demoted_0" [ style = bold]
"pgsql_demote_0 pgsr02" -> "pgsql_stop_0 pgsr02" [ style = bold]
"pgsql_demote_0 pgsr02" [ style=bold color="green" fontcolor="orange"]
"pgsql_demote_0 pgsr02" -> "pgsr02_stop_0 bl460g8n4" [ style = bold]
"pgsql_demote_0 pgsr02" [ style=bold color="green" fontcolor="black"]
"pgsql_monitor_9000 pgsr01" [ style=bold color="green" fontcolor="black"]
"pgsql_post_notify_demoted_0 pgsr01" -> "msPostgresql_confirmed-post_notify_demoted_0" [ style = bold]
"pgsql_post_notify_demoted_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"pgsql_post_notify_stonith_0 pgsr01" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold]
"pgsql_post_notify_stonith_0 pgsr01" -> "pgsql_confirmed-post_notify_stonith_0" [ style = bold]
"pgsql_post_notify_stonith_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"pgsql_post_notify_stonith_0" -> "pgsql_confirmed-post_notify_stonith_0" [ style = bold]
"pgsql_post_notify_stonith_0" -> "pgsql_post_notify_stonith_0 pgsr01" [ style = bold]
"pgsql_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"]
"pgsql_post_notify_demoted_0 pgsr02" -> "msPostgresql_confirmed-post_notify_demoted_0" [ style = bold]
"pgsql_post_notify_demoted_0 pgsr02" [ style=bold color="green" fontcolor="black"]
"pgsql_post_notify_stopped_0 pgsr01" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold]
"pgsql_post_notify_stopped_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"pgsql_pre_notify_demote_0 pgsr01" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold]
"pgsql_pre_notify_demote_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"pgsql_pre_notify_demote_0 pgsr02" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold]
"pgsql_pre_notify_demote_0 pgsr02" [ style=bold color="green" fontcolor="black"]
"pgsql_pre_notify_stop_0 pgsr01" -> "msPostgresql_confirmed-pre_notify_stop_0" [ style = bold]
"pgsql_pre_notify_stop_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"pgsql_pre_notify_stop_0 pgsr02" -> "msPostgresql_confirmed-pre_notify_stop_0" [ style = bold]
"pgsql_pre_notify_stop_0 pgsr02" [ style=bold color="green" fontcolor="black"]
"pgsql_stop_0 pgsr02" -> "msPostgresql_stopped_0" [ style = bold]
"pgsql_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"]
"pgsql_stop_0 pgsr02" -> "pgsr02_stop_0 bl460g8n4" [ style = bold]
"pgsql_stop_0 pgsr02" [ style=bold color="green" fontcolor="black"]
"pgsr01_monitor_0 bl460g8n4" [ style=bold color="green" fontcolor="black"]
"pgsr02_monitor_0 bl460g8n3" [ style=bold color="green" fontcolor="black"]
"pgsr02_stop_0 bl460g8n4" -> "prmDB2_stop_0 bl460g8n4" [ style = bold]
"pgsr02_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"]
"prmDB2_stop_0 bl460g8n4" -> "stonith 'off' pgsr02" [ style = bold]
"prmDB2_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"]
"stonith 'off' pgsr02" -> "master-group_stop_0" [ style = bold]
"stonith 'off' pgsr02" -> "msPostgresql_stop_0" [ style = bold]
"stonith 'off' pgsr02" -> "pgsql_demote_0 pgsr02" [ style = bold]
"stonith 'off' pgsr02" -> "pgsql_post_notify_stonith_0" [ style = bold]
"stonith 'off' pgsr02" -> "pgsql_stop_0 pgsr02" [ style = bold]
"stonith 'off' pgsr02" -> "vip-master_start_0 pgsr01" [ style = bold]
"stonith 'off' pgsr02" -> "vip-master_stop_0 pgsr02" [ style = bold]
"stonith 'off' pgsr02" -> "vip-rep_start_0 pgsr01" [ style = bold]
"stonith 'off' pgsr02" -> "vip-rep_stop_0 pgsr02" [ style = bold]
"stonith 'off' pgsr02" [ style=bold color="green" fontcolor="orange"]
"vip-master_monitor_0 pgsr01" -> "master-group_stopped_0" [ style = bold]
"vip-master_monitor_0 pgsr01" -> "pgsr02_stop_0 bl460g8n4" [ style = bold]
"vip-master_monitor_0 pgsr01" -> "vip-master_start_0 pgsr01" [ style = bold]
"vip-master_monitor_0 pgsr01" -> "vip-rep_stop_0 pgsr02" [ style = bold]
"vip-master_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"vip-master_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"]
"vip-master_start_0 pgsr01" -> "master-group_running_0" [ style = bold]
"vip-master_start_0 pgsr01" -> "vip-master_monitor_10000 pgsr01" [ style = bold]
"vip-master_start_0 pgsr01" -> "vip-rep_start_0 pgsr01" [ style = bold]
"vip-master_start_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"vip-master_stop_0 pgsr02" -> "master-group_stopped_0" [ style = bold]
"vip-master_stop_0 pgsr02" -> "pgsr02_stop_0 bl460g8n4" [ style = bold]
"vip-master_stop_0 pgsr02" -> "vip-master_start_0 pgsr01" [ style = bold]
"vip-master_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"]
"vip-master_stop_0 pgsr02" [ style=bold color="green" fontcolor="black"]
"vip-rep_monitor_0 pgsr01" -> "master-group_stopped_0" [ style = bold]
"vip-rep_monitor_0 pgsr01" -> "pgsr02_stop_0 bl460g8n4" [ style = bold]
"vip-rep_monitor_0 pgsr01" -> "vip-master_stop_0 pgsr02" [ style = bold]
"vip-rep_monitor_0 pgsr01" -> "vip-rep_start_0 pgsr01" [ style = bold]
"vip-rep_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"]
Expand All @@ -101,7 +99,8 @@
"vip-rep_start_0 pgsr01" -> "vip-rep_monitor_10000 pgsr01" [ style = bold]
"vip-rep_start_0 pgsr01" [ style=bold color="green" fontcolor="black"]
"vip-rep_stop_0 pgsr02" -> "master-group_stopped_0" [ style = bold]
"vip-rep_stop_0 pgsr02" -> "pgsr02_stop_0 bl460g8n4" [ style = bold]
"vip-rep_stop_0 pgsr02" -> "vip-master_stop_0 pgsr02" [ style = bold]
"vip-rep_stop_0 pgsr02" -> "vip-rep_start_0 pgsr01" [ style = bold]
"vip-rep_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"]
"vip-rep_stop_0 pgsr02" [ style=bold color="green" fontcolor="black"]
}
8 changes: 2 additions & 6 deletions cts/scheduler/dot/bundle-connection-with-container.dot
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
"httpd-bundle-podman-0_start_0 rhel8-1" [ style=bold color="green" fontcolor="black"]
"httpd-bundle-podman-0_stop_0 rhel8-1" -> "httpd-bundle-podman-0_start_0 rhel8-1" [ style = bold]
"httpd-bundle-podman-0_stop_0 rhel8-1" -> "httpd-bundle_stopped_0" [ style = bold]
"httpd-bundle-podman-0_stop_0 rhel8-1" -> "stonith 'reboot' httpd-bundle-0" [ style = bold]
"httpd-bundle-podman-0_stop_0 rhel8-1" [ style=bold color="green" fontcolor="black"]
"httpd-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
"httpd-bundle_start_0" -> "httpd-bundle-clone_start_0" [ style = bold]
Expand All @@ -41,11 +40,8 @@
"httpd_start_0 httpd-bundle-0" -> "httpd-bundle-clone_running_0" [ style = bold]
"httpd_start_0 httpd-bundle-0" -> "httpd_monitor_15000 httpd-bundle-0" [ style = bold]
"httpd_start_0 httpd-bundle-0" [ style=bold color="green" fontcolor="black"]
"httpd_stop_0 httpd-bundle-0" -> "httpd-bundle-0_stop_0 rhel8-1" [ style = bold]
"httpd_stop_0 httpd-bundle-0" -> "httpd-bundle-clone_stopped_0" [ style = bold]
"httpd_stop_0 httpd-bundle-0" -> "httpd_start_0 httpd-bundle-0" [ style = bold]
"httpd_stop_0 httpd-bundle-0" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' httpd-bundle-0" -> "httpd-bundle-clone_stop_0" [ style = bold]
"stonith 'reboot' httpd-bundle-0" -> "httpd-bundle-podman-0_start_0 rhel8-1" [ style = bold]
"stonith 'reboot' httpd-bundle-0" -> "httpd_start_0 httpd-bundle-0" [ style = bold]
"stonith 'reboot' httpd-bundle-0" [ style=bold color="green" fontcolor="orange"]
"httpd_stop_0 httpd-bundle-0" [ style=bold color="green" fontcolor="black"]
}
26 changes: 4 additions & 22 deletions cts/scheduler/dot/nested-remote-recovery.dot
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,16 @@
"galera-bundle-docker-0_start_0 database-0" [ style=bold color="green" fontcolor="black"]
"galera-bundle-docker-0_stop_0 database-0" -> "galera-bundle-docker-0_start_0 database-0" [ style = bold]
"galera-bundle-docker-0_stop_0 database-0" -> "galera-bundle_stopped_0" [ style = bold]
"galera-bundle-docker-0_stop_0 database-0" -> "stonith 'reboot' galera-bundle-0" [ style = bold]
"galera-bundle-docker-0_stop_0 database-0" [ style=bold color="green" fontcolor="black"]
"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle-master_demoted_0" -> "galera-bundle-master_promote_0" [ style = bold]
"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = bold]
"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold]
"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold]
"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle-master_promote_0" -> "galera_promote_0 galera-bundle-0" [ style = bold]
"galera-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle-master_promoted_0" -> "galera-bundle_promoted_0" [ style = bold]
"galera-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle-master_running_0" -> "galera-bundle-master_promote_0" [ style = bold]
"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
Expand All @@ -44,17 +41,6 @@
"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = bold]
"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold]
"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold]
"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold]
"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_demoted_0" -> "galera-bundle_promote_0" [ style = bold]
"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = bold]
"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold]
"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_promote_0" -> "galera-bundle-master_promote_0" [ style = bold]
"galera-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_running_0" -> "galera-bundle_promote_0" [ style = bold]
"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 database-0" [ style = bold]
"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
Expand All @@ -63,13 +49,13 @@
"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
"galera-bundle_stopped_0" -> "galera-bundle_promote_0" [ style = bold]
"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = bold]
"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 controller-0" [ style = bold]
"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold]
"galera_demote_0 galera-bundle-0" -> "galera_promote_0 galera-bundle-0" [ style = bold]
"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="orange"]
"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
"galera_monitor_10000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
"galera_promote_0 galera-bundle-0" -> "galera-bundle-master_promoted_0" [ style = bold]
"galera_promote_0 galera-bundle-0" -> "galera_monitor_10000 galera-bundle-0" [ style = bold]
Expand All @@ -78,12 +64,8 @@
"galera_start_0 galera-bundle-0" -> "galera_monitor_10000 galera-bundle-0" [ style = bold]
"galera_start_0 galera-bundle-0" -> "galera_promote_0 galera-bundle-0" [ style = bold]
"galera_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 controller-0" [ style = bold]
"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold]
"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = bold]
"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="orange"]
"stonith 'reboot' galera-bundle-0" -> "galera-bundle-docker-0_start_0 database-0" [ style = bold]
"stonith 'reboot' galera-bundle-0" -> "galera-bundle-master_stop_0" [ style = bold]
"stonith 'reboot' galera-bundle-0" -> "galera_promote_0 galera-bundle-0" [ style = bold]
"stonith 'reboot' galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = bold]
"stonith 'reboot' galera-bundle-0" [ style=bold color="green" fontcolor="orange"]
"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
}
9 changes: 9 additions & 0 deletions cts/scheduler/dot/remote-failed-migrate1.dot
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
digraph "g" {
"fastvm-fedora39-23_monitor_60000 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" -> "fastvm-fedora39-23_monitor_60000 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-22" -> "fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-22" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-24" -> "fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
}
11 changes: 11 additions & 0 deletions cts/scheduler/dot/remote-failed-migrate2.dot
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
digraph "g" {
"dummy_monitor_10000 fastvm-fedora39-23" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_monitor_60000 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" -> "dummy_monitor_10000 fastvm-fedora39-23" [ style = bold]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" -> "fastvm-fedora39-23_monitor_60000 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-22" -> "fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-22" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-24" -> "fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
}
2 changes: 0 additions & 2 deletions cts/scheduler/dot/remote-fence-unclean-3.dot
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,4 @@
"redis-bundle-1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"]
"redis-bundle-2_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"]
"redis-bundle-2_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"]
"stonith 'reboot' overcloud-novacompute-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold]
"stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"]
}
9 changes: 9 additions & 0 deletions cts/scheduler/dot/remote-partial-migrate3.dot
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
digraph "g" {
"fastvm-fedora39-23_monitor_60000 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" -> "fastvm-fedora39-23_monitor_60000 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-22" -> "fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-22" [ style=bold color="green" fontcolor="black"]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-24" -> "fastvm-fedora39-23_start_0 fastvm-fedora39-24" [ style = bold]
"fastvm-fedora39-23_stop_0 fastvm-fedora39-24" [ style=bold color="green" fontcolor="black"]
}
Loading