From f8254dc4278d0afcf66ebfa944f3046e82f0238b Mon Sep 17 00:00:00 2001 From: vsoch Date: Fri, 14 Jun 2024 11:48:03 -0600 Subject: [PATCH] save state Signed-off-by: vsoch --- config/manager/kustomization.yaml | 1 + .../multiple-pods-per-node/README.md | 10 +---- .../tests/osu-benchmarks/minicluster.yaml | 2 +- pkg/flux/templates/components.sh | 37 +++++++++++++++++++ pkg/flux/templates/start.sh | 2 + pkg/flux/templates/wait.sh | 18 +-------- 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 67ba7813..1d8b5340 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,3 +13,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/flux-framework/flux-operator + newTag: test diff --git a/examples/experimental/multiple-pods-per-node/README.md b/examples/experimental/multiple-pods-per-node/README.md index 51f85939..63c8a8fd 100644 --- a/examples/experimental/multiple-pods-per-node/README.md +++ b/examples/experimental/multiple-pods-per-node/README.md @@ -27,18 +27,12 @@ gcloud container clusters create test-cluster \ ### Install the Flux Operator -We are going to install the Flux operator from the refactor branch (with the feature added to disable affinity). +As follows: ```bash -git clone -b test-refactor-modular -cd test-refactor-modular - -# You might need other dependencies, etc. here or to specify your own registry you can push to. -make test-deploy-recreate +kubectl apply -f https://raw.githubusercontent.com/flux-framework/flux-operator/main/examples/dist/flux-operator.yaml ``` -If/when this is released, you can install from a release. - ### Experiments Then create the flux operator pods. diff --git a/examples/tests/osu-benchmarks/minicluster.yaml b/examples/tests/osu-benchmarks/minicluster.yaml index bf6a3d9f..f03a6dff 100644 --- a/examples/tests/osu-benchmarks/minicluster.yaml +++ b/examples/tests/osu-benchmarks/minicluster.yaml @@ -21,4 +21,4 @@ spec: containers: - image: ghcr.io/converged-computing/metric-osu-benchmark:latest workingDir: /opt/osu-benchmark/build.openmpi/libexec/osu-micro-benchmarks/mpi/one-sided - command: ./osu_get_latency \ No newline at end of file + command: ./osu_get_latency diff --git a/pkg/flux/templates/components.sh b/pkg/flux/templates/components.sh index c246d285..1e923b51 100644 --- a/pkg/flux/templates/components.sh +++ b/pkg/flux/templates/components.sh @@ -63,6 +63,43 @@ command="/bin/bash ./custom-entrypoint.sh" {{end}} {{end}} +{{define "broker"}} +brokerOptions="-Scron.directory=/etc/flux/system/cron.d \ + -Stbon.fanout=256 \ + -Srundir=${viewroot}/run/flux {{ if .Spec.Interactive }}-Sbroker.rc2_none {{ end }} \ + -Sstatedir=${STATE_DIR} \ + -Slocal-uri=local://$viewroot/run/flux/local \ +{{ if .Spec.Flux.ConnectTimeout }}-Stbon.connect_timeout={{ .Spec.Flux.ConnectTimeout }}{{ end }} \ +{{ if .RequiredRanks }}-Sbroker.quorum={{ .RequiredRanks }}{{ end }} \ +{{ if .Spec.Logging.Zeromq }}-Stbon.zmqdebug=1{{ end }} \ +{{ if not .Spec.Logging.Quiet }} -Slog-stderr-level={{or .Spec.Flux.LogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \ + -Slog-stderr-mode=local" + + +# Run an interactive cluster, giving no command to flux start +function run_interactive_cluster() { + echo "🌀 flux broker --config-path ${cfg} ${brokerOptions}" + flux broker --config-path ${cfg} ${brokerOptions} +} +{{end}} + +{{define "worker-broker"}} +cfg="${viewroot}/etc/flux/config" +brokerOptions="-Stbon.fanout=256 \ + -Srundir=${viewroot}/run/flux {{ if .Spec.Interactive }}-Sbroker.rc2_none {{ end }} \ + -Slocal-uri=local://$viewroot/run/flux/local \ +{{ if .Spec.Flux.ConnectTimeout }}-Stbon.connect_timeout={{ .Spec.Flux.ConnectTimeout }}{{ end }} \ +{{ if .Spec.Logging.Zeromq }}-Stbon.zmqdebug=1{{ end }} \ +{{ if not .Spec.Logging.Quiet }} -Slog-stderr-level={{or .Spec.Flux.LogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \ + -Slog-stderr-mode=local" + +# This is provided as an optional function for a worker +function run_interactive_cluster() { + echo "🌀 flux broker --config-path ${cfg} ${brokerOptions}" + flux broker --config-path ${cfg} ${brokerOptions} +} +{{end}} + {{define "paths"}} foundroot=$(find $viewroot -maxdepth 2 -type d -path $viewroot/lib/python3\*) {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }} pythonversion=$(basename ${foundroot}) diff --git a/pkg/flux/templates/start.sh b/pkg/flux/templates/start.sh index 3c984d4e..fc0f7992 100644 --- a/pkg/flux/templates/start.sh +++ b/pkg/flux/templates/start.sh @@ -29,6 +29,8 @@ fluxsocket="local://$fluxsocket" # Is a custom script provided? {{template "custom-script" .}} +{{template "worker-broker" .}} + {{ .Container.Command }} {{ .Container.Commands.Post}} diff --git a/pkg/flux/templates/wait.sh b/pkg/flux/templates/wait.sh index dc489d45..7286b91c 100644 --- a/pkg/flux/templates/wait.sh +++ b/pkg/flux/templates/wait.sh @@ -82,23 +82,7 @@ echo "The working directory is ${workdir}, contents include:" ls . {{ end }} -brokerOptions="-Scron.directory=/etc/flux/system/cron.d \ - -Stbon.fanout=256 \ - -Srundir=${viewroot}/run/flux {{ if .Spec.Interactive }}-Sbroker.rc2_none {{ end }} \ - -Sstatedir=${STATE_DIR} \ - -Slocal-uri=local://$viewroot/run/flux/local \ -{{ if .Spec.Flux.ConnectTimeout }}-Stbon.connect_timeout={{ .Spec.Flux.ConnectTimeout }}{{ end }} \ -{{ if .RequiredRanks }}-Sbroker.quorum={{ .RequiredRanks }}{{ end }} \ -{{ if .Spec.Logging.Zeromq }}-Stbon.zmqdebug=1{{ end }} \ -{{ if not .Spec.Logging.Quiet }} -Slog-stderr-level={{or .Spec.Flux.LogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \ - -Slog-stderr-mode=local" - - -# Run an interactive cluster, giving no command to flux start -function run_interactive_cluster() { - echo "🌀 flux broker --config-path ${cfg} ${brokerOptions}" - flux broker --config-path ${cfg} ${brokerOptions} -} +{{template "broker" .}} # if we are given an archive to use, load first, not required to exist # Note that we ask the user to dump in interactive mode - I am not