save state

Signed-off-by: vsoch <[email protected]>

config/manager/kustomization.yaml

@@ -13,3 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/flux-framework/flux-operator
+  newTag: test

examples/experimental/multiple-pods-per-node/README.md

@@ -27,18 +27,12 @@ gcloud container clusters create test-cluster \
 
 ### Install the Flux Operator
 
-We are going to install the Flux operator from the refactor branch (with the feature added to disable affinity).
+As follows:
 
 ```bash
-git clone -b test-refactor-modular 
-cd test-refactor-modular
-
-# You might need other dependencies, etc. here or to specify your own registry you can push to.
-make test-deploy-recreate
+kubectl apply -f https://raw.githubusercontent.com/flux-framework/flux-operator/main/examples/dist/flux-operator.yaml
 ```
 
-If/when this is released, you can install from a release.
-
 ### Experiments
 
 Then create the flux operator pods. 

examples/tests/osu-benchmarks/minicluster.yaml

@@ -21,4 +21,4 @@ spec:
   containers:
     - image: ghcr.io/converged-computing/metric-osu-benchmark:latest
       workingDir: /opt/osu-benchmark/build.openmpi/libexec/osu-micro-benchmarks/mpi/one-sided
-      command: ./osu_get_latency
+      command: ./osu_get_latency

pkg/flux/templates/components.sh

@@ -63,6 +63,43 @@ command="/bin/bash ./custom-entrypoint.sh"
 {{end}}
 {{end}}
 
+{{define "broker"}}
+brokerOptions="-Scron.directory=/etc/flux/system/cron.d \
+  -Stbon.fanout=256 \
+  -Srundir=${viewroot}/run/flux {{ if .Spec.Interactive }}-Sbroker.rc2_none {{ end }} \
+  -Sstatedir=${STATE_DIR} \
+  -Slocal-uri=local://$viewroot/run/flux/local \
+{{ if .Spec.Flux.ConnectTimeout }}-Stbon.connect_timeout={{ .Spec.Flux.ConnectTimeout }}{{ end }} \
+{{ if .RequiredRanks }}-Sbroker.quorum={{ .RequiredRanks }}{{ end }} \
+{{ if .Spec.Logging.Zeromq }}-Stbon.zmqdebug=1{{ end }} \
+{{ if not .Spec.Logging.Quiet }} -Slog-stderr-level={{or .Spec.Flux.LogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \
+  -Slog-stderr-mode=local"
+
+
+# Run an interactive cluster, giving no command to flux start
+function run_interactive_cluster() {
+    echo "🌀 flux broker --config-path ${cfg} ${brokerOptions}"
+    flux broker --config-path ${cfg} ${brokerOptions}
+}
+{{end}}
+
+{{define "worker-broker"}}
+cfg="${viewroot}/etc/flux/config"
+brokerOptions="-Stbon.fanout=256 \
+  -Srundir=${viewroot}/run/flux {{ if .Spec.Interactive }}-Sbroker.rc2_none {{ end }} \
+  -Slocal-uri=local://$viewroot/run/flux/local \
+{{ if .Spec.Flux.ConnectTimeout }}-Stbon.connect_timeout={{ .Spec.Flux.ConnectTimeout }}{{ end }} \
+{{ if .Spec.Logging.Zeromq }}-Stbon.zmqdebug=1{{ end }} \
+{{ if not .Spec.Logging.Quiet }} -Slog-stderr-level={{or .Spec.Flux.LogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \
+  -Slog-stderr-mode=local"
+
+# This is provided as an optional function for a worker
+function run_interactive_cluster() {
+    echo "🌀 flux broker --config-path ${cfg} ${brokerOptions}"
+    flux broker --config-path ${cfg} ${brokerOptions}
+}
+{{end}}
+
 {{define "paths"}}
 foundroot=$(find $viewroot -maxdepth 2 -type d -path $viewroot/lib/python3\*) {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }}
 pythonversion=$(basename ${foundroot})

pkg/flux/templates/start.sh

@@ -29,6 +29,8 @@ fluxsocket="local://$fluxsocket"
 # Is a custom script provided?
 {{template "custom-script" .}}
 
+{{template "worker-broker" .}}
+
 {{ .Container.Command }}
 
 {{ .Container.Commands.Post}}

pkg/flux/templates/wait.sh

@@ -82,23 +82,7 @@ echo "The working directory is ${workdir}, contents include:"
 ls .
 {{ end }}
 
-brokerOptions="-Scron.directory=/etc/flux/system/cron.d \
-  -Stbon.fanout=256 \
-  -Srundir=${viewroot}/run/flux {{ if .Spec.Interactive }}-Sbroker.rc2_none {{ end }} \
-  -Sstatedir=${STATE_DIR} \
-  -Slocal-uri=local://$viewroot/run/flux/local \
-{{ if .Spec.Flux.ConnectTimeout }}-Stbon.connect_timeout={{ .Spec.Flux.ConnectTimeout }}{{ end }} \
-{{ if .RequiredRanks }}-Sbroker.quorum={{ .RequiredRanks }}{{ end }} \
-{{ if .Spec.Logging.Zeromq }}-Stbon.zmqdebug=1{{ end }} \
-{{ if not .Spec.Logging.Quiet }} -Slog-stderr-level={{or .Spec.Flux.LogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \
-  -Slog-stderr-mode=local"
-
-
-# Run an interactive cluster, giving no command to flux start
-function run_interactive_cluster() {
-    echo "🌀 flux broker --config-path ${cfg} ${brokerOptions}"
-    flux broker --config-path ${cfg} ${brokerOptions}
-}
+{{template "broker" .}}
 
 # if we are given an archive to use, load first, not required to exist
 # Note that we ask the user to dump in interactive mode - I am not