project-codeflare · jbusche · Dec 5, 2023 · Dec 18, 2023
diff --git a/test/perf-test/kwokmcadperf.sh b/test/perf-test/kwokmcadperf.sh
@@ -10,7 +10,7 @@ function help() {
     echo
     echo "Preconditions: "
     echo "   - The script assumes you've logged into your cluster already. If not, it will tell you to login."
-    echo "   - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
+    echo "   - The script checks that you have the mcad-controller installed (or the newer CodeFlare Operator), otherwise it'll tell you to install it first."
     echo "   - The script checks that you have the kwok-controller installed, otherwise it'll tell you to install it first."
     echo
     echo "Options:"
@@ -40,12 +40,15 @@ function check_mcad_installed_status() {
     res2="$?"
     kubectl get crd |grep appwrapper &> /dev/null
     res3="$?"
+    kubectl get pod -A |grep codeflare-operator-manager &> /dev/null
+    res4="$?"
     set -e
     MCAD="$res2"
     CRD="$res3"
-      if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
+    CODEFLARE="$res4"
+      if [[ $MCAD == 1 ]] && [[ $CODEFLARE == 1 ]] || [[ $CRD == 1 ]]
       then
-        echo "You need Install MCAD Controller first before running this script"
+        echo "You need Install the MCAD Controller or the latest CodeFlare Operator first before running this script"
         exit 1
       else
         echo "Nice, MCAD Controller is installed"
@@ -95,65 +98,65 @@ echo
 check_kwok_installed_status
 
 echo
-read -p "How many fake KWOK appwrapper jobs do you want?" jobs
+read -p "How many fake KWOK appwrapper jobs do you want?" JOBS
 
 # Start the timer now
 SECONDS=0
 
-echo "jobs number is $jobs"
+echo "jobs number is $JOBS"
 export STARTTIME=`date +"%T"`
 echo " "
 echo "Jobs started at: $STARTTIME" |tee fake-job-$STARTTIME.log
 echo " "
 
-# This fixes the number of jobs to be one less so the for loop gets the right amount
-((realjobs=$jobs-1))
-
-for num in $(eval echo "{0.."$realjobs"}")
+COUNTER=1
+while [ $COUNTER -le $JOBS ]
 do
-    next_num=$(($num + 1))
-    echo "Submitting job $next_num"
+    ORIG_COUNTER=$(($COUNTER - 1))
+    echo "Submitting job $COUNTER"
+
 # Had to do this OSTYPE because sed acts differently on Linux versus Mac
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
-      darwin*) 
-        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
-      *) 
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+      darwin*)
+        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+      *)
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
     esac
     kubectl apply -f ${SCRIPT_DIR}/preempt-exp-kwok.yaml
+COUNTER=$[$COUNTER +1]
 done
 
     # Let's reset the original preempt-exp-kwok.yaml file back to original value 
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
       darwin*) 
-        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
       *) 
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
     esac
 
 # Check for all jobs to report complete
-jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
+JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
 
-while [ $jobstatus -lt $jobs ]
+while [ $JOBSTATUS -lt $JOBS ]
 do
-   echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs
+   echo "Number of completed jobs is: " $JOBSTATUS " and the goal is: " $JOBS
    sleep 10
-   jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
+   JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
 done
 
 echo " "
 export FINISHTIME=`date +"%T"`
-echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
-echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
+echo "All $JOBSTATUS jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
+echo "Total amount of time for $JOBS appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
 echo " "
-echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log"
+echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$JOBS-$STARTTIME.log"
 
 # Rename the log to show the number of jobs used
-mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log
+mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$JOBS-$STARTTIME.log
 
 #Ask if you want to auto-cleanup the appwrapper jobs
 echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"

diff --git a/test/perf-test/nodes.sh b/test/perf-test/nodes.sh
@@ -65,51 +65,49 @@ echo "Checking whether we have a valid cluster login or not..."
 check_kubectl_login_status
 
 # Track whether you have the KWOK controller installed
-echo "Checking MCAD Controller installation status"
+echo "Checking KWOK Controller installation status"
 echo
 check_kwok_installed_status
 
 echo
-read -p "How many simulated KWOK nodes do you want?" nodes
+read -p "How many simulated KWOK nodes do you want?" NODES
 
-echo "Nodes number is $nodes"
+echo "Nodes number is $NODES"
 echo " "
 
-# This fixes the number of jobs to be one less so the for loop gets the right amount
-((realnodes=$nodes-1))
-echo "The real number of nodes is $realnodes"
-
-for num in $(eval echo "{0.."$realnodes"}")
+COUNTER=1
+while [ $COUNTER -le $NODES ]
 do
-    next_num=$(($num + 1))
-    echo "Submitting node $next_num"
+    ORIG_COUNTER=$(($COUNTER - 1))
+    echo "Submitting node $COUNTER"
 # Had to do this OSTYPE because sed acts differently on Linux versus Mac
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
-      darwin*) 
-        sed -i '' "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
-      *) 
-        sed -i "/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i "s/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ;;
+      darwin*)
+        sed -i '' "s/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
+      *)
+        sed -i "/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ;;
     esac
     kubectl apply -f ${SCRIPT_DIR}/node.yaml
+COUNTER=$[$COUNTER +1]
 done
 
     # Let's reset the original node.yaml file back to original value 
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
       darwin*) 
-        sed -i '' "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i '' "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
       *) 
-        sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
     esac
 
 # Check for all nodes to report complete
 echo "Waiting until all the simualted pods become ready:"
 kubectl wait --for=condition=Ready nodes --selector type=kwok --timeout=600s
 echo " "
-echo "Total amount of simulated nodes requested is: $nodes"
+echo "Total amount of simulated nodes requested is: $NODES"
 echo "Total number of created nodes is: "`kubectl get nodes --selector type=kwok -o name |wc -l`
 kubectl get nodes --selector type=kwok
 

diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh
@@ -9,7 +9,7 @@ function help() {
     echo
     echo "Preconditions: "
     echo "   - The script assumes you've logged into your cluster already. If not, it will tell you to login."
-    echo "   - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
+    echo "   - The script checks that you have the mcad-controller installed (or the newer CodeFlare Operator), otherwise it'll tell you to install it first."
     echo
     echo "Options:"
     echo "  -h       Print this help message"
@@ -38,12 +38,15 @@ function check_mcad_installed_status() {
     res2="$?"
     kubectl get crd |grep appwrapper &> /dev/null
     res3="$?"
+    kubectl get pod -A |grep codeflare-operator-manager &> /dev/null
+    res4="$?"
     set -e
     MCAD="$res2"
     CRD="$res3"
-      if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
+    CODEFLARE="$res4"
+      if [[ $MCAD == 1 ]] && [[ $CODEFLARE == 1 ]] || [[ $CRD == 1 ]]
       then
-        echo "You need Install MCAD Controller first before running this script"
+        echo "You need Install the MCAD Controller or the latest CodeFlare Operator first before running this script"
         exit 1
       else
         echo "Nice, MCAD Controller is installed"
@@ -70,68 +73,68 @@ check_kubectl_login_status
 # Track whether you have the MCAD controller installed
 echo "Checking MCAD Controller installation status"
 echo
-#check_mcad_installed_status
+check_mcad_installed_status
 
 echo
-read -p "How many appwrapper jobs do you want?" jobs
+read -p "How many appwrapper jobs do you want?" JOBS
 
 # Start the timer now
 SECONDS=0
 
-echo "Appwrapper number is $jobs"
+echo "Appwrapper number is $JOBS"
 export STARTTIME=`date +"%T"`
 echo " "
 echo "Appwrappers started at: $STARTTIME" |tee job-$STARTTIME.log
 echo " "
 
-# This fixes the number of jobs to be one less so the for loop gets the right amount
-((realjobs=$jobs-1))
-
-for num in $(eval echo "{0.."$realjobs"}")
+COUNTER=1
+while [ $COUNTER -le $JOBS ]
 do
-    next_num=$(($num + 1))
-    echo "Submitting job $next_num"
+    ORIG_COUNTER=$(($COUNTER - 1))
+    echo "Submitting job $COUNTER"
+
 # Had to do this OSTYPE because sed acts differently on Linux versus Mac
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
-      darwin*) 
-        sed -i '' "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
-      *) 
-        sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+      darwin*)
+        sed -i '' "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+      *)
+        sed -i "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
     esac
     kubectl apply -f ${SCRIPT_DIR}/preempt-exp.yaml
+COUNTER=$[$COUNTER +1]
 done
 
     # Let's reset the original preempt-exp.yaml file back to original value 
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
       darwin*) 
-        sed -i '' "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i '' "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
       *) 
-        sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
     esac
 
 # Check for all appwrappers to report complete
-jobstatus=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
+JOBSTATUS=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
 
-while [ $jobstatus -lt $jobs ]
+while [ $JOBSTATUS -lt $JOBS ]
 do
-   echo "Number of completed appwrappers is: " $jobstatus " and the goal is: " $jobs
+   echo "Number of completed appwrappers is: " $JOBSTATUS " and the goal is: " $JOBS
    sleep 10
-   jobstatus=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
+   JOBSTATUS=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
 done
 
 echo " "
 export FINISHTIME=`date +"%T"`
-echo "All $jobstatus appwrappers finished: $FINISHTIME" |tee -a job-$STARTTIME.log
-echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log
+echo "All $JOBSTATUS appwrappers finished: $FINISHTIME" |tee -a job-$STARTTIME.log
+echo "Total amount of time for $JOBS appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log
 echo " "
-echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log"
+echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$JOBS-$STARTTIME.log"
 
 # Rename the log to show the number of jobs used
-mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log
+mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$JOBS-$STARTTIME.log
 
 #Ask if you want to auto-cleanup the appwrapper jobs
 echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"

diff --git a/test/perf-test/preempt-exp-kwok.yaml b/test/perf-test/preempt-exp-kwok.yaml
@@ -1,7 +1,7 @@
 apiVersion: workload.codeflare.dev/v1beta1
 kind: AppWrapper
 metadata:
-  name: fake-defaultaw-schd-spec-with-timeout-1
+  name: fake-defaultaw-schd-spec-with-timeout-0
   namespace: default
 spec:
   schedulingSpec:
@@ -11,7 +11,6 @@ spec:
         growthType: "exponential" 
   priority: 9
   resources:
-    Items: []
     GenericItems:
     - replicas: 1
       completionstatus: Complete
@@ -30,15 +29,15 @@ spec:
         kind: Job
         metadata:
           namespace: default
-          name: fake-defaultaw-schd-spec-with-timeout-1
+          name: fake-defaultaw-schd-spec-with-timeout-0
         spec:
           parallelism: 1
           completions: 1
           template:
             metadata:
               namespace: default
               labels:
-                appwrapper.mcad.ibm.com: "fake-defaultaw-schd-spec-with-timeout-1"
+                appwrapper.mcad.ibm.com: "fake-defaultaw-schd-spec-with-timeout-0"
             spec:
               affinity:
                 nodeAffinity:
@@ -56,6 +55,6 @@ spec:
                   operator: "Exists"
                   effect: "NoSchedule"
               containers:
-              - name: fake-defaultaw-schd-spec-with-timeout-1
+              - name: fake-defaultaw-schd-spec-with-timeout-0
                 image: fake-image
               restartPolicy: Never
diff --git a/test/perf-test/preempt-exp.yaml b/test/perf-test/preempt-exp.yaml
@@ -1,7 +1,7 @@
 apiVersion: workload.codeflare.dev/v1beta1
 kind: AppWrapper
 metadata:
-  name: defaultaw-schd-spec-with-timeout-1
+  name: defaultaw-schd-spec-with-timeout-0
   namespace: default
 spec:
   schedulingSpec:
@@ -29,20 +29,20 @@ spec:
         kind: Job
         metadata:
           namespace: default
-          name: defaultaw-schd-spec-with-timeout-1
+          name: defaultaw-schd-spec-with-timeout-0
           labels:
-            appwrapper.mcad.ibm.com: defaultaw-schd-spec-with-timeout-1
+            appwrapper.mcad.ibm.com: defaultaw-schd-spec-with-timeout-0
         spec:
           parallelism: 1
           completions: 1
           template:
             metadata:
               namespace: default
               labels:
-                appwrapper.mcad.ibm.com: "defaultaw-schd-spec-with-timeout-1"
+                appwrapper.mcad.ibm.com: "defaultaw-schd-spec-with-timeout-0"
             spec:
               containers:
-              - name: defaultaw-schd-spec-with-timeout-1
+              - name: defaultaw-schd-spec-with-timeout-0
                 image: ubi8-minimal:latest
                 command: [ "/bin/bash", "-c", "--" ]
                 args: [ "sleep 10" ]