Parameter sweep

Added HPL params, template, and slurm script for

hopper/gcc-11/param_sweep/HPL_params.csv

@@ -0,0 +1,2 @@
+192, ANOTHER_PARAM, PARAM3
+256, ANOTHER_PARAM, PARAM3

hopper/gcc-11/param_sweep/HPL_results.csv

@@ -0,0 +1,2 @@
+256, 5.6687e+02
+192, 5.4575e+02

hopper/gcc-11/param_sweep/HPL_template.dat

@@ -0,0 +1,31 @@
+HPLinpack benchmark input file
+Innovative Computing Laboratory, University of Tennessee
+HPL.out      output file name (if any)
+1            device out (6=stdout,7=stderr,file)
+1            # of problems sizes (N)
+57600	     Ns # Appropriate for 16GB RAM
+1            # of NBs
+<BLOCK_SIZE>	     NBs
+0            PMAP process mapping (0=Row-,1=Column-major)
+1            # of process grids (P x Q)
+2            Ps
+8            Qs
+16.0         threshold
+1            # of panel fact
+2            PFACTs (0=left, 1=Crout, 2=Right)
+1            # of recursive stopping criterium
+4            NBMINs (>= 1)
+1            # of panels in recursion
+2            NDIVs
+1            # of recursive panel fact.
+2            RFACTs (0=left, 1=Crout, 2=Right)
+1            # of broadcast
+2            BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)
+1            # of lookahead depth
+1            DEPTHs (>=0)
+1            SWAP (0=bin-exch,1=long,2=mix)
+64           swapping threshold
+0            L1 in (0=transposed,1=no-transposed) form
+0            U  in (0=transposed,1=no-transposed) form
+1            Equilibration (0=no,1=yes)
+8            memory alignment in double (> 0)

hopper/gcc-11/param_sweep/parameter_sweep_array.slurm

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+#SBATCH --job-name HPL_ParamSweep
+#SBATCH --partition general
+#SBATCH --time 00:10:00
+#SBATCH --nodes 2
+#SBATCH --ntasks-per-node 8
+#SBATCH --mem 16GB
+#SBATCH --array 1-2
+
+# Load the required packages (gcc 11 and HPL)
+module load gcc/11.2.0-655h hpl
+
+export OMP_PROC_BIND=TRUE
+export OMP_PLACES=cores
+export OMP_NUM_THREADS=4
+
+# Set a place to record the results
+RESULTS_FILE=$SLURM_SUBMIT_DIR/HPL_results.csv
+
+PARAMS_FILE=$SLURM_SUBMIT_DIR/HPL_params.csv
+TEMPLATE_FILE=$SLURM_SUBMIT_DIR/HPL_template.dat
+
+# Check for errors
+if test -f $PARAMS_FILE; then
+    echo Using parameter file $PARAMS_FILE
+else
+    echo Error $PARAMS_FILE not found
+    exit 1
+fi
+
+if test -f $TEMPLATE_FILE; then
+    echo Using template file $TEMPLATE_FILE
+else
+    echo Error $TEMPLATE_FILE not found
+    exit 2
+fi
+
+# Get the Nth line from our parameter file - where N is the array ID
+PARAMS=$(head -n $SLURM_ARRAY_TASK_ID $PARAMS_FILE | tail -n 1)
+echo Read param line $SLURM_ARRAY_TASK_ID: $PARAMS
+
+# Get the Xth element of that line (comma separated)
+BLOCK_SIZE=$(echo $PARAMS | awk -F"," '{print $1}') # awk: Aho, Weinberger, and Kernighan
+echo Read param BLOCK_SIZE: $BLOCK_SIZE
+
+# Create a new working directory for each instance of xhpl since it needs it expects it's own HPL.dat
+SCRATCH_DIR=/carc/scratch/users/$USER
+
+# Make a temporary directory for our work - we will delete this at the end
+TMP_DIR=$(mktemp --directory -p $SCRATCH_DIR)
+echo Temp directory: $TMP_DIR
+
+# Make a subdirectory with the SLURM array task id to make debugging easier
+TMP_WORKING_DIR=$TMP_DIR/$SLURM_ARRAY_TASK_ID
+mkdir -p $TMP_WORKING_DIR
+echo Created temporary working directory: $TMP_WORKING_DIR
+
+# Make the new working directory the current directory so xhpl runs in there
+cd $TMP_WORKING_DIR
+echo Now running in $PWD
+
+# Substitute the parameter value read from the PARAMS file above into the HLP.dat for this instance
+# Copy the template file into the temp working directory
+cp $TEMPLATE_FILE $TMP_WORKING_DIR/HPL.dat
+# Substitute the new block size in for the string <BLOCK_SIZE>
+sed -i "s/<BLOCK_SIZE>/$BLOCK_SIZE/g" HPL.dat # sed: string edit
+
+echo Running xhpl in $TMP_WORKING_DIR...
+srun --mpi=pmi2 xhpl
+echo xhpl finished
+
+# The HPL.dat file tells xhpl to write to HPL.out.
+# Extract the throughput with grep and awk
+
+# 1. Find the line containing Gflops and print it and the following line
+RESULT_HEADER_AND_DATA_LINES=$(grep --after 2 Gflops HPL.out)
+echo Results: $RESULT_HEADER_AND_DATA_LINES
+
+# 2. Get just the data line
+RESULT_DATA_LINE=$(echo $RESULT_HEADER_AND_DATA_LINES | tail -n 1)
+echo Results data: $RESULT_DATA_LINE
+
+# 3. Get the last field in the data line, that's the Gigaflops.
+GFLOPS=$(echo $RESULT_DATA_LINE | awk -F" " '{print $NF}')
+echo Results Gflops: $GFLOPS 
+
+echo Writing input parameters and gflops to $RESULTS_FILE
+echo $BLOCK_SIZE, $GFLOPS >> $RESULTS_FILE
+
+# Clean up the temporary working directory
+rm -r $TMP_DIR
+echo Deleted $TMP_DIR