fix: do not force transient qa-physics files to be in qa-physics/…

… source directory (#123)

.gitignore

@@ -16,7 +16,6 @@ tmp
 outfiles
 
 # physics timelines
-qa-physics/outdat.*
 qa-physics/QA/qa
 qaTree*.json*
 chargeTree*.json*

README.md

@@ -14,7 +14,7 @@ To build,
 mvn package
 ```
 #### Additional Build Notes:
-- Use `mvn clean` if you want to clean build targets. 
+- Use `mvn clean` if you want to clean build targets.
 - Use the `-f` option of `mvn` to build individual submodules:
   1. [`monitoring`](monitoring): generates histograms for detectors
   1. [`detectors`](detectors): uses detector histograms to generate timelines
@@ -163,6 +163,15 @@ outfiles
     │   ├── 5001                      # for run number 5001
     │   └── ...
     │
+    ├── timeline_physics_qa           # transient files for the physics QA
+    │   ├── outdat
+    │   │   ├── qaTree.json           # QADB
+    │   │   ├── qaTreeFT.json         # QADB for FT only
+    │   │   ├── chargeTree.json       # FC charge info
+    │   │   └── data_table.dat        # combined data_table*.dat from each run
+    │   ├── outmon                    # timeline (and other) HIPO files
+    │   └── outmon.qa                 # QADB timelines
+    │
     ├── timeline_web_preQA            # detector timelines, before QA lines are drawn
     │   ├── htcc
     │   ├── ltcc

bin/run-physics-timelines.sh

@@ -50,6 +50,7 @@ inputDir=$($inputCmd $inputCmdOpts -I)
 [ -z "$outputDir" ] && outputDir=$(pwd -P)/outfiles/$dataset
 
 # set subdirectories
+qaDir=$outputDir/timeline_physics_qa
 finalDir=$outputDir/timeline_web
 logDir=$outputDir/log
 
@@ -84,22 +85,22 @@ function exe {
 }
 
 # organize the data into datasets
-exe ./datasetOrganize.sh $dataset $inputDir
+exe ./datasetOrganize.sh $dataset $inputDir $qaDir
 
 # produce chargeTree.json
-exe run-groovy $TIMELINE_GROOVY_OPTS buildChargeTree.groovy $dataset
+exe run-groovy $TIMELINE_GROOVY_OPTS buildChargeTree.groovy $qaDir
 
 # loop over datasets
 # trigger electrons monitor
-exe run-groovy $TIMELINE_GROOVY_OPTS qaPlot.groovy $dataset
-exe run-groovy $TIMELINE_GROOVY_OPTS qaCut.groovy $dataset
+exe run-groovy $TIMELINE_GROOVY_OPTS qaPlot.groovy $qaDir
+exe run-groovy $TIMELINE_GROOVY_OPTS qaCut.groovy $qaDir $dataset
 # FT electrons
-exe run-groovy $TIMELINE_GROOVY_OPTS qaPlot.groovy $dataset FT
-exe run-groovy $TIMELINE_GROOVY_OPTS qaCut.groovy $dataset FT
+exe run-groovy $TIMELINE_GROOVY_OPTS qaPlot.groovy $qaDir FT
+exe run-groovy $TIMELINE_GROOVY_OPTS qaCut.groovy $qaDir $dataset FT
 # general monitor
-exe run-groovy $TIMELINE_GROOVY_OPTS monitorPlot.groovy $dataset
+exe run-groovy $TIMELINE_GROOVY_OPTS monitorPlot.groovy $qaDir
 # move timelines to output area
-exe ./stageTimelines.sh $dataset $finalDir
+exe ./stageTimelines.sh $qaDir $finalDir
 
 popd
 

qa-physics/QA/import.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
-# copy qaTree.json from ../outdat.$dataset, so we can start the QA
-if [ $# -lt 1 ]; then
+# copy qaTree.json, so we can start the QA
+if [ $# -lt 2 ]; then
   echo """
-  USAGE: $0 [dataset] [optional: path to qaTree.json] [optional: options for parseQaTree.groovy]
+  USAGE: $0 [dataset] [path to qaTree.json] [optional: options for parseQaTree.groovy]
 
   - to see parseQaTree options: $0 [dataset] -h
                            and: $0 [dataset] -l
@@ -20,13 +20,14 @@ rm -r qa.${dataset}
 mkdir -p qa.${dataset}
 
 # parse arguments
-qatree=../outdat.${dataset}/qaTree.json # default
+qatree=""
 opts=""
 for opt in "$@"; do
   if [[ $opt =~ \.json$ ]]; then qatree=$opt
   else opts="$opts $opt"
   fi
 done
+[ -z "$qatree" ] && echo "ERROR: no qaTree.json file specified" && exit 100
 
 # import the JSON file, and symlink qa
 cp -v $qatree qa.${dataset}/qaTree.json

qa-physics/README.md

@@ -84,21 +84,19 @@ First step is to read DST or Skim files, producing HIPO files and data tables
       * DAQ-ungated FC charge at beginning of 5-file (minimum readout)
       * DAQ-ungated FC charge at end of 5-file (maximum readout)
       * average livetime
-    * `[output_directory]/monitor_${runnum}.hipo` contains several plots 
+    * monitoring HIPO file, `[output_directory]/monitor_${runnum}.hipo`, contains several plots 
       * there is one plot per 'segment' where a segment is a single DST file
         (5-file) or a set of 10000 events for skim files
 
 ### Data Organization
 * use the script `datasetOrganize.sh`
   * this will concatenate `dat` files from the input directory into a single file
-    `outdat.${dataset}/data_table.dat`, for the specified dataset
-  * it will also generate symlinks from `outmon.${dataset}/monitor*.hipo` to the
-    relevant HIPO files
+  * it will also generate symlinks to the relevant monitoring HIPO files
 
 
 ### Plotting Scripts
 * `monitorPlot.groovy`
-  * this will read `outmon.${dataset}/monitor*.hipo` files and produce several timelines
+  * this will read monitoring HIPO files and produce several timelines
     * Let X represent a quantity plotted in a timeline; the timeline plots the
       average value of X, versus run number
       * Several variables may be plotted together, on the same timeline plot,
@@ -112,7 +110,7 @@ First step is to read DST or Skim files, producing HIPO files and data tables
         * graph of the average value of X versus segment number
 
 * `qaPlot.groovy` 
-  * reads `outdat.${dataset}/data_table.dat` and generates `outmon.${dataset}/monitorElec.hipo`
+  * reads data table and generates `monitorElec.hipo`
     * within this HIPO file, there is one directory for each run, containing several
       plots:
       * `grA*`: N/F vs. file number (the `A` notation is so it appears first in the
@@ -153,7 +151,7 @@ generate QA timelines, and a `json` file which is used for the manual followup Q
   * Several other timelines are generated as well, such as standard deviation of 
     the number of electrons
 * `qaCut.groovy`
-  * reads `outmon.${dataset}/monitorElec.hipo`, along with `epochs/epochs.${dataset}.txt`, to build
+  * reads `monitorElec.hipo`, along with `epochs/epochs.${dataset}.txt`, to build
     timelines for the online monitor
   * if `$useFT` is set, it will use FT electrons instead
   * the runs are organized into epochs, wherein each:
@@ -164,7 +162,7 @@ generate QA timelines, and a `json` file which is used for the manual followup Q
     * QA cut lines are set using an interquartile range (IQR) rule: `cutFactor` * IQR,
       where `cutFactor` adjusts the overall width of the cuts
       * this is done for each sector individually
-      * results are stored in `outdat.${dataset}/qaTree.json`
+      * results are stored in `qaTree.json`
       * timelines HIPO files are also generated (which can be uploaded to the web server):
 
 ## Manual QA procedure
@@ -180,19 +178,15 @@ the timelines and recording features not identified by the automatic QA in
   * execute `getListOfDSTs.sh [dataset]` to obtain a list of run numbers and file
     numbers from the DST file directory; this script takes some time to run
   * execute `integrityCheck.sh [dataset]` to compare the list of DST files
-    to those which appear in `outdat.$dataset/data_table.dat`; if any DST files
+    to those which appear in `data_table.dat`; if any DST files
     do not appear in `data_table.dat`, the check will fail, the missing files
     will be printed, and it is recommended to re-run the automatic QA, either for
     those specific runs, or in its entirety
 * `cd QA`; this subdirectory contains code for the "manual QA"
-* `import.sh [dataset]` to import the automatically generated `qaTree.json`; this
+* `import.sh` to import the automatically generated `qaTree.json`; this
   will also generate a human-readable file, `qa/qaTable.dat`  
   * append the option `-cnds=user_comment` to copy the user comment from RCDB
     to `qa/qaTable.dat`, which is helpful for the manual QA procedure
-  * by default, the `json` file is in `../outdat.${dataset}/qaTree.json`;
-    you can also specify a path to a specific `qaTree.json` file; this is 
-    useful if you have a more up-to-date version somewhere else, and you
-    want to use the tools in this QA directory to make revisions
 * open `qa/qaTable.dat` in another window; this is the human-readable version of
   the imported `qaTree.json`
 * now scan through `qaTable.dat`, inspecting each run:
@@ -238,7 +232,6 @@ the timelines and recording features not identified by the automatic QA in
       document these cases with the Misc defect bit
 * after scanning through `qaTable.dat` and revising `qaTree.json`, return to the parent
 directory and call `exeQAtimelines.sh` to produce the updated QA timelines
-  * these QA timelines are stored in `outmon.${dataset}.qa`
   * it copies the revised`qaTree.json` (`QA/qa.${dataset}/qaTree.json`) to
     the new QA timeline directory, which can then be deployed to the web servers
   * this final `qaTree.json` is stored in the 

qa-physics/buildChargeTree.groovy

@@ -5,8 +5,11 @@ Tools T = new Tools()
 
 //----------------------------------------------------------------------------------
 // ARGUMENTS:
-def dataset = 'rga_inbending'
-if(args.length>=1) dataset = args[0]
+if(args.length<1) {
+  System.err.println "USAGE: run-groovy ${this.class.getSimpleName()}.groovy [INPUT_DIR]"
+  System.exit(101)
+}
+inDir = args[0] + "/outdat"
 //----------------------------------------------------------------------------------
 
 def tok
@@ -21,7 +24,7 @@ def ufcCharge
 def chargeTree = [:] // [runnum][filenum] -> charge
 
 // open data_table.dat
-def dataFile = new File("outdat.${dataset}/data_table.dat")
+def dataFile = new File("${inDir}/data_table.dat")
 if(!(dataFile.exists())) throw new Exception("data_table.dat not found")
 dataFile.eachLine { line ->
 
@@ -59,4 +62,4 @@ dataFile.eachLine { line ->
 
 chargeTree.each { chargeRun, chargeRunTree -> chargeRunTree.sort{it.key.toInteger()} }
 chargeTree.sort()
-new File("outdat.${dataset}/chargeTree.json").write(JsonOutput.toJson(chargeTree))
+new File("${inDir}/chargeTree.json").write(JsonOutput.toJson(chargeTree))

qa-physics/datasetOrganize.sh

@@ -2,18 +2,19 @@
 
 set -e
 
-if [ $# -ne 2 ];then echo "USAGE: $0 [dataset] [input_dir]" >&2; exit 101; fi
+if [ $# -ne 3 ];then echo "USAGE: $0 [dataset] [input_dir] [output_dir]" >&2; exit 101; fi
 dataset=$1
 inputDir=$2
+outputDir=$3
 
 if [ -z "$TIMELINESRC" ]; then
   echo "ERROR: please source environ.sh first" >&2
   exit 100
 fi
 
 # cleanup / generate new dataset subdirs
-OUTMON_DIR=$TIMELINESRC/qa-physics/outmon.${dataset}
-OUTDAT_DIR=$TIMELINESRC/qa-physics/outdat.${dataset}
+OUTMON_DIR=$outputDir/outmon
+OUTDAT_DIR=$outputDir/outdat
 for dir in $OUTMON_DIR $OUTDAT_DIR; do
   echo "clean $dir"
   mkdir -p $dir
@@ -26,5 +27,9 @@ for file in $(find $inputDir -name "monitor_*.hipo"); do
   ln -sv $file $OUTMON_DIR/
 done
 for file in $(find $inputDir -name "data_table_*.dat"); do
-  cat $file >> $OUTDAT_DIR/data_table.dat
+  cat $file >> $OUTDAT_DIR/data_table.dat.tmp
 done
+
+# be sure the data table is sorted
+sort -n -o $OUTDAT_DIR/data_table.dat{,.tmp}
+rm $OUTDAT_DIR/data_table.dat.tmp

qa-physics/docs/docDiagram.md

@@ -22,6 +22,8 @@ flowchart TB
 
 ## Flowchart
 
+Note: output directories `$output_dir` and `$qa_dir` are typically set by wrapper scripts, and may vary depending on how they are run.
+
 ```mermaid
 flowchart TB
 
@@ -35,33 +37,33 @@ flowchart TB
 
     subgraph "Automated by ../bin/run-physics-timelines.sh"
       datasetOrganize[datasetOrganize.sh]:::auto
-      outmonFiles{{outmon.$dataset/monitor_$run.hipo}}:::data
-      outdatFiles{{outdat.$dataset/data_table.dat}}:::data
+      outmonFiles{{$qa_dir/outmon/monitor_$run.hipo}}:::data
+      outdatFiles{{$qa_dir/outdat/data_table.dat}}:::data
       monitorReadOut --> datasetOrganize
       datasetOrganize --> outmonFiles
       datasetOrganize --> outdatFiles
       
       monitorPlot[monitorPlot.groovy]:::auto
-      timelineFiles{{outmon.$dataset/$timeline.hipo}}:::timeline
+      timelineFiles{{$qa_dir/outmon/$timeline.hipo}}:::timeline
       outmonFiles --> monitorPlot
       monitorPlot --> timelineFiles
 
       qaPlot[qaPlot.groovy]:::auto
       createEpochs[create or edit<br>epochs/epochs.$dataset.txt<br>see mkTree.sh]:::manual
-      monitorElec{{outmon.$dataset/monitorElec.hipo}}:::data
+      monitorElec{{$qa_dir/outmon/monitorElec.hipo}}:::data
       outdatFiles --> qaPlot
       outdatFiles --> createEpochs
       qaPlot --> monitorElec
 
       qaCut[qaCut.groovy]:::auto
-      qaTree{{outdat.$dataset/qaTree.json}}:::data
+      qaTree{{$qa_dir/outdat/qaTree.json}}:::data
       monitorElec --> qaCut
       createEpochs --> qaCut
       qaCut --> timelineFiles
       qaCut --> qaTree
 
       buildCT[buildChargeTree.groovy]:::auto
-      chargeTree{{outdat.$dataset/chargeTree.json}}:::data
+      chargeTree{{$qa_dir/outdat/chargeTree.json}}:::data
       stage0[stageTimelines.sh]:::auto
       outdatFiles --> buildCT
       buildCT --> chargeTree
@@ -94,8 +96,8 @@ flowchart TB
 
     subgraph Finalize
       exeQAtimelines[exeQAtimelines.sh]:::manual
-      qaTreeUpdated{{outdat.$dataset/qaTree.json}}:::data
-      qaTL{{outmon.$dataset.qa/$timeline.hipo}}:::timeline
+      qaTreeUpdated{{$qa_dir/outdat/qaTree.json}}:::data
+      qaTL{{$qa_dir/outmon.qa/$timeline.hipo}}:::timeline
       stage1[stageTimelines.sh]:::manual
       qaLoc --> exeQAtimelines
       exeQAtimelines --> qaTL

qa-physics/exeQAtimelines.sh

@@ -4,24 +4,25 @@
 
 if [ -z "$TIMELINESRC" ]; then source `dirname $0`/../bin/environ.sh; fi
 
-if [ $# -ne 1 ]; then
-  echo "USAGE: $0 [dataset]" >&2
+if [ $# -ne 2 ]; then
+  echo "USAGE: $0 [INPUT_DIR] [DATASET]" >&2
   exit 101
 fi
-dataset=$1
+inDir=$1
+dataset=$2
 
-qaDir=outmon.${dataset}.qa
+qaDir=$inDir/outmon.qa
 
 mkdir -p $qaDir
 rm -r $qaDir
 mkdir -p $qaDir
 
 for bit in {0..5} 100; do
-  run-groovy $TIMELINE_GROOVY_OPTS qaCut.groovy $dataset false $bit
-  qa=$(ls -t outmon.${dataset}/electron_trigger_*QA*.hipo | grep -v epoch | head -n1)
+  run-groovy $TIMELINE_GROOVY_OPTS qaCut.groovy $inDir $dataset false $bit
+  qa=$(ls -t $inDir/outmon/electron_trigger_*QA*.hipo | grep -v epoch | head -n1)
   mv $qa ${qaDir}/$(echo $qa | sed 's/^.*_QA_//g')
 done
 
 cp QA/qa.${dataset}/qaTree.json $qaDir
 echo ""
-cat outdat.${dataset}/passFractions.dat
+cat $inDir/outdat/passFractions.dat

qa-physics/getListOfDSTs.sh

@@ -29,5 +29,5 @@ while read dst; do
 done < dstfilelist.tmp
 rm dstfilelist.tmp
 
-cat dstlist.tmp | sort -n > dstlist.${dataset}.dat
+cat dstlist.tmp | sort -n > dstlist.dat
 rm dstlist.tmp

qa-physics/integrityCheck.sh

@@ -10,31 +10,31 @@ if [ -z "$TIMELINESRC" ]; then
   exit 100
 fi
 
-if [ $# -ne 1 ];then echo "USAGE: $0 [dataset]" >&2; exit 101; fi
-dataset=$1
+if [ $# -ne 1 ];then echo "USAGE: $0 [INPUT_DIR]" >&2; exit 101; fi
+inDir=$1
 
-if [ ! -f dstlist.${dataset}.dat ]; then
-  echo "ERROR: dstlist.${dataset}.dat does not exist" >&2
+if [ ! -f dstlist.dat ]; then
+  echo "ERROR: dstlist.dat does not exist" >&2
   echo "execute getListOfDSTs.sh first" >&2
   exit 100
 fi
 
-if [ ! -f outdat.${dataset}/data_table.dat ]; then
-  echo "ERROR: outdat.${dataset}/data_table.dat does not exist" >&2
+if [ ! -f $inDir/outdat/data_table.dat ]; then
+  echo "ERROR: $inDir/outdat/data_table.dat does not exist" >&2
   echo "execute ../bin/run-physics-timelines.sh first" >&2
   exit 100
 fi
 
-cat outdat.${dataset}/data_table.dat | awk '{print $1" "$2}' > qalist.tmp
+cat $inDir/outdat/data_table.dat | awk '{print $1" "$2}' > qalist.tmp
 cat qalist.tmp | sort -n | uniq > qalist.dat
 rm qalist.tmp
 
-diff dstlist.${dataset}.dat qalist.dat | tee diff.dat
+diff dstlist.dat qalist.dat | tee diff.dat
 nl=$(cat diff.dat|wc -l)
 echo "differences printed to diff.dat, which has $nl lines"
 if [ $nl -gt 0 ]; then
   echo "integrity check FAILED"
-  echo "execute vimdiff dstlist.${dataset}.dat qalist.dat"
+  echo "execute vimdiff dstlist.dat qalist.dat"
 else
   echo "integrity check PASSED"
 fi