TRD raw reader update QC statistics

DataFormats/Detectors/TRD/CMakeLists.txt

@@ -45,7 +45,8 @@ o2_target_root_dictionary(DataFormatsTRD
                        include/DataFormatsTRD/KrCluster.h
                        include/DataFormatsTRD/KrClusterTriggerRecord.h
                        include/DataFormatsTRD/NoiseCalibration.h
-                        include/DataFormatsTRD/PHData.h
+                       include/DataFormatsTRD/PHData.h
+                       include/DataFormatsTRD/RawDataStats.h
                        include/DataFormatsTRD/CTF.h
                        include/DataFormatsTRD/CalVdriftExB.h
                        include/DataFormatsTRD/CalGain.h

DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h

@@ -16,13 +16,11 @@
 #ifndef O2_TRD_RAWDATASTATS
 #define O2_TRD_RAWDATASTATS
 
-#include "TObject.h"
+#include "Rtypes.h"
 #include <string>
 #include <cstdint>
 #include <array>
-#include <chrono>
 #include <unordered_map>
-#include <gsl/span>
 #include "DataFormatsTRD/Constants.h"
 
 namespace o2::trd
@@ -123,59 +121,44 @@ enum OptionBits {
   TRDSortDigits
 }; // this is currently 16 options, the array is 16, if you add here you need to change the 16;
 
-//Data to be stored and accumulated on an event basis.
-//events are spread out with in the data coming in with a halfcruheader per event, per ... half cru.
-//this is looked up via the interaction record (orbit and bunchcrossing).
-//this permits averaging in the data that gets senton per timeframe
-struct TRDDataCountersPerEvent {
-  TRDDataCountersPerEvent() = default;
-  double mTimeTaken = 0.;             // time take to process an event (summed trackletparsing and digitparsing) parts not accounted for.
-  double mTimeTakenForDigits = 0.;    // time take to process tracklet data blocks [us].
-  double mTimeTakenForTracklets = 0.; // time take to process digit data blocks [us].
-  uint64_t mWordsRead = 0;            // words read in
-  uint64_t mWordsRejected = 0;        // words skipped for various reasons.
-  uint16_t mTrackletsFound = 0;       // tracklets found in the event
-  uint16_t mDigitsFound = 0;          // digits found in the event
-};
-
 //Data to be stored on a timeframe basis to then be sent as a message to be ultimately picked up by qc.
 //Some countes include a average over the numbers stored on a per event basis, e.g. digits per event.
 class TRDDataCountersPerTimeFrame
 {
  public:
-  std::array<uint8_t, o2::trd::constants::NSECTOR * 60> mLinkErrorFlag{};                              // status of the error flags for this timeframe, 8bit values from cru halfchamber header.
-  std::array<uint16_t, o2::trd::constants::NSECTOR * 60> mLinkNoData;                                  // Link had no data or was not present.
-  std::array<uint16_t, o2::trd::constants::NSECTOR * 60> mLinkWords{};                                 // units of 256bits, read from the cru half chamber header
-  std::array<uint16_t, o2::trd::constants::NSECTOR * 60> mLinkWordsRead{};                             // units of 32 bits the data words read before dumping or finishing
-  std::array<uint16_t, o2::trd::constants::NSECTOR * 60> mLinkWordsRejected{};                         // units of 32 bits the data dumped due to some or other error
+  std::array<uint8_t, constants::MAXHALFCHAMBER> mLinkErrorFlag{};                                     // status of the error flags for this timeframe, 8bit values from cru halfchamber header.
+  std::array<uint16_t, constants::MAXHALFCHAMBER> mLinkNoData;                                         // Link had no data or was not present.
+  std::array<uint16_t, constants::MAXHALFCHAMBER> mLinkWords{};                                        // units of 256bits, read from the cru half chamber header
+  std::array<uint16_t, constants::MAXHALFCHAMBER> mLinkWordsRead{};                                    // units of 32 bits the data words read before dumping or finishing
+  std::array<uint16_t, constants::MAXHALFCHAMBER> mLinkWordsRejected{};                                // units of 32 bits the data dumped due to some or other error
+  std::array<uint16_t, constants::MAXHALFCHAMBER> mParsingOK{};                                        // count how often given link could be parsed without any errors
   std::array<uint16_t, TRDLastParsingError> mParsingErrors{};                                          // errors in parsing, indexed by enum above of ParsingErrors
-  std::array<uint32_t, o2::trd::constants::NSECTOR * 60 * TRDLastParsingError> mParsingErrorsByLink{}; // errors in parsing, indexed by enum above of ParsingErrors
-  uint16_t mDigitsPerEvent;                                                                                                  // average digits found per event in this timeframe, ignoring the no digit events where there is no calibration trigger.
-  uint16_t mTrackletsPerEvent;                                                                                               // average tracklets found per event in this timeframe
-  double mTimeTaken;                                                                                   // time taken to process the entire timeframe [ms].
-  double mTimeTakenForDigits;                                                                          // time take to process tracklet data blocks [ms].
-  double mTimeTakenForTracklets;                                                                       // time take to process digit data blocks [ms].
+  std::vector<uint32_t> mParsingErrorsByLink{};                                                        // each entry is for a single parsing error on a given link (HCID * number of Errors + error index)
+  float mTimeTaken;                                                                                    // time taken to process all half-CRU data blocks combined [us].
+  float mTimeTakenForDigits;                                                                           // time take to process tracklet data blocks [us].
+  float mTimeTakenForTracklets;                                                                        // time take to process digit data blocks [us].
   uint32_t mDigitsFound;                                                                               // digits found in the time frame.
   uint32_t mTrackletsFound;                                                                            // tracklets found in the time frame.
-  std::array<uint64_t, 256> mDataFormatRead{};                                                         // We just keep the major version number
+  uint16_t mNTriggersCalib;                                                                            // number of triggers with digit readout
+  uint16_t mNTriggersTotal;                                                                            // total number of triggers
+  std::array<int, 256> mDataFormatRead{};                                                              // We just keep the major version number
   void clear()
   {
     mLinkNoData.fill(0);
     mLinkWords.fill(0);
     mLinkWordsRead.fill(0);
     mLinkWordsRejected.fill(0);
+    mParsingOK.fill(0);
     mParsingErrors.fill(0);
-    mParsingErrorsByLink.fill(0);
-    mDigitsPerEvent = 0;
-    mTrackletsPerEvent = 0;
+    mParsingErrorsByLink.clear();
     mTimeTaken = 0;
     mTimeTakenForDigits = 0;
     mTimeTakenForTracklets = 0;
     mDigitsFound = 0;
-    mTrackletsFound = 0; //tracklets found in timeframe.
+    mTrackletsFound = 0;
     mDataFormatRead.fill(0);
   };
-  ClassDefNV(TRDDataCountersPerTimeFrame, 1); // primarily for serialisation so we can send this as a message in o2
+  ClassDefNV(TRDDataCountersPerTimeFrame, 2); // primarily for serialisation so we can send this as a message in o2
 };
 
 } // namespace o2::trd

DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h

@@ -43,6 +43,7 @@
 #pragma link C++ class o2::trd::ChannelInfo + ;
 #pragma link C++ class o2::trd::ChannelInfoContainer + ;
 #pragma link C++ struct o2::trd::PHData + ;
+#pragma link C++ class o2::trd::TRDDataCountersPerTimeFrame + ;
 #pragma link C++ class std::vector < o2::trd::Tracklet64> + ;
 #pragma link C++ class std::vector < o2::trd::CalibratedTracklet> + ;
 #pragma link C++ class std::vector < o2::trd::TrackTriggerRecord> + ;

Detectors/TRD/reconstruction/include/TRDReconstruction/EventRecord.h

@@ -46,31 +46,30 @@ class EventRecord
 
   const std::vector<Digit>& getDigits() const { return mDigits; }
   const std::vector<Tracklet64>& getTracklets() const { return mTracklets; }
+  bool getIsCalibTrigger() const { return mIsCalibTrigger; }
+  float getTotalTime() const { return mTimeTaken; }
+  float getDigitTime() const { return mTimeTakenForDigits; }
+  float getTrackletTime() const { return mTimeTakenForTracklets; }
 
   // needed, in order to check if a trigger already exist for this bunch crossing
   bool operator==(const EventRecord& o) const { return mBCData == o.mBCData; }
 
   // sort the tracklets (and optionally digits) by detector, pad row, pad column
   void sortData(bool sortDigits);
 
-  //statistics stuff these get passed to the per tf data at the end of the timeframe,
-  //but as we read in per link, events are seperated hence these counters
-  const TRDDataCountersPerEvent& getEventStats() const { return mEventStats; }
-  void incTrackletTime(double timeadd) { mEventStats.mTimeTakenForTracklets += timeadd; }
-  void incDigitTime(double timeadd) { mEventStats.mTimeTakenForDigits += timeadd; }
-  void incTime(double duration) { mEventStats.mTimeTaken += duration; }
-  void incWordsRead(int count) { mEventStats.mWordsRead += count; }         // words read in
-  void incWordsRejected(int count) { mEventStats.mWordsRejected += count; } // words read in
-  void incTrackletsFound(int count) { mEventStats.mTrackletsFound += count; }
-  void incDigitsFound(int count) { mEventStats.mDigitsFound += count; }
-  // OS: Do we need to keep event statistics at all? Are they not anyhow accumulated for a whole TF?
-  //     or are they also used on a per event basis somewhere?
+  void incTrackletTime(float timeadd) { mTimeTakenForTracklets += timeadd; }
+  void incDigitTime(float timeadd) { mTimeTakenForDigits += timeadd; }
+  void incTime(float duration) { mTimeTaken += duration; }
+  void setIsCalibTrigger() { mIsCalibTrigger = true; }
 
  private:
   BCData mBCData;                       /// orbit and Bunch crossing data of the physics trigger
   std::vector<Digit> mDigits{};         /// digit data, for this event
   std::vector<Tracklet64> mTracklets{}; /// tracklet data, for this event
-  TRDDataCountersPerEvent mEventStats{}; /// statistics, for this trigger
+  float mTimeTaken = 0.;                // total parsing time [us] (including digit and tracklet parsing time)
+  float mTimeTakenForDigits = 0.;       // time take to process tracklet data blocks [us].
+  float mTimeTakenForTracklets = 0.;    // time take to process digit data blocks [us].
+  bool mIsCalibTrigger = false;         // flag calibration trigger
 };
 
 /// \class EventRecordContainer
@@ -87,11 +86,7 @@ class EventRecordContainer
   void setCurrentEventRecord(const InteractionRecord& ir);
   EventRecord& getCurrentEventRecord() { return mEventRecords.at(mCurrEventRecord); }
 
-  //statistics to keep
-  void incTrackletTime(double timeadd) { mTFStats.mTimeTakenForTracklets += timeadd; }
-  void incDigitTime(double timeadd) { mTFStats.mTimeTakenForDigits += timeadd; }
-  void incTrackletsFound(int count) { mTFStats.mTrackletsFound += count; }
-  void incDigitsFound(int count) { mTFStats.mDigitsFound += count; }
+  // statistics to keep
   void incLinkErrorFlags(int hcid, unsigned int flag) { mTFStats.mLinkErrorFlag[hcid] |= flag; }
   void incLinkNoData(int hcid) { mTFStats.mLinkNoData[hcid]++; }
   void incLinkWords(int hcid, int count) { mTFStats.mLinkWords[hcid] += count; }
@@ -103,7 +98,11 @@ class EventRecordContainer
   {
     mTFStats.mParsingErrors[error]++;
     if (hcid >= 0) { // hcid==-1 is reserved for those errors where we don't have the corresponding link ID
-      mTFStats.mParsingErrorsByLink[hcid * TRDLastParsingError + error]++;
+      if (error == NoError) {
+        mTFStats.mParsingOK[hcid]++;
+      } else {
+        mTFStats.mParsingErrorsByLink.push_back(hcid * TRDLastParsingError + error);
+      }
     }
   }
   void reset();

Detectors/TRD/reconstruction/src/CruRawReader.cxx

@@ -480,6 +480,9 @@ bool CruRawReader::processHalfCRU(int iteration)
     mIR.bc -= o2::ctp::TriggerOffsetsParam::Instance().LM_L0;
   }
   mEventRecords.setCurrentEventRecord(mIR);
+  if (mCurrentHalfCRUHeader.EventType == ETYPECALIBRATIONTRIGGER) {
+    mEventRecords.getCurrentEventRecord().setIsCalibTrigger();
+  }
 
   //loop over links
   uint32_t linksizeAccum32 = 0;     // accumulated size of all links in 32-bit words
@@ -516,7 +519,7 @@ bool CruRawReader::processHalfCRU(int iteration)
       }
       int trackletWordsRejected = 0;
       int trackletWordsRead = parseTrackletLinkData(currentlinksize32, halfChamberId, trackletWordsRejected);
-      std::chrono::duration<double, std::micro> trackletparsingtime = std::chrono::high_resolution_clock::now() - trackletparsingstart;
+      std::chrono::duration<float, std::micro> trackletparsingtime = std::chrono::high_resolution_clock::now() - trackletparsingstart;
       if (trackletWordsRead == -1) {
         // something went wrong bailout of here.
         mHBFoffset32 = hbfOffsetTmp + linksizeAccum32;
@@ -529,24 +532,19 @@ bool CruRawReader::processHalfCRU(int iteration)
       mHBFoffset32 += trackletWordsRead;
       if (mCurrentHalfCRUHeader.EventType == ETYPEPHYSICSTRIGGER &&
           endOfCurrentLink - mHBFoffset32 >= 8) {
-        /*
-        // disabled for the same reason as for the warning after the digits parsing below
         if (mMaxWarnPrinted > 0) {
           LOGF(warn, "After successfully parsing the tracklet data for link %i there are %u words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32);
           checkNoWarn();
         }
-        */
         incrementErrors(UnparsedTrackletDataRemaining, halfChamberId, fmt::format("On link {} there are {} words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32));
         linkOK = false;
       }
-      mEventRecords.getCurrentEventRecord().incTrackletTime((double)std::chrono::duration_cast<std::chrono::microseconds>(trackletparsingtime).count());
+      mEventRecords.getCurrentEventRecord().incTrackletTime(trackletparsingtime.count());
       if (mOptions[TRDVerboseBit]) {
         LOGF(info, "Read %i tracklet words and rejected %i words", trackletWordsRead, trackletWordsRejected);
       }
       mTrackletWordsRejected += trackletWordsRejected;
       mTrackletWordsRead += trackletWordsRead;
-      mEventRecords.getCurrentEventRecord().incWordsRead(trackletWordsRead);
-      mEventRecords.getCurrentEventRecord().incWordsRejected(trackletWordsRejected);
       mEventRecords.incLinkWordsRead(halfChamberId, trackletWordsRead);
       mEventRecords.incLinkWordsRejected(halfChamberId, trackletWordsRejected);
 
@@ -587,7 +585,7 @@ bool CruRawReader::processHalfCRU(int iteration)
           auto digitsparsingstart = std::chrono::high_resolution_clock::now();
           int digitWordsRejected = 0;
           int digitWordsRead = parseDigitLinkData(endOfCurrentLink - mHBFoffset32, halfChamberId, digitWordsRejected);
-          std::chrono::duration<double, std::micro> digitsparsingtime = std::chrono::high_resolution_clock::now() - digitsparsingstart;
+          std::chrono::duration<float, std::micro> digitsparsingtime = std::chrono::high_resolution_clock::now() - digitsparsingstart;
           if (digitWordsRead == -1) {
             // something went wrong bailout of here.
             mHBFoffset32 = hbfOffsetTmp + linksizeAccum32;
@@ -597,23 +595,17 @@ bool CruRawReader::processHalfCRU(int iteration)
           if (endOfCurrentLink - mHBFoffset32 >= 8) {
             // check if some data is lost (probably due to bug in CRU user logic)
             // we should have max 7 padding words to align the link to 256 bits
-            /*
-            // due to the current CRU bug this is almost always the case
-            // TODO enable warning again when CRU UL is fixed
             if (mMaxWarnPrinted > 0) {
               LOGF(warn, "After successfully parsing the digit data for link %i there are %u words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32);
               checkNoWarn();
             }
-            */
             incrementErrors(UnparsedDigitDataRemaining, halfChamberId, fmt::format("On link {} there are {} words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32));
             linkOK = false;
           }
           if (digitWordsRejected > 0) {
             linkOK = false;
           }
-          mEventRecords.getCurrentEventRecord().incDigitTime((double)std::chrono::duration_cast<std::chrono::microseconds>(digitsparsingtime).count());
-          mEventRecords.getCurrentEventRecord().incWordsRead(digitWordsRead);
-          mEventRecords.getCurrentEventRecord().incWordsRejected(digitWordsRejected);
+          mEventRecords.getCurrentEventRecord().incDigitTime(digitsparsingtime.count());
           mEventRecords.incLinkWordsRead(halfChamberId, digitWordsRead);
           mEventRecords.incLinkWordsRejected(halfChamberId, digitWordsRejected);
 
@@ -648,7 +640,7 @@ bool CruRawReader::processHalfCRU(int iteration)
   //extract the vectors and copy them to tracklets and digits here, building the indexing(triggerrecords)
   //as this is for a single cru half chamber header all the tracklets and digits are for the same trigger defined by the bc and orbit in the rdh which we hold in mIR
 
-  std::chrono::duration<double, std::milli> cruparsingtime = std::chrono::high_resolution_clock::now() - crustart;
+  std::chrono::duration<float, std::micro> cruparsingtime = std::chrono::high_resolution_clock::now() - crustart;
   mEventRecords.getCurrentEventRecord().incTime(cruparsingtime.count());
 
   //if we get here all is ok.
@@ -801,7 +793,6 @@ int CruRawReader::parseDigitLinkData(int maxWords32, int hcid, int& wordsRejecte
             break;
           }
           mEventRecords.getCurrentEventRecord().addDigit(Digit(hcid / 2, (int)mcmHeader.rob, (int)mcmHeader.mcm, iChannel, adcValues, mPreTriggerPhase));
-          mEventRecords.getCurrentEventRecord().incDigitsFound(1);
           ++mDigitsFound;
         } // end active channel
       }   // end channel loop
@@ -939,7 +930,6 @@ int CruRawReader::parseTrackletLinkData(int linkSize32, int& hcid, int& wordsRej
         TrackletMCMData mcmData;
         mcmData.word = currWord;
         mEventRecords.getCurrentEventRecord().addTracklet(assembleTracklet64(hcHeader.format, mcmHeader, mcmData, iCpu, hcid));
-        mEventRecords.getCurrentEventRecord().incTrackletsFound(1);
         ++numberOfTrackletsFound;
         ++mTrackletsFound;
         addedTracklet = true;

Detectors/TRD/reconstruction/src/EventRecord.cxx

@@ -88,27 +88,16 @@ void EventRecordContainer::sendData(o2::framework::ProcessingContext& pc, bool g
 
 void EventRecordContainer::accumulateStats()
 {
-  int eventcount = mEventRecords.size();
-  int sumtracklets = 0;
-  int sumdigits = 0;
-  double sumdigittime = 0;
-  double sumtracklettime = 0;
-  double sumtime = 0;
-  uint64_t sumwordsrejected = 0;
-  uint64_t sumwordsread = 0;
-  for (auto event : mEventRecords) {
-    sumtracklets += event.getEventStats().mTrackletsFound;
-    sumdigits += event.getEventStats().mDigitsFound;
-    sumtracklettime += event.getEventStats().mTimeTakenForTracklets;
-    sumdigittime += event.getEventStats().mTimeTakenForDigits;
-    sumtime += event.getEventStats().mTimeTaken;
-  }
-  if (eventcount != 0) {
-    mTFStats.mTrackletsPerEvent = sumtracklets / eventcount;
-    mTFStats.mDigitsPerEvent = sumdigits / eventcount;
-    mTFStats.mTimeTakenForTracklets = sumtracklettime;
-    mTFStats.mTimeTakenForDigits = sumdigittime;
-    mTFStats.mTimeTaken = sumtime;
+  mTFStats.mNTriggersTotal = mEventRecords.size();
+  for (const auto& event : mEventRecords) {
+    mTFStats.mTrackletsFound += event.getTracklets().size();
+    mTFStats.mDigitsFound += event.getDigits().size();
+    mTFStats.mTimeTakenForTracklets += event.getTrackletTime();
+    mTFStats.mTimeTakenForDigits += event.getDigitTime();
+    mTFStats.mTimeTaken += event.getTotalTime();
+    if (event.getIsCalibTrigger()) {
+      ++mTFStats.mNTriggersCalib;
+    }
   }
 }