MesyDaqParsingSupport.h

Go to the documentation of this file.

/* 
 * mvme2root by Sean Finch. Modified from 
 * mvme-listfile-dumper - format mvme listfile content and print it to stdout
 *
 * Copyright (C) 2017  Florian Lüke <f.lueke@mesytec.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 *
 */
#include <cerrno>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <map>
 
#include "MesyDAQEvent.h"
typedef uint8_t  u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
 
typedef int8_t  s8;
typedef int16_t s16;
typedef int32_t s32;
typedef int64_t s64;
 
using std::cout;
using std::cerr;
using std::endl;
using std::string;
 
#include "TFile.h"
#include "TROOT.h"
#include "TTree.h"
/*  ===== VERSION 0 =====
 *
 *  ------- Section (Event) Header ----------
 *  33222222222211111111110000000000
 *  10987654321098765432109876543210
 * +--------------------------------+
 * |ttt         eeeessssssssssssssss|
 * +--------------------------------+
 *
 * t =  3 bit section type
 * e =  4 bit event type (== event number/index) for event sections
 * s = 16 bit size in units of 32 bit words (fillwords added to data if needed) -> 256k section max size
 *
 * Section size is the number of following 32 bit words not including the header word itself.
* Sections with SectionType_Event contain subevents with the following header:
 
 *  ------- Subevent (Module) Header --------
 *  33222222222211111111110000000000
 *  10987654321098765432109876543210
 * +--------------------------------+
 * |              mmmmmm  ssssssssss|
 * +--------------------------------+
 *
 * m =  6 bit module type (VMEModuleType enum from globals.h)
 * s = 10 bit size in units of 32 bit words
 *
 * The last word of each event section is the EndMarker (globals.h)
 *
*/
struct listfile_v0
{
  static const int Version = 0;
  static const int FirstSectionOffset = 0;
  static const int SectionMaxWords  = 0xffff;
  static const int SectionMaxSize   = SectionMaxWords * sizeof(u32);
  static const int SectionTypeMask  = 0xe0000000; // 3 bit section type
  static const int SectionTypeShift = 29;
  static const int SectionSizeMask  = 0xffff;    // 16 bit section size in 32 bit words
  static const int SectionSizeShift = 0;
  static const int EventTypeMask  = 0xf0000;   // 4 bit event type
  static const int EventTypeShift = 16;
  // Subevent containing module data
  static const int ModuleTypeMask  = 0x3f000; // 6 bit module type
  static const int ModuleTypeShift = 12;
  //added by jb 7 20 2019
  static const int ModuleIDMask     = 0x00ff0000;
  static const int ModuleIDShift    = 16;
  static const int SubEventMaxWords  = 0x3ff;
  static const int SubEventMaxSize   = SubEventMaxWords * sizeof(u32);
  static const int SubEventSizeMask  = 0x3ff; // 10 bit subevent size in 32 bit words
  static const int SubEventSizeShift = 0;
};
 
/*  ===== VERSION 1 =====
 *
 * Differences to version 0:
 * - Starts with the FourCC "MVME" followed by a 32 bit word containing the
 *   listfile version number.
 * - Larger section and subevent sizes: 16 -> 20 bits for sections and 10 -> 20
 *   bits for subevents.
 * - Module type is now 8 bit instead of 6.
 *
 *  ------- Section (Event) Header ----------
 *  33222222222211111111110000000000
 *  10987654321098765432109876543210
 * +--------------------------------+
 * |ttteeee     ssssssssssssssssssss|
 * +--------------------------------+
 *
 * t =  3 bit section type
 * e =  4 bit event type (== event number/index) for event sections
 * s = 20 bit size in units of 32 bit words (fillwords added to data if needed) -> 256k section max size
 *
 * Section size is the number of following 32 bit words not including the header word itself.
 
 * Sections with SectionType_Event contain subevents with the following header:
 
 *  ------- Subevent (Module) Header --------
 *  33222222222211111111110000000000
 *  10987654321098765432109876543210
 * +--------------------------------+
 * |mmmmmmmm    ssssssssssssssssssss|
 * +--------------------------------+
 *
 * m =  8 bit module type (VMEModuleType enum from globals.h)
 * s = 10 bit size in units of 32 bit words
 *
 * The last word of each event section is the EndMarker (globals.h)
 *
*/
struct listfile_v1
{
  static const int Version = 1;
  static const int FirstSectionOffset = 8;
  static const int SectionMaxWords  = 0xfffff;
  static const int SectionMaxSize   = SectionMaxWords * sizeof(u32);
  static const int SectionTypeMask  = 0xe0000000; // 3 bit section type
  static const int SectionTypeShift = 29;
  static const int SectionSizeMask  = 0x000fffff; // 20 bit section size in 32 bit words
  static const int SectionSizeShift = 0;
  static const int EventTypeMask    = 0x1e000000; // 4 bit event type
  static const int EventTypeShift   = 25;
  // Subevent containing module data
  static const int ModuleTypeMask  = 0xff000000;  // 8 bit module type
  static const int ModuleTypeShift = 24;
    //added by jb 7 20 2019
  static const int ModuleIDMask     = 0x00ff0000;
  static const int ModuleIDShift    = 16;
  static const int SubEventMaxWords  = 0xfffff;
  static const int SubEventMaxSize   = SubEventMaxWords * sizeof(u32);
  static const int SubEventSizeMask  = 0x000fffff; // 20 bit subevent size in 32 bit words
  static const int SubEventSizeShift = 0;
};
 
namespace listfile
{
  enum SectionType
  {
      /* The config section contains the mvmecfg as a json string padded with
       * spaces to the next 32 bit boundary. If the config data size exceeds
       * the maximum section size multiple config sections will be written at
       * the start of the file. */
      SectionType_Config      = 0,
      /* Readout data generated by one VME Event. Contains Subevent Headers
       * to split into VME Module data. */
      SectionType_Event       = 1,
      /* Last section written to a listfile before closing the file. Used for
       * verification purposes. */
      SectionType_End         = 2,
      /* Marker section written once at the start of a run and then once per
       * elapsed second. */
      SectionType_Timetick    = 3,
      /* Max section type possible. */
      SectionType_Max         = 7
  };
 
  enum VMEModuleType
  {
      Invalid         = 0,
      MADC32          = 1,
      MQDC32          = 2,
      MTDC32          = 3,
      MDPP16_SCP      = 4,
      MDPP32          = 5,
      MDI2            = 6,
      MDPP16_RCP      = 7,
      MDPP16_QDC      = 8,
      VMMR            = 9,
      MesytecCounter = 16,
      VHS4030p = 21,
  };
 
  static const std::map<VMEModuleType, const char *> VMEModuleTypeNames =
  {
      { VMEModuleType::MADC32,            "MADC-32" },
      { VMEModuleType::MQDC32,            "MQDC-32" },
      { VMEModuleType::MTDC32,            "MTDC-32" },
      { VMEModuleType::MDPP16_SCP,        "MDPP-16_SCP" },
      { VMEModuleType::MDPP32,            "MDPP-32" },
      { VMEModuleType::MDI2,              "MDI-2" },
      { VMEModuleType::MDPP16_RCP,        "MDPP-16_RCP" },
      { VMEModuleType::MDPP16_QDC,        "MDPP-16_QDC" },
      { VMEModuleType::VMMR,              "VMMR" },
      { VMEModuleType::VHS4030p,          "iseg VHS4030p" },
      { VMEModuleType::MesytecCounter,    "Mesytec Counter" },
  };
 
  const char *get_vme_module_name(VMEModuleType moduleType)
  {
      auto it = VMEModuleTypeNames.find(moduleType);
      if (it != VMEModuleTypeNames.end())
      {
          return it->second;
      }
      return "unknown";
  }
 
} // end namespace listfile
 
inline int bitExtractor(int word, int numbits, int position){
    return (((1 << numbits) - 1) & (word >> position)); 
}
 
template<typename LF>
void process_listfile(std::ifstream &infile,
                      TString       filename, 
                      bool          optverbose,
                      bool          writeZeros,
                      int           a_numSection = -1,
                      int           a_numStartEvent  =-1
                      )
{
  int maxEvents = a_numSection;
  using namespace listfile;
  bool continueReading = true;
  bool SCPon = 0;
  bool QDCon = 0;
  int counter = 0;
  int pu = 0;
  int ov = 0;
  int chn = 0;
  int data = 0;
  int time = 0;
  int extended = 0;
  int sig = 0;
  TString rootfilename = filename;
  rootfilename.ReplaceAll("mvmelst","root");
  rootfilename.ReplaceAll("listfiles","data_root"); cout << "Root file name: " << rootfilename << endl;
  TFile *rootfile = new TFile(rootfilename, "RECREATE");
  TTree* dataTree = new TTree("daqEvents","daqEvents",99,rootfile);
  MesyDAQEvent* daqEv = new MesyDAQEvent;
  dataTree->Branch("events",&daqEv); 
  int sectionCounter = 0;
  int eventCounter = 0;
  int eventNumber = 0;
  int timeTickCounter = 0;
  while (continueReading)
  {
    u32 sectionHeader;
    infile.read((char *)&sectionHeader, sizeof(u32));
    u32 sectionType   = (sectionHeader & LF::SectionTypeMask) >> LF::SectionTypeShift;
    u32 sectionSize   = (sectionHeader & LF::SectionSizeMask) >> LF::SectionSizeShift;
    sectionCounter++;
    switch (sectionType)
    {
      case SectionType_Config:
      {
        if (optverbose)
            cout << "Config section of size " << sectionSize << endl;
        infile.seekg(sectionSize * sizeof(u32), std::ifstream::cur);
      } 
      break;
      case SectionType_Event:
      {
        if(eventNumber < a_numStartEvent)
        {
          infile.seekg(sectionSize * sizeof(u32), std::ifstream::cur);
          eventNumber++;
          break;
        }
        eventNumber++;
        //only count read events
        eventCounter++;
        daqEv->m_modules.clear();
        if (optverbose)
        {
          cout << "Event " << counter << endl;
        }
        else
        {
          if (counter%10000==0)
          {
              cout << '\r' << "Processing event " << counter;
          }
        }
        // rootdata_SCP.initEvent();
        // rootdata_QDC.initEvent();
        pu = 0;
        ov = 0;
        time = 0;
        extended = 0;
        sig = 0;
        u32 eventType = (sectionHeader & LF::EventTypeMask) >> LF::EventTypeShift;
        if (optverbose)
        {
            printf("Event section: eventHeader=0x%08x, eventType=%d, eventSize=%u\n",
                   sectionHeader, eventType, sectionSize);
        }
        u32 wordsLeft = sectionSize;
        int hadEvent = 0;
        while (wordsLeft > 1)
        {
          u32 subEventHeader;
          infile.read((char *)&subEventHeader, sizeof(u32));
          --wordsLeft;
          if (subEventHeader ==0x87654321)
          {
            if (optverbose)
            {
                cout << "\tFill" << endl;
            }
            continue;
          }
          u32 moduleType = (subEventHeader & LF::ModuleTypeMask) >> LF::ModuleTypeShift;
          u32 subEventSize = (subEventHeader & LF::SubEventSizeMask) >> LF::SubEventSizeShift;
          int adcWords = 0; 
          int tdcWords = 0;
          if (optverbose)
          {
              printf("  subEventHeader=0x%08x, moduleType=%u (%s), subEventSize=%u\n ",
                     subEventHeader, moduleType, get_vme_module_name((VMEModuleType)moduleType),
                     subEventSize);
          }
          if ((moduleType==4)||(moduleType==7)||(moduleType==8))
          {
            //create an event class vector
            map < int , vector<MDPP16Event> > boardEvents;
            //storage for charge values
            map< int , vector<MDPP16ChargeData> > chargeData;
            map < int , vector<MDPP16ChargeData> >  peakChargeData;
            //storage for times 
            map < int, vector<uint16_t> >  timeData; 
            map < int, vector<uint16_t> >  trigTime;           
            int moduleID = -1;
            uint16_t extendTimeStamp = -1;
            //keeps track of what type the last data word was
            for (u32 i=0; i<subEventSize; ++i)
            {
              //get an event container
 
              u32 subEventData = 0x0;
              infile.read((char *)&subEventData, sizeof(u32));
              wordsLeft--;
              if (optverbose)
              printf("    %2u = 0x%08x\n", i, subEventData);
              if (subEventData == 0x00000000||subEventData ==0x87654321)
              {
                if (optverbose)
                {
                    cout << "\tFill" << endl;
                }
                continue;
              }
              SCPon = 1;
              sig = 0;
              sig = bitExtractor(subEventData, 4, 28);
              if (sig==4)
              { //header
                  moduleID = (subEventData & LF::ModuleIDMask) >> LF::ModuleIDShift;
                  if (optverbose)
                  {
                    cout << "\tHeader" << endl;
                    
                  }
              }
              else if (sig==1)
              {//data
                //MDPP16 with SCP or RCP firmware
                int chan = bitExtractor(subEventData, 6, 16);
                uint16_t data = bitExtractor(subEventData, 16, 0);
                // rootdata_SCP.setADC(chn, data);
                if (chan<16)
                {   
                    MDPP16ChargeData dat;
                    dat.m_charge = data;
                    dat.m_pileUp = bitExtractor(subEventData, 1, 23);
                    dat.m_overFlow = bitExtractor(subEventData, 1, 22);
                    if (writeZeros) chargeData[chan].push_back(dat);
                    else if (data!=0) chargeData[chan].push_back(dat);
                    else timeData[chan].pop_back();
                }
                else if(chan >= 16 && chan < 32)
                {
                  timeData[chan%16].push_back(data);
                }
                else if (chan==32||chan==33)
                {
                  trigTime[chan%16].push_back(data);
                }
                else if (chan>47)
                {
                    MDPP16ChargeData dat;
                    dat.m_charge = data;
                    dat.m_pileUp = bitExtractor(subEventData, 1, 23);
                    dat.m_overFlow = bitExtractor(subEventData, 1, 22);
                    // rootdata_SCP.setPileup(chn, pu);
                    // rootdata_SCP.setOverflow(chn, ov);
                    if (writeZeros) peakChargeData[chan%16].push_back(dat);
                    else if (chargeData[chan%16].size()==peakChargeData[chan%16].size()+1) peakChargeData[chan%16].push_back(dat);
                }
                if (optverbose)
                {
                    cout << "\tData" << endl;
                    cout << "\t" << ov << "\t" 
                    << chan << "\t" << data << endl;
                }
              }
              else if(sig==2)
              {//extended time stamp
                extendTimeStamp = bitExtractor(subEventData, 16, 0);
                // rootdata_SCP.setExtendedTime(extended);
                if (optverbose)
                {
                  cout << "\tExtended time stamp:\t" << extended << endl;
                }
              }
              else if((sig&0xc)==0xc)
              {//end of event
                //build an event and associate it with a module when the event is closed
                MDPP16Event ev;
                for (auto& charges : chargeData )
                {
                  if (writeZeros)
                  {
                    ev.addCharges(charges.first, charges.second);
                    ev.addTimes(charges.first, timeData[charges.first]);
                  }
                  else
                  {
                    int numGoodHits = 0;
                    for (int hit = 0; hit < charges.second.size(); hit++)
                    {
                      if (charges.second[hit].m_charge!=0 && timeData[charges.first][hit]!=0) numGoodHits++;
                    }
                    if (numGoodHits == charges.second.size())
                    {
                      ev.addCharges(charges.first, charges.second);
                      ev.addTimes(charges.first, timeData[charges.first]);
                    }
                  }
                  
                }
                // for(auto& charges : chargeData )
                // {
                //   ev.addCharges(charges.first, charges.second);
                // }
                // for(auto& times : timeData )
                // {
                //   ev.addTimes(times.first, times.second);
                // }
                for(auto& charges : peakChargeData )
                {
                  ev.addPeakCharges(charges.first, charges.second);
                }
                for(auto& trigChannel : trigTime )
                {
                  for(auto& time : trigChannel.second)
                  {
                  ev.addTrigTimeStamp(trigChannel.first, time);
                  }
                }                
                ev.setExtTimeStamp(extendTimeStamp);
                ev.setEndOfEventWord(bitExtractor(subEventData, 30, 0));
                if (optverbose)
                {
                    cout << "\tEnd of event" << endl;
                    cout << "\tTime:\t" << bitExtractor(subEventData, 30, 0) << endl;
                }
                if (writeZeros) 
                {
                  daqEv->m_modules[moduleID].push_back(ev);
                  hadEvent++;
                }
                else if (ev.isGoodEvent()) 
                {
                  daqEv->m_modules[moduleID].push_back(ev);
                  if (optverbose) cout << "good event!" <<endl;
                  hadEvent++;
                }
                extendTimeStamp = -1;
                moduleID = -1;
                chargeData.clear();
                peakChargeData.clear();
                timeData.clear();
                trigTime.clear();
 
               
                // break;
              }
 
            }//end of sub event loop
          }//end of module 4 or 7 or 8
        }//end of while over event section
        //read the last word
        u32 eventEndMarker;
        infile.read((char *)&eventEndMarker, sizeof(u32));
        if (optverbose)
        {
            printf("   eventEndMarker=0x%08x\n", eventEndMarker);
        }
        // rootdata_QDC.writeEvent();
        // rootdata_SCP.writeEvent();
        counter++;
        //fill the tree
        if (hadEvent>0) dataTree->Fill();
        if(maxEvents >0 && counter >= maxEvents)
        {
          continueReading = false;
        }
      }
      break;
      case SectionType_Timetick:
          {
              if (optverbose)
                  printf("Timetick\n");
          } break;
      case SectionType_End:
          {
              printf("\nFound Listfile End section\n");
              continueReading = false;
              auto currentFilePos = infile.tellg();
              infile.seekg(0, std::ifstream::end);
              auto endFilePos = infile.tellg();
              if (currentFilePos != endFilePos)
              {
                  cout << "Warning: " << (endFilePos - currentFilePos)
                      << " bytes left after Listfile End Section" << endl;
              }
              break;
          }
      default:
          {
              printf("Warning: Unknown section type %u of size %u, skipping...\n",
                     sectionType, sectionSize);
              infile.seekg(sectionSize * sizeof(u32), std::ifstream::cur);
          } break;
    }
  }
  cout << counter << " events total" << endl;
  // rootfile->cd();
  // if(SCPon){
  //     // rootdata_SCP.writeTree();
  //     rootfile->mkdir("histos_SCP");
  //     rootfile->cd("histos_SCP");
  //     // rootdata_SCP.writeHistos();
  // }
  // if(QDCon){
  //     // rootdata_QDC.writeTree();
  //     rootfile->mkdir("histos_QDC");
  //     rootfile->cd("histos_QDC");
  //     // rootdata_QDC.writeHistos();
  // }
  rootfile->Write();
  rootfile->Close();
}
 
void process_listfile(std::ifstream &infile,
                      TString       filename, 
                      bool          optverbose,
                      bool          writeZeros,
                      int           a_numSection = -1,
                      int           a_numStartEvent  =-1
                      )
{
    u32 fileVersion = 0;
 
    // Read the fourCC that's at the start of listfiles from version 1 and up.
    const size_t bytesToRead = 4;
    char fourCC[bytesToRead] = {};
 
    infile.read(fourCC, bytesToRead);
    static const char * const FourCC = "MVME";
 
    if (std::strncmp(fourCC, FourCC, bytesToRead) == 0)
    {
        infile.read(reinterpret_cast<char *>(&fileVersion), sizeof(fileVersion));
    }
 
    // Move to the start of the first section
    auto firstSectionOffset = ((fileVersion == 0)
                               ? listfile_v0::FirstSectionOffset
                               : listfile_v1::FirstSectionOffset);
 
    infile.seekg(firstSectionOffset, std::ifstream::beg);
 
    cout << "Detected listfile version " << fileVersion << endl;
 
    // if (fileVersion == 0)
    // {
    //     process_listfile<listfile_v0>(infile, filename, optverbose);
    // }
    // else
    {
        process_listfile<listfile_v1>(infile, filename, optverbose,writeZeros,a_numSection, a_numStartEvent);
    }
}