Adds event rates CI and example test result for beam sample

.github/workflows/EventRates.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
     branches: [develop]
 
+  push:
+    branches: [hwallace/feature/event_rate_bot]
+
 permissions:
   contents: read
   packages: write
@@ -15,11 +18,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: alma9
-            file: doc/MaCh3DockerFiles/Alma9/Dockerfile
-            tag: alma9test
+          - name: Event Rate Validation
+            test_exe: /tests/TestEventRates
+            test_config: tests/test_configs/EventRates_Beam_TEST.yaml
+
+
 
-    name: Build CI ${{ matrix.os }}
+    name: Run Test ${{matrix.name}}
 
     steps:
       # Set up CVMFS
@@ -36,26 +41,27 @@ jobs:
       - name: start docker container with bind mount cvmfs
         run: |
           docker build . \
-          --file ${{ matrix.file }} \
-          --tag ghcr.io/dune/mach3:${{ matrix.tag }} \
+          --file doc/MaCh3DockerFiles/Alma9/Dockerfile \
+          --tag ghcr.io/dune/mach3:alma9test \
           --build-arg MACH3_DUNE_VERSION=${{ github.ref_name }} \
-          --build-arg MACH3_DUNE_BUILD_ARGS="-DCUDAProb3Linear_ENABLED=ON -DUSE_CPU=ON"
+          --build-arg MACH3_DUNE_BUILD_ARGS="-DCUDAProb3Linear_ENABLED=ON -DMACH3_BUILD_TESTS=ON"
       
 
-
       # Run event rates
       - name: Run Event Rates
         uses: addnab/docker-run-action@v3
         with:
-          image: ghcr.io/dune/mach3:${{ matrix.tag }}
+          image: ghcr.io/dune/mach3:alma9test
           shell: bash
           options: -v /cvmfs:/cvmfs:shared
           run: |          
             ln -sf \
-            /cvmfs/dune.osgstorage.org/pnfs/fnal.gov/usr/dune/persistent/stash/MaCh3/inputs/TDR/v1/* \
-            inputs
+            /cvmfs/dune.osgstorage.org/pnfs/fnal.gov/usr/dune/persistent/stash/MaCh3/inputs/TDR/v2/DUNE_2021_FD_splines inputs/DUNE_spline_files
+            
+            ln -sf \
+            /cvmfs/dune.osgstorage.org/pnfs/fnal.gov/usr/dune/persistent/stash/MaCh3/inputs/TDR/v2/DUNE_2023_FD_CAFs inputs/DUNE_CAF_files
 
             source ./bin/setup.MaCh3.sh
             source ./bin/setup.NuOscillator.sh
             source ./bin/setup.MaCh3DUNE.sh
-            ./src/EventRates configs/EventRates_Beam.yaml
+            .${{matrix.test_exe}} ${{matrix.test_config}}

CMakeLists.txt

@@ -243,6 +243,13 @@ add_subdirectory(splines)
 add_subdirectory(samplePDFDUNE)
 add_subdirectory(src)
 
+
+# Do we want to build our integration tests?
+if(DEFINED MACH3_BUILD_TESTS AND MACH3_BUILD_TESTS)
+  add_subdirectory(tests)
+endif()
+
+
 add_library(MaCh3DUNEAll INTERFACE)
 set_target_properties(MaCh3DUNEAll PROPERTIES EXPORT_NAME All)
 target_link_libraries(MaCh3DUNEAll INTERFACE SamplePDFDUNE splinesDUNE MaCh3DUNECompilerOptions)

tests/CMakeLists.txt

@@ -0,0 +1,49 @@
+add_custom_target(MaCh3DUNETests)
+
+# Wet up utils lib
+set(HEADERS
+    Comparison.h
+)
+
+add_library(
+    MaCh3DuneTestUtils SHARED
+    Comparison.cpp
+)
+
+set_target_properties(MaCh3DuneTestUtils PROPERTIES
+    PUBLIC_HEADER "${HEADERS}"
+    EXPORT_NAME MaCh3DuneTestUtils
+)
+
+target_link_libraries(MaCh3DuneTestUtils PUBLIC MaCh3::All MaCh3DUNECompilerOptions)
+
+target_include_directories(MaCh3DuneTestUtils PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/../>
+  $<INSTALL_INTERFACE:include>)
+
+  install(TARGETS MaCh3DuneTestUtils
+  EXPORT mach3dune-targets
+  LIBRARY DESTINATION lib/
+  PUBLIC_HEADER DESTINATION include/MaCh3DuneTestUtils)
+
+add_library(MACH3DUNE::MaCh3DuneTestUtils ALIAS MaCh3DuneTestUtils)
+
+
+# Now we set up test apps
+foreach(app
+    TestEventRates
+)
+
+    add_executable(${app} ${app}.cpp)
+    target_link_libraries(${app} MaCh3DUNE::All MaCh3DuneTestUtils Minuit)
+    add_dependencies(MaCh3DUNEApps ${app})
+    install(TARGETS ${app} DESTINATION test)
+
+endforeach(app)
+
+
+# Now copy remaining stuff
+install(DIRECTORY test_configs DESTINATION test)
+install(DIRECTORY test_results DESTINATION test)
+
+

tests/Comparison.cpp

@@ -0,0 +1,50 @@
+// MaCh3 spline includes
+#include "tests/Comparison.h"
+
+bool CompareTwoFiles(const std::string& File1, const std::string& File2) {
+  std::ifstream file1(File1);
+  std::ifstream file2(File2);
+
+  // Check if both files opened successfully
+  if (!file1.is_open()) {
+    MACH3LOG_ERROR("Error opening file:  {}", File1);
+    throw MaCh3Exception(__FILE__ , __LINE__ );
+  }
+  if (!file2.is_open()) {
+    MACH3LOG_ERROR("Error opening file:  {}", File2);
+    throw MaCh3Exception(__FILE__ , __LINE__ );
+  }
+
+  std::string line1, line2;
+  bool sameFiles = true;
+  int lineNumber = 1;
+
+  // Read and compare line by line
+  while (std::getline(file1, line1) && std::getline(file2, line2)) {
+    if (line1 != line2) {
+      MACH3LOG_WARN("Difference found on line {}:", lineNumber);
+      MACH3LOG_WARN("File1: {}", line1);
+      MACH3LOG_WARN("File2: {}", line2);
+      sameFiles = false;
+    }
+    ++lineNumber;
+  }
+
+  // Check if one file has extra lines
+  while (std::getline(file1, line1)) {
+    MACH3LOG_WARN("Extra line in {} on line {}: {}", File1, lineNumber, line1);
+    sameFiles = false;
+    ++lineNumber;
+  }
+  while (std::getline(file2, line2)) {
+    MACH3LOG_WARN("Extra line in {} on line {}: {}", File2, lineNumber, line2);
+    sameFiles = false;
+    ++lineNumber;
+  }
+
+  // Close files
+  file1.close();
+  file2.close();
+
+  return sameFiles;
+}

tests/Comparison.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "manager/manager.h"
+
+bool CompareTwoFiles(const std::string& File1, const std::string& File2);

tests/TestEventRates.cpp

@@ -0,0 +1,111 @@
+#include <iostream>
+#include <chrono>
+#include <iomanip>
+#include <vector>
+#include <string>
+
+#include <TH1D.h>
+#include <THStack.h>
+#include <TStyle.h>
+#include <TCanvas.h>
+#include <TRint.h>
+#include <TLegend.h>
+#include <TColor.h>
+#include <TMath.h>
+
+#include "samplePDFDUNE/MaCh3DUNEFactory.h"
+#include "samplePDFDUNE/StructsDUNE.h"
+#include "tests/Comparison.h"
+
+
+int main(int argc, char * argv[]) {
+  MaCh3Utils::MaCh3Usage(argc, argv);
+
+  auto fitMan = std::unique_ptr<manager>(new manager(argv[1]));
+
+  bool skip_checks = false;
+
+  if (CheckNodeExists(fitMan->raw(), "General", "Tests", "SkipChecks") ){
+    skip_checks = fitMan->raw()["General"]["Tests"]["SkipChecks"].as<bool>();
+  }
+
+  //###############################################################################################################################
+  //Create samplePDFFD objects
+
+  covarianceXsec* xsec = nullptr;
+  covarianceOsc* osc = nullptr;
+
+  std::vector<samplePDFFDBase*> DUNEPdfs;
+  MakeMaCh3DuneInstance(fitMan.get(), DUNEPdfs, xsec, osc);
+
+
+  //###############################################################################################################################
+  //Perform reweight and print total integral
+  //###############################################################################################################################
+  //Make oscillation channel breakdown
+  // Initialise output file
+  std::ofstream outFile("TestNewSampleOut.txt");
+
+
+  for(auto Sample : DUNEPdfs) {
+    int nOscChannels = Sample->getNMCSamples();
+    for (int iOscChan=0;iOscChan<nOscChannels;iOscChan++) {
+      std::vector< std::vector<double> > SelectionVec;
+
+      std::vector<double> SelecChannel(3);
+      SelecChannel[0] = Sample->ReturnKinematicParameterFromString("OscChannel");
+      SelecChannel[1] = iOscChan;
+      SelecChannel[2] = iOscChan+1;
+      SelectionVec.push_back(SelecChannel);
+
+      TH1* Hist = Sample->get1DVarHist("TrueNeutrinoEnergy",SelectionVec);
+      outFile<<Sample->GetName()<<" "<<Sample->getFlavourName(iOscChan)<<" "<<Hist->Integral()<<"\n";
+    }
+
+    TH1* Hist = Sample->get1DVarHist("TrueNeutrinoEnergy");
+    outFile<<Sample->GetName()<<" "<<Hist->Integral()<<"\n";
+  }
+
+  //###############################################################################################################################
+  //Make interaction channel breakdown
+
+  for(auto Sample : DUNEPdfs) {
+    int nModeChannels = kMaCh3_nModes;
+    for (int iModeChan=0;iModeChan<nModeChannels;iModeChan++) {
+      std::vector< std::vector<double> > SelectionVec;
+
+      std::vector<double> SelecChannel(3);
+      SelecChannel[0] = Sample->ReturnKinematicParameterFromString("Mode");
+      SelecChannel[1] = iModeChan;
+      SelecChannel[2] = iModeChan+1;
+      SelectionVec.push_back(SelecChannel);
+
+      TH1* Hist = Sample->get1DVarHist("TrueNeutrinoEnergy",SelectionVec);
+      outFile<<Sample->GetName()<<" "<<MaCh3mode_ToDUNEString((MaCh3_Mode)iModeChan)<<" "<<Hist->Integral()<<"\n";
+    }
+
+    TH1* Hist = Sample->get1DVarHist("TrueNeutrinoEnergy");
+    outFile<<Sample->GetName()<<" "<<Hist->Integral()<<"\n";
+  }
+
+  // Do you want to gener
+  if(skip_checks){
+    return 0;
+  }
+
+  std::string results_file = fitMan->raw()["General"]["Tests"]["TestResultsFile"].as<std::string>();
+
+
+  // Okay now we've written we need to compare 
+  bool TheSame = CompareTwoFiles(results_file, "TestNewSampleOut.txt");
+
+  // Are we right?
+  if(!TheSame) {
+    MACH3LOG_CRITICAL("Different event rates detected");
+    throw MaCh3Exception(__FILE__ , __LINE__ );
+  } else {
+    MACH3LOG_INFO("Event rates match");
+  }
+
+  return 0;
+}

tests/test_configs/EventRates_Beam_TEST.yaml

@@ -0,0 +1,32 @@
+General:
+  OutputFile: "DuneEventRates.root"
+  DUNESamples: ["configs/Samples/SamplePDFDune_FHC_numuselec.yaml"] #, "configs/Samples/SamplePDFDune_RHC_nueselec.yaml"]	
+  #Nu-FIT
+  #OscillationParameters: [0.310, 0.582, 0.224, 7.39E-5, 2.5254E-3, -2.498]
+  # T2K-like best-fit
+  OscillationParameters: [0.307, 0.528, 0.0218, 7.53e-5, 2.509e-3, -1.601, 1284.9, 2.848]
+  OscillatorConfigName: "configs/OscillatorObj.yaml"
+  Systematics:
+    XsecCovFile: ["configs/CovObjs/xsec_covariance_DUNE_systs_2022a_FD_v3_xsec.yaml"]
+    XsecCovName: "xsec_cov"
+    XsecStepScale: 0.1
+    XsecAtGen: false 
+    OscCovFile: ["configs/CovObjs/OscCov_PDG2021_v2.yaml"]
+    OscCovName: "osc_cov"
+
+  Tests:
+    SkipChecks: True
+    TestResultsFile: tests/test_results/EventRatesV2TDR.txt
+
+  Fitter:
+    FitTestLikelihood: false
+  MCMC:
+    NSteps: 2000
+    AutoSave: 10000
+  Output:
+    FileName: "TestEventRates.root"
+    OUTPUTNAME: "TestLLH.root"
+  ProcessMCMC: No
+  Seed: 0
+  Debug: No
+

tests/test_results/EventRatesV2TDR.txt

@@ -0,0 +1,41 @@
+FHC_numu FHC_numu_x_numu -9.97428e+06
+FHC_numu FHC_nue_x_nue 0
+FHC_numu FHC_numubar_x_numubar -1.64516e+07
+FHC_numu FHC_nuebar_x_nuebar -1.43885e+07
+FHC_numu FHC_numu_x_nue -276245
+FHC_numu FHC_numubar_x_nuebar 0
+FHC_numu FHC_nue_x_numu -168.017
+FHC_numu FHC_nuebar_x_numubar 0
+FHC_numu FHC_numu_x_nutau 0
+FHC_numu FHC_nue_x_nutau -2.75556e+06
+FHC_numu FHC_numubar_x_nutaubar 0
+FHC_numu FHC_nuebar_x_nutaubar 0
+FHC_numu -4.38459e+07
+FHC_numu ccqe -9.97428e+06
+FHC_numu unknown 0
+FHC_numu ccdis -1.64516e+07
+FHC_numu ccres -1.43885e+07
+FHC_numu cccoh -276245
+FHC_numu ccdiff 0
+FHC_numu ccnueel -168.017
+FHC_numu ccIMD 0
+FHC_numu ccamnugamma 0
+FHC_numu ccmec -2.75556e+06
+FHC_numu cccohel 0
+FHC_numu ccibd 0
+FHC_numu ccglasres 0
+FHC_numu ccimdannihilation 0
+FHC_numu ncqe 6.94706
+FHC_numu ncdis 341.075
+FHC_numu ncres 106.298
+FHC_numu nccoh 0.502197
+FHC_numu ncdiff 0
+FHC_numu ncnueel 0
+FHC_numu ncIMD 0
+FHC_numu ncamnugamma 0
+FHC_numu ncmec 0
+FHC_numu nccohel 0
+FHC_numu ncibd 0
+FHC_numu ncglasres 0
+FHC_numu ncimdannihilation 0
+FHC_numu -4.38459e+07