NanoAOD prototype [RFC]

Configuration/Applications/python/ConfigBuilder.py

@@ -626,6 +626,7 @@ def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+th
 		if self._options.timeoutOutput:
 			CppType='TimeoutPoolOutputModule'
 		if streamType=='DQM' and tier=='DQMIO': CppType='DQMRootOutputModule'
+		if "NANOAOD" in streamType and tier=='NANO': CppType='NanoAODOutputModule'
                 output = cms.OutputModule(CppType,
                                           theEventContent,
                                           fileName = cms.untracked.string(theFileName),
@@ -921,6 +922,7 @@ def define_Configs(self):
         self.L1TrackTriggerDefaultCFF="Configuration/StandardSequences/L1TrackTrigger_cff"
         self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff"
         self.PATDefaultCFF="Configuration/StandardSequences/PAT_cff"
+        self.NANODefaultCFF="PhysicsTools/NanoAOD/nano_cff"
 	self.EIDefaultCFF=None
         self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff"
         self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff"
@@ -980,6 +982,7 @@ def define_Configs(self):
         self.REPACKDefaultSeq='DigiToRawRepack'
 	self.PATDefaultSeq='miniAOD'
 	self.PATGENDefaultSeq='miniGEN'
+	self.NANODefaultSeq='nanoSequence'
 
         self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff"
 
@@ -994,6 +997,7 @@ def define_Configs(self):
 		self.PATGENDefaultCFF="Configuration/StandardSequences/PATGEN_cff"
                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff"
                 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff"
+	        self.NANODefaultSeq='nanoSequenceMC'
 	else:
 		self._options.beamspot = None
 
@@ -1687,6 +1691,16 @@ def prepare_PATGEN(self, sequence = "miniGEN"):
             raise Exception("PATGEN step can only run on MC")
         return
 
+    def prepare_NANO(self, sequence = "nanoAOD"):
+        ''' Enrich the schedule with NANO '''
+        self.loadDefaultOrSpecifiedCFF(sequence,self.NANODefaultCFF)
+	self.scheduleSequence(sequence.split('.')[-1],'nanoAOD_step')
+        custom = "nanoAOD_customizeData" if self._options.isData else "nanoAOD_customizeMC"
+        if self._options.runUnscheduled:
+            self._options.customisation_file_unsch.insert(0,"PhysicsTools/NanoAOD/nano_cff."+custom)
+        else:
+            self._options.customisation_file.insert(0,"PhysicsTools/NanoAOD/nano_cff."+custom)
+
     def prepare_EI(self, sequence = None):
         ''' Enrich the schedule with event interpretation '''
 	from Configuration.StandardSequences.EventInterpretation import EventInterpretation

Configuration/Applications/python/cmsDriverOptions.py

@@ -101,6 +101,7 @@ def OptionsFromItems(items):
                  "HARVESTING":"RECO",
                  "ALCAHARVEST":"RECO",
                  "PAT":"RECO",
+                 "NANO":"PAT",
                  "PATGEN":"GEN"}
 
     trimmedEvtType=options.evt_type.split('/')[-1]

Configuration/Eras/python/Modifier_run2_nanoAOD_92X_cff.py

@@ -0,0 +1,3 @@
+import FWCore.ParameterSet.Config as cms
+
+run2_nanoAOD_92X = cms.Modifier()

Configuration/StandardSequences/python/Eras.py

@@ -41,7 +41,7 @@ def __init__(self):
                            'phase2_hgcal', 'phase2_muon', 'phase2_timing',
                            'phase2_timing_layer','phase2_hcal',
                            'trackingLowPU', 'trackingPhase1', 'trackingPhase1QuadProp', 'ctpps_2016', 'trackingPhase2PU140',
-                           'tracker_apv_vfp30_2016', 'run2_miniAOD_80XLegacy',
+                           'tracker_apv_vfp30_2016', 'run2_miniAOD_80XLegacy', 'run2_nanoAOD_92X',
                            'hcalHardcodeConditions', 'hcalSkipPacker']
         internalUseModChains = ['run2_2017_noTrackingModifier']
 

DataFormats/NanoAOD/BuildFile.xml

@@ -0,0 +1,8 @@
+<use   name="FWCore/Utilities"/>
+<use   name="FWCore/Common"/>
+<use   name="DataFormats/Common"/>
+<use   name="DataFormats/StdDictionaries"/>
+<use   name="boost"/>
+<export>
+  <lib   name="1"/>
+</export>

DataFormats/NanoAOD/interface/FlatTable.h

@@ -0,0 +1,163 @@
+#ifndef DataFormats_NanoAOD_FlatTable_h
+#define DataFormats_NanoAOD_FlatTable_h
+
+#include <cstdint>
+#include <vector>
+#include <string>
+#include <boost/range/sub_range.hpp>
+#include "FWCore/Utilities/interface/Exception.h"
+#include "DataFormats/PatCandidates/interface/libminifloat.h"
+
+namespace nanoaod {
+
+namespace flatTableHelper {
+    template<typename T> struct MaybeMantissaReduce { 
+        MaybeMantissaReduce(int mantissaBits) {}
+        inline T one(const T &val) const  { return val; }
+        inline void bulk(boost::sub_range<std::vector<T>> data) const {  }
+    };
+    template<> struct MaybeMantissaReduce<float> {
+        int bits_; 
+        MaybeMantissaReduce(int mantissaBits) : bits_(mantissaBits) {}
+        inline float one(const float &val) const  { return (bits_ > 0 ? MiniFloatConverter::reduceMantissaToNbitsRounding(val, bits_) : val); }
+        inline void bulk(boost::sub_range<std::vector<float>> data) const { if (bits_ > 0) MiniFloatConverter::reduceMantissaToNbitsRounding(bits_, data.begin(), data.end(), data.begin()); }
+    };
+}
+
+class FlatTable {
+  public:
+    enum ColumnType { FloatColumn, IntColumn, UInt8Column, BoolColumn }; // We could have other Float types with reduced mantissa, and similar
+
+    FlatTable() : size_(0) {}
+    FlatTable(unsigned int size, const std::string & name, bool singleton, bool extension=false) : size_(size), name_(name), singleton_(singleton), extension_(extension)  {}
+    ~FlatTable() {}
+
+    unsigned int nColumns() const { return columns_.size(); };
+    unsigned int nRows() const { return size_; };
+    unsigned int size() const { return size_; }
+    bool singleton() const { return singleton_; }
+    bool extension() const { return extension_; }
+    const std::string & name() const { return name_; }
+
+    const std::string & columnName(unsigned int col) const { return columns_[col].name; }
+    int columnIndex(const std::string & name) const ; 
+
+    ColumnType columnType(unsigned int col) const { return columns_[col].type; }
+
+    void setDoc(const std::string & doc) { doc_ = doc; }
+    const std::string & doc() const { return doc_; }
+    const std::string & columnDoc(unsigned int col) const { return columns_[col].doc; }
+
+    /// get a column by index (const)
+    template<typename T>
+    boost::sub_range<const std::vector<T>> columnData(unsigned int column) const { 
+         auto begin = beginData<T>(column);
+         return boost::sub_range<const std::vector<T>>(begin, begin+size_);
+    }
+
+    /// get a column by index (non-const)
+    template<typename T>
+    boost::sub_range<std::vector<T>> columnData(unsigned int column) { 
+         auto begin = beginData<T>(column);
+         return boost::sub_range<std::vector<T>>(begin, begin+size_);
+    }
+
+    /// get a column value for singleton (const)
+    template<typename T>
+    const T & columValue(unsigned int column) const {
+         if (!singleton()) throw cms::Exception("LogicError", "columnValue works only for singleton tables");
+         return * beginData<T>(column);
+    }
+
+    template<typename T, typename C = std::vector<T>>
+    void addColumn(const std::string & name, const C & values, const std::string & docString, ColumnType type = defaultColumnType<T>(),int mantissaBits=-1) {
+        if (columnIndex(name) != -1) throw cms::Exception("LogicError", "Duplicated column: "+name); 
+        if (values.size() != size()) throw cms::Exception("LogicError", "Mismatched size for "+name); 
+        check_type<T>(type); // throws if type is wrong
+        auto & vec = bigVector<T>();
+        columns_.emplace_back(name,docString,type,vec.size());
+        vec.insert(vec.end(), values.begin(), values.end());
+        if (type == FloatColumn) {
+            flatTableHelper::MaybeMantissaReduce<T>(mantissaBits).bulk(columnData<T>(columns_.size()-1));
+        }
+    }
+    template<typename T, typename C>
+    void addColumnValue(const std::string & name, const C & value, const std::string & docString, ColumnType type = defaultColumnType<T>(),int mantissaBits=-1) {
+        if (!singleton()) throw cms::Exception("LogicError", "addColumnValue works only for singleton tables");
+        if (columnIndex(name) != -1) throw cms::Exception("LogicError", "Duplicated column: "+name);
+        check_type<T>(type); // throws if type is wrong
+        auto & vec = bigVector<T>();
+        columns_.emplace_back(name,docString,type,vec.size());
+        if (type == FloatColumn) {
+            vec.push_back( flatTableHelper::MaybeMantissaReduce<T>(mantissaBits).one(value) );
+        } else {
+            vec.push_back( value );
+        }
+    }
+
+    template<typename T> static ColumnType defaultColumnType() { throw cms::Exception("unsupported type"); }
+
+    // this below needs to be public for ROOT, but it is to be considered private otherwise
+    struct Column {
+        std::string name, doc;
+        ColumnType type;
+        unsigned int firstIndex;
+        Column() {} // for ROOT
+        Column(const std::string & aname, const std::string & docString, ColumnType atype, unsigned int anIndex) : name(aname), doc(docString), type(atype), firstIndex(anIndex) {}
+    };
+
+  private:
+
+     template<typename T>
+     typename std::vector<T>::const_iterator beginData(unsigned int column) const {
+         const Column & col = columns_[column];
+         check_type<T>(col.type); // throws if type is wrong
+         return bigVector<T>().begin() + col.firstIndex;
+     }
+     template<typename T>
+     typename std::vector<T>::iterator beginData(unsigned int column) {
+         const Column & col = columns_[column];
+         check_type<T>(col.type); // throws if type is wrong
+         return bigVector<T>().begin() + col.firstIndex;
+     }
+
+     template<typename T>
+     const std::vector<T> & bigVector() const { throw cms::Exception("unsupported type"); }
+     template<typename T>
+     std::vector<T> & bigVector() { throw cms::Exception("unsupported type"); }
+
+
+     unsigned int size_;
+     std::string name_, doc_;
+     bool singleton_, extension_;
+     std::vector<Column> columns_;
+     std::vector<float> floats_;
+     std::vector<int> ints_;
+     std::vector<uint8_t> uint8s_;
+
+     template<typename T> 
+     static void check_type(FlatTable::ColumnType type) { throw cms::Exception("unsupported type"); }
+};
+
+template<> inline void FlatTable::check_type<float>(FlatTable::ColumnType type) {
+     if (type != FlatTable::FloatColumn) throw cms::Exception("mismatched type");
+}
+template<> inline void FlatTable::check_type<int>(FlatTable::ColumnType type) {
+     if (type != FlatTable::IntColumn) throw cms::Exception("mismatched type");
+}
+template<> inline void FlatTable::check_type<uint8_t>(FlatTable::ColumnType type) {
+     if (type != FlatTable::UInt8Column && type != FlatTable::BoolColumn) throw cms::Exception("mismatched type");
+}
+
+
+
+template<> inline const std::vector<float>   & FlatTable::bigVector<float>()   const { return floats_; }
+template<> inline const std::vector<int>     & FlatTable::bigVector<int>()     const { return ints_; }
+template<> inline const std::vector<uint8_t> & FlatTable::bigVector<uint8_t>() const { return uint8s_; }
+template<> inline std::vector<float>   & FlatTable::bigVector<float>()   { return floats_; }
+template<> inline std::vector<int>     & FlatTable::bigVector<int>()     { return ints_; }
+template<> inline std::vector<uint8_t> & FlatTable::bigVector<uint8_t>() { return uint8s_; }
+
+} // nanoaod
+
+#endif

DataFormats/NanoAOD/interface/MergeableCounterTable.h

@@ -0,0 +1,111 @@
+#ifndef DataFormats_NanoAOD_MergeableCounterTable_h
+#define DataFormats_NanoAOD_MergeableCounterTable_h
+
+#include "FWCore/Utilities/interface/Exception.h"
+#include <vector>
+#include <string>
+
+namespace nanoaod {
+
+class MergeableCounterTable {
+    public:
+        MergeableCounterTable() {}
+        typedef long long int_accumulator; // we accumulate in long long int, to avoid overflow
+        typedef double float_accumulator; // we accumulate in double, to preserve precision
+
+        template<typename T>
+        struct SingleColumn {
+            typedef T value_type;
+            SingleColumn() {}
+            SingleColumn(const std::string & aname, const std::string & adoc, T avalue = T()) : name(aname), doc(adoc), value(avalue) {}
+            std::string name, doc;
+            T value;
+            void operator+=(const SingleColumn<T> & other) {
+                //// if one arrives here from tryMerge the checks are already done in the compatible() function before.
+                //// you may however want to enable these and remove the 'return false' in tryMerge in order to see what's incompatible between the tables.
+                //if (name != other.name) throw cms::Exception("LogicError", "Trying to merge "+name+" with "+other.name+"\n");
+                value += other.value;
+            }
+            bool compatible(const SingleColumn<T> & other) {
+                return name == other.name;  // we don't check the doc, not needed
+            }
+        };
+        typedef SingleColumn<float_accumulator> FloatColumn; 
+        typedef SingleColumn<int_accumulator> IntColumn;  
+
+        template<typename T>
+        struct VectorColumn {
+            typedef T element_type;
+            VectorColumn() {}
+            VectorColumn(const std::string & aname, const std::string & adoc, unsigned int size) : name(aname), doc(adoc), values(size, T()) {}
+            VectorColumn(const std::string & aname, const std::string & adoc, const std::vector<T> & somevalues) : name(aname), doc(adoc), values(somevalues) {}
+            std::string name, doc;
+            std::vector<T> values;
+            void operator+=(const VectorColumn<T> & other) {
+                //// if one arrives here from tryMerge the checks are already done in the compatible() function before.
+                //// you may however want to enable these and remove the 'return false' in tryMerge in order to see what's incompatible between the tables.
+                //if (name != other.name) throw cms::Exception("LogicError", "Trying to merge "+name+" with "+other.name+"\n");
+                //if (values.size() != other.values.size()) throw cms::Exception("LogicError", "Trying to merge "+name+" with different number of values!\n");
+                for (unsigned int i = 0, n = values.size(); i < n; ++i) {
+                    values[i] += other.values[i];
+                }
+            }
+            bool compatible(const VectorColumn<T> & other) {
+                return name == other.name && values.size() == other.values.size(); // we don't check the doc, not needed
+            }
+        };
+        typedef VectorColumn<float_accumulator> VFloatColumn; 
+        typedef VectorColumn<int_accumulator> VIntColumn;
+
+        const std::vector<FloatColumn> & floatCols() const { return floatCols_; }
+        const std::vector<VFloatColumn> & vfloatCols() const { return vfloatCols_; }
+        const std::vector<IntColumn> & intCols() const { return intCols_; }
+        const std::vector<VIntColumn> & vintCols() const { return vintCols_; }
+
+        template<typename F>
+        void addFloat(const std::string & name, const std::string & doc, F value) { floatCols_.push_back(FloatColumn(name, doc, value)); } 
+
+        template<typename I>
+        void addInt(const std::string & name, const std::string & doc, I value) { intCols_.push_back(IntColumn(name, doc, value)); } 
+
+        template<typename F>
+        void addVFloat(const std::string & name, const std::string & doc, const std::vector<F> values) { 
+            vfloatCols_.push_back(VFloatColumn(name, doc, values.size())); 
+            std::copy(values.begin(), values.end(), vfloatCols_.back().values.begin());
+        } 
+
+        template<typename I>
+        void addVInt(const std::string & name, const std::string & doc, const std::vector<I> values) { 
+            vintCols_.push_back(VIntColumn(name, doc, values.size())); 
+            std::copy(values.begin(), values.end(), vintCols_.back().values.begin());
+        } 
+
+
+        bool mergeProduct(const MergeableCounterTable & other) {
+            if (!tryMerge(intCols_, other.intCols_)) return false;
+            if (!tryMerge(vintCols_, other.vintCols_)) return false;
+            if (!tryMerge(floatCols_, other.floatCols_)) return false;
+            if (!tryMerge(vfloatCols_, other.vfloatCols_)) return false;
+            return true; 
+         }
+
+   private:
+        std::vector<FloatColumn> floatCols_;
+        std::vector<VFloatColumn> vfloatCols_;
+        std::vector<IntColumn> intCols_;
+        std::vector<VIntColumn> vintCols_;
+
+        template<typename T>
+        bool tryMerge(std::vector<T> & one, const std::vector<T> & two) {
+            if (one.size() != two.size()) return false;
+            for (unsigned int i = 0, n = one.size(); i < n; ++i) {
+                if (!one[i].compatible(two[i])) return false;
+                one[i] += two[i];
+            }
+            return true;
+        }
+};
+
+} // namespace nanoaod
+
+#endif

DataFormats/NanoAOD/interface/UniqueString.h

@@ -0,0 +1,22 @@
+#ifndef PhysicsTools_NanoAOD_UniqueString_h
+#define PhysicsTools_NanoAOD_UniqueString_h
+
+#include <string>
+
+namespace nanoaod {
+
+class UniqueString {
+    public:
+        UniqueString() {}
+        UniqueString(const std::string & str) : str_(str) {}
+        const std::string & str() const { return str_; }
+        bool operator==(const std::string & other) const { return str_ == other; }
+        bool operator==(const UniqueString & other) const { return str_ == other.str_; }
+        bool isProductEqual(const UniqueString & other) const { return (*this) == other; }
+    private:
+        std::string str_;
+};
+
+} // namespace nanoaod
+
+#endif

DataFormats/NanoAOD/src/FlatTable.cc

@@ -0,0 +1,8 @@
+#include "DataFormats/NanoAOD/interface/FlatTable.h"
+
+int nanoaod::FlatTable::columnIndex(const std::string & name) const {
+    for (unsigned int i = 0, n = columns_.size(); i < n; ++i) {
+        if (columns_[i].name == name) return i;
+    }
+    return -1;
+}

DataFormats/NanoAOD/src/MergeableCounterTable.cc

@@ -0,0 +1 @@
+#include "DataFormats/NanoAOD/interface/MergeableCounterTable.h"