From 08cd101cd8035f5ad27e3b28e301e746a3d2c1c6 Mon Sep 17 00:00:00 2001 From: Cms Build Date: Wed, 15 May 2019 23:00:07 +0200 Subject: [PATCH] Clang-Format --- IOMC/Input/interface/HepMCFileReader.h | 49 +- IOMC/Input/interface/MCFileSource.h | 8 +- IOMC/Input/plugins/HepMCEventWriter.cc | 38 +- IOMC/Input/src/HepMCFileReader.cc | 229 +-- IOMC/Input/src/MCFileSource.cc | 83 +- IOMC/Input/src/SealModule.cc | 4 +- IOMC/RandomEngine/plugins/Module.cc | 4 +- .../src/RandomEngineStateProducer.cc | 17 +- .../src/RandomEngineStateProducer.h | 16 +- IOMC/RandomEngine/src/RandomFilter.cc | 20 +- IOMC/RandomEngine/src/RandomFilter.h | 5 +- .../src/RandomNumberGeneratorService.cc | 868 ++++----- .../src/RandomNumberGeneratorService.h | 55 +- IOMC/RandomEngine/src/TRandomAdaptor.cc | 204 ++- IOMC/RandomEngine/src/TRandomAdaptor.h | 36 +- .../test/TestRandomNumberServiceGlobal.cc | 259 +-- IgTools/IgProf/plugins/IgProfModule.cc | 193 +- IgTools/IgProf/plugins/IgProfService.cc | 167 +- IgTools/IgProf/plugins/IgProfService.h | 34 +- .../Callgrind/interface/ProfilerService.h | 46 +- .../Callgrind/plugins/CallgrindAnalyzer.cc | 96 +- .../Callgrind/plugins/ProfilerAnalyzer.cc | 60 +- .../plugins/ProfilerServicePlugin.cc | 2 +- PerfTools/Callgrind/src/ProfilerService.cc | 128 +- .../test/ProfilerServiceTest.cppunit.cc | 450 +++-- PerfTools/EdmEvent/bin/edmEventSize.cpp | 152 +- PerfTools/EdmEvent/interface/EdmEventSize.h | 39 +- PerfTools/EdmEvent/src/EdmEventSize.cc | 297 ++-- PerfTools/EdmEvent/test/branchSizes.cpp | 298 ++-- .../DCacheAdaptor/interface/DCacheFile.h | 65 +- .../plugins/DCacheStorageMaker.cc | 61 +- Utilities/DCacheAdaptor/src/DCacheFile.cc | 250 +-- Utilities/DavixAdaptor/interface/DavixFile.h | 19 +- .../DavixAdaptor/plugins/DavixStorageMaker.cc | 16 +- Utilities/DavixAdaptor/src/DavixFile.cc | 89 +- Utilities/DavixAdaptor/test/davixcheck.cpp | 9 +- .../test/davixreadfilenotfound.cpp | 4 +- .../DavixAdaptor/test/davixreadwrongurl.cpp | 6 +- Utilities/DavixAdaptor/test/http.cpp | 22 +- Utilities/DavixAdaptor/test/http2.cpp | 14 +- Utilities/DavixAdaptor/test/http3.cpp | 12 +- Utilities/DavixAdaptor/test/http4.cpp | 14 +- .../LStoreAdaptor/interface/LStoreFile.h | 78 +- .../plugins/LStoreStorageMaker.cc | 48 +- Utilities/LStoreAdaptor/src/LStoreFile.cc | 417 ++--- .../StaticAnalyzers/src/ArgSizeChecker.cpp | 306 ++-- .../StaticAnalyzers/src/ArgSizeChecker.h | 17 +- Utilities/StaticAnalyzers/src/CatchAll.cpp | 51 +- Utilities/StaticAnalyzers/src/CatchAll.h | 8 +- .../src/ClangCmsCheckerPluginRegister.cpp | 80 +- .../StaticAnalyzers/src/ClassChecker.cpp | 1318 +++++++------- Utilities/StaticAnalyzers/src/ClassChecker.h | 16 +- Utilities/StaticAnalyzers/src/ClassDumper.cpp | 422 ++--- Utilities/StaticAnalyzers/src/ClassDumper.h | 79 +- .../StaticAnalyzers/src/CmsException.cpp | 104 +- Utilities/StaticAnalyzers/src/CmsException.h | 43 +- Utilities/StaticAnalyzers/src/CmsSupport.cpp | 270 +-- Utilities/StaticAnalyzers/src/CmsSupport.h | 74 +- .../src/ConstCastAwayChecker.cpp | 73 +- .../src/ConstCastAwayChecker.h | 19 +- .../StaticAnalyzers/src/ConstCastChecker.cpp | 51 +- .../StaticAnalyzers/src/ConstCastChecker.h | 17 +- .../StaticAnalyzers/src/EDMPluginDumper.cc | 47 +- .../StaticAnalyzers/src/EDMPluginDumper.h | 21 +- .../StaticAnalyzers/src/FiniteMathChecker.cc | 64 +- .../StaticAnalyzers/src/FiniteMathChecker.h | 13 +- .../StaticAnalyzers/src/FunctionChecker.cpp | 333 ++-- .../StaticAnalyzers/src/FunctionChecker.h | 40 +- .../StaticAnalyzers/src/FunctionDumper.cpp | 368 ++-- .../StaticAnalyzers/src/FunctionDumper.h | 44 +- .../src/GlobalStaticChecker.cpp | 63 +- .../StaticAnalyzers/src/GlobalStaticChecker.h | 20 +- .../src/MutableMemberChecker.cpp | 58 +- .../src/MutableMemberChecker.h | 19 +- .../src/StaticLocalChecker.cpp | 57 +- .../StaticAnalyzers/src/StaticLocalChecker.h | 20 +- .../src/ThrUnsafeFCallChecker.cpp | 155 +- .../src/ThrUnsafeFCallChecker.h | 35 +- .../StaticAnalyzers/src/TrunCastChecker.cpp | 318 ++-- .../StaticAnalyzers/src/TrunCastChecker.h | 21 +- .../StaticAnalyzers/src/UsingNamespace.cpp | 45 +- .../StaticAnalyzers/src/UsingNamespace.h | 15 +- Utilities/StaticAnalyzers/src/edmChecker.cpp | 48 +- Utilities/StaticAnalyzers/src/edmChecker.h | 19 +- .../StaticAnalyzers/src/getByChecker.cpp | 307 ++-- Utilities/StaticAnalyzers/src/getByChecker.h | 34 +- .../StaticAnalyzers/src/getParamDumper.cpp | 454 ++--- .../StaticAnalyzers/src/getParamDumper.h | 27 +- Utilities/StorageFactory/bin/anycp.cpp | 216 +-- Utilities/StorageFactory/interface/File.h | 92 +- Utilities/StorageFactory/interface/IOBuffer.h | 49 +- .../StorageFactory/interface/IOChannel.h | 39 +- Utilities/StorageFactory/interface/IOFlags.h | 32 +- Utilities/StorageFactory/interface/IOInput.h | 25 +- Utilities/StorageFactory/interface/IOOutput.h | 25 +- .../StorageFactory/interface/IOPosBuffer.h | 80 +- Utilities/StorageFactory/interface/IOTypes.h | 27 +- .../StorageFactory/interface/LocalCacheFile.h | 63 +- .../interface/LocalFileSystem.h | 26 +- .../StorageFactory/interface/RemoteFile.h | 26 +- .../interface/StatisticsSenderService.h | 81 +- Utilities/StorageFactory/interface/Storage.h | 53 +- .../StorageFactory/interface/StorageAccount.h | 116 +- .../interface/StorageAccountProxy.h | 49 +- .../StorageFactory/interface/StorageFactory.h | 116 +- .../StorageFactory/interface/StorageMaker.h | 43 +- .../interface/StorageMakerFactory.h | 6 +- .../plugins/GsiFTPStorageMaker.cc | 28 +- .../plugins/HttpStorageMaker.cc | 27 +- .../plugins/LocalStorageMaker.cc | 58 +- .../plugins/StatisticsSenderServiceMaker.cc | 1 - .../plugins/StormLCGStorageMaker.cc | 55 +- .../plugins/StormStorageMaker.cc | 54 +- Utilities/StorageFactory/src/File.cc | 169 +- Utilities/StorageFactory/src/IOChannel.cc | 28 +- Utilities/StorageFactory/src/IOInput.cc | 57 +- Utilities/StorageFactory/src/IOOutput.cc | 53 +- .../StorageFactory/src/LocalCacheFile.cc | 141 +- .../StorageFactory/src/LocalFileSystem.cc | 313 ++-- Utilities/StorageFactory/src/RemoteFile.cc | 114 +- .../src/StatisticsSenderService.cc | 183 +- Utilities/StorageFactory/src/Storage.cc | 116 +- .../StorageFactory/src/StorageAccount.cc | 94 +- .../StorageFactory/src/StorageAccountProxy.cc | 162 +- .../StorageFactory/src/StorageFactory.cc | 271 +-- Utilities/StorageFactory/src/StorageMaker.cc | 23 +- Utilities/StorageFactory/src/SysFile.h | 36 +- Utilities/StorageFactory/src/SysIOChannel.h | 14 +- Utilities/StorageFactory/src/Throw.cc | 18 +- Utilities/StorageFactory/src/Throw.h | 12 +- Utilities/StorageFactory/src/UnixFile.cc | 92 +- Utilities/StorageFactory/src/UnixIOChannel.cc | 94 +- Utilities/StorageFactory/test/Test.h | 31 +- Utilities/StorageFactory/test/any.cpp | 29 +- Utilities/StorageFactory/test/ftp.cpp | 12 +- Utilities/StorageFactory/test/ftp2.cpp | 14 +- Utilities/StorageFactory/test/local.cpp | 19 +- Utilities/StorageFactory/test/local2.cpp | 13 +- Utilities/StorageFactory/test/local3.cpp | 11 +- Utilities/StorageFactory/test/mkstemp.cpp | 21 +- Utilities/StorageFactory/test/randomread.cpp | 63 +- Utilities/StorageFactory/test/rfio2.cpp | 16 +- Utilities/StorageFactory/test/rfio3.cpp | 13 +- Utilities/StorageFactory/test/t0Repack.cpp | 170 +- Utilities/StorageFactory/test/write.cpp | 34 +- .../Testing/interface/CppUnit_testdriver.icpp | 75 +- Utilities/Xerces/interface/XercesStrUtils.h | 40 +- .../XrdAdaptor/plugins/XrdStorageMaker.cc | 107 +- Utilities/XrdAdaptor/src/QualityMetric.cc | 264 ++- Utilities/XrdAdaptor/src/QualityMetric.h | 75 +- Utilities/XrdAdaptor/src/XrdFile.cc | 337 ++-- Utilities/XrdAdaptor/src/XrdFile.h | 89 +- Utilities/XrdAdaptor/src/XrdHostHandler.hh | 54 +- Utilities/XrdAdaptor/src/XrdRequest.cc | 178 +- Utilities/XrdAdaptor/src/XrdRequest.h | 71 +- Utilities/XrdAdaptor/src/XrdRequestManager.cc | 1582 ++++++++--------- Utilities/XrdAdaptor/src/XrdRequestManager.h | 182 +- Utilities/XrdAdaptor/src/XrdSource.cc | 658 ++++--- Utilities/XrdAdaptor/src/XrdSource.h | 51 +- Utilities/XrdAdaptor/src/XrdStatistics.cc | 265 ++- Utilities/XrdAdaptor/src/XrdStatistics.h | 102 +- 161 files changed, 8462 insertions(+), 10065 deletions(-) mode change 100755 => 100644 PerfTools/EdmEvent/bin/edmEventSize.cpp mode change 100755 => 100644 PerfTools/EdmEvent/test/branchSizes.cpp diff --git a/IOMC/Input/interface/HepMCFileReader.h b/IOMC/Input/interface/HepMCFileReader.h index a25106dcd8bd2..ef633d41eb1cb 100644 --- a/IOMC/Input/interface/HepMCFileReader.h +++ b/IOMC/Input/interface/HepMCFileReader.h @@ -1,7 +1,6 @@ #ifndef Input_HepMCFileReader_h #define Input_HepMCFileReader_h - /** \class HepMCFileReader * * This class is used by the implementation of DaqEventFactory present @@ -12,7 +11,7 @@ * it exposes. * * \author G. Bruno - CERN, EP Division -*/ +*/ #include #include @@ -23,60 +22,52 @@ namespace HepMC { class IO_BaseClass; class GenEvent; class GenParticle; -} - +} // namespace HepMC class HepMCFileReader { - - protected: +protected: HepMCFileReader(); - - public: - virtual ~HepMCFileReader(); - virtual void initialize(const std::string &filename); + +public: + virtual ~HepMCFileReader(); + virtual void initialize(const std::string &filename); inline bool isInitialized() const; virtual bool setEvent(int event); virtual bool readCurrentEvent(); - virtual bool printHepMcEvent() const; + virtual bool printHepMcEvent() const; HepMC::GenEvent *fillCurrentEventData(); // virtual bool fillEventData(HepMC::GenEvent *event); - // this method prints the event information as + // this method prints the event information as // obtained by the input file in HepEvt style void printEvent() const; - // get all the 'integer' properties of a particle + // get all the 'integer' properties of a particle // like mother, daughter, pid and status // 'j' is the number of the particle in the HepMc - virtual void getStatsFromTuple(int &mo1, int &mo2, int &da1, int &da2, - int &status, int &pid, int j) const; + virtual void getStatsFromTuple(int &mo1, int &mo2, int &da1, int &da2, int &status, int &pid, int j) const; virtual void ReadStats(); static HepMCFileReader *instance(); - private: - HepMC::IO_BaseClass const* input() const {return get_underlying_safe(input_);} - HepMC::IO_BaseClass*& input() {return get_underlying_safe(input_);} +private: + HepMC::IO_BaseClass const *input() const { return get_underlying_safe(input_); } + HepMC::IO_BaseClass *&input() { return get_underlying_safe(input_); } // current HepMC evt - edm::propagate_const evt_; - edm::propagate_const input_; + edm::propagate_const evt_; + edm::propagate_const input_; static HepMCFileReader *instance_; int rdstate() const; //maps to convert HepMC::GenParticle to particles # and vice versa // -> needed for HepEvt like output - std::vector index_to_particle; - std::map particle_to_index; + std::vector index_to_particle; + std::map particle_to_index; // find index to HepMC::GenParticle* p in map m - int find_in_map(const std::map& m, - HepMC::GenParticle *p) const; + int find_in_map(const std::map &m, HepMC::GenParticle *p) const; }; - -bool HepMCFileReader::isInitialized() const -{ - return input_ != nullptr; -} +bool HepMCFileReader::isInitialized() const { return input_ != nullptr; } #endif diff --git a/IOMC/Input/interface/MCFileSource.h b/IOMC/Input/interface/MCFileSource.h index ce2b967eae461..de85a6ab4e652 100644 --- a/IOMC/Input/interface/MCFileSource.h +++ b/IOMC/Input/interface/MCFileSource.h @@ -12,7 +12,7 @@ class HepMCFileReader; -namespace HepMC{ +namespace HepMC { class GenEvent; } @@ -30,13 +30,13 @@ namespace edm { private: bool setRunAndEventInfo(EventID&, TimeValue_t& time, EventAuxiliary::ExperimentType& eType) override; - void produce(Event &e) override; + void produce(Event& e) override; void clear(); - + edm::propagate_const reader_; edm::propagate_const evt_; bool useExtendedAscii_; }; -} +} // namespace edm #endif diff --git a/IOMC/Input/plugins/HepMCEventWriter.cc b/IOMC/Input/plugins/HepMCEventWriter.cc index df0095c346ada..d21734c3239d8 100644 --- a/IOMC/Input/plugins/HepMCEventWriter.cc +++ b/IOMC/Input/plugins/HepMCEventWriter.cc @@ -17,54 +17,42 @@ #include "HepMC/IO_GenEvent.h" - class HepMCEventWriter : public edm::EDAnalyzer { public: explicit HepMCEventWriter(const edm::ParameterSet ¶ms); ~HepMCEventWriter() override; - + protected: void beginRun(const edm::Run &run, const edm::EventSetup &es) override; void endRun(const edm::Run &run, const edm::EventSetup &es) override; void analyze(const edm::Event &event, const edm::EventSetup &es) override; - + private: - edm::propagate_const _output; + edm::propagate_const _output; edm::InputTag hepMCProduct_; }; -HepMCEventWriter::HepMCEventWriter(const edm::ParameterSet ¶ms) : - hepMCProduct_(params.getParameter("hepMCProduct")) -{ -} - -HepMCEventWriter::~HepMCEventWriter() -{ -} - -void HepMCEventWriter::beginRun(const edm::Run &run, const edm::EventSetup &es) -{ +HepMCEventWriter::HepMCEventWriter(const edm::ParameterSet ¶ms) + : hepMCProduct_(params.getParameter("hepMCProduct")) {} - _output = new HepMC::IO_GenEvent("GenEvent_ASCII.dat",std::ios::out); +HepMCEventWriter::~HepMCEventWriter() {} +void HepMCEventWriter::beginRun(const edm::Run &run, const edm::EventSetup &es) { + _output = new HepMC::IO_GenEvent("GenEvent_ASCII.dat", std::ios::out); } - -void HepMCEventWriter::endRun(const edm::Run &run, const edm::EventSetup &es) -{ - if (_output) delete _output.get(); +void HepMCEventWriter::endRun(const edm::Run &run, const edm::EventSetup &es) { + if (_output) + delete _output.get(); } -void HepMCEventWriter::analyze(const edm::Event &event, const edm::EventSetup &es) -{ - +void HepMCEventWriter::analyze(const edm::Event &event, const edm::EventSetup &es) { edm::Handle product; event.getByLabel(hepMCProduct_, product); - const HepMC::GenEvent* evt = product->GetEvent(); + const HepMC::GenEvent *evt = product->GetEvent(); _output->write_event(evt); - } DEFINE_FWK_MODULE(HepMCEventWriter); diff --git a/IOMC/Input/src/HepMCFileReader.cc b/IOMC/Input/src/HepMCFileReader.cc index a1f283684efb7..bca51b22cb898 100644 --- a/IOMC/Input/src/HepMCFileReader.cc +++ b/IOMC/Input/src/HepMCFileReader.cc @@ -22,48 +22,38 @@ using namespace std; - -HepMCFileReader *HepMCFileReader::instance_=nullptr; - +HepMCFileReader *HepMCFileReader::instance_ = nullptr; //------------------------------------------------------------------------- -HepMCFileReader *HepMCFileReader::instance() -{ +HepMCFileReader *HepMCFileReader::instance() { // Implement a HepMCFileReader singleton. - if (instance_== nullptr) { + if (instance_ == nullptr) { instance_ = new HepMCFileReader(); } return instance_; } - //------------------------------------------------------------------------- -HepMCFileReader::HepMCFileReader() : - evt_(nullptr), input_(nullptr) -{ +HepMCFileReader::HepMCFileReader() : evt_(nullptr), input_(nullptr) { // Default constructor. if (instance_ == nullptr) { - instance_ = this; + instance_ = this; } else { edm::LogError("HepMCFileReader") << "Constructing a new instance"; } -} - +} //------------------------------------------------------------------------- -HepMCFileReader::~HepMCFileReader() -{ +HepMCFileReader::~HepMCFileReader() { edm::LogInfo("HepMCFileReader") << "Destructing HepMCFileReader"; - - instance_=nullptr; + + instance_ = nullptr; delete input_.get(); } - //------------------------------------------------------------------------- -void HepMCFileReader::initialize(const string &filename) -{ +void HepMCFileReader::initialize(const string &filename) { if (isInitialized()) { edm::LogError("HepMCFileReader") << "Was already initialized... reinitializing"; delete input_.get(); @@ -73,220 +63,193 @@ void HepMCFileReader::initialize(const string &filename) input_ = new HepMC::IO_GenEvent(filename, std::ios::in); if (rdstate() == std::ios::failbit) { - throw cms::Exception("FileNotFound", "HepMCFileReader::initialize()") - << "File " << filename << " was not found.\n"; + throw cms::Exception("FileNotFound", "HepMCFileReader::initialize()") << "File " << filename << " was not found.\n"; } } - //------------------------------------------------------------------------- -int HepMCFileReader::rdstate() const -{ +int HepMCFileReader::rdstate() const { // work around a HepMC IO_ inheritence shortfall - HepMC::IO_GenEvent const* p = dynamic_cast(input()); - if (p) return p->rdstate(); + HepMC::IO_GenEvent const *p = dynamic_cast(input()); + if (p) + return p->rdstate(); return std::ios::failbit; } - //------------------------------------------------------------------------- -bool HepMCFileReader::readCurrentEvent() -{ +bool HepMCFileReader::readCurrentEvent() { evt_ = input_->read_next_event(); - if (evt_) { - edm::LogInfo("HepMCFileReader") << "| --- Event Nr. " << evt_->event_number() - << " with " << evt_->particles_size() << " particles --- !"; + if (evt_) { + edm::LogInfo("HepMCFileReader") << "| --- Event Nr. " << evt_->event_number() << " with " << evt_->particles_size() + << " particles --- !"; ReadStats(); // printHepMcEvent(); // printEvent(); } else { - edm::LogInfo("HepMCFileReader") << "Got no event" <print(); +bool HepMCFileReader::printHepMcEvent() const { + if (evt_ != nullptr) + evt_->print(); return true; } - //------------------------------------------------------------------------- -HepMC::GenEvent * HepMCFileReader::fillCurrentEventData() -{ +HepMC::GenEvent *HepMCFileReader::fillCurrentEventData() { readCurrentEvent(); return evt_; } - //------------------------------------------------------------------------- // Print out in old CMKIN style for comparisons void HepMCFileReader::printEvent() const { - int mo1=0,mo2=0,da1=0,da2=0,status=0,pid=0; - if (evt_ != nullptr) { + int mo1 = 0, mo2 = 0, da1 = 0, da2 = 0, status = 0, pid = 0; + if (evt_ != nullptr) { cout << "---#-------pid--st---Mo1---Mo2---Da1---Da2------px------py------pz-------E-"; cout << "------m---------x---------y---------z---------t-"; cout << endl; cout.setf(ios::right, ios::adjustfield); - for(int n=1; n<=evt_->particles_size(); n++) { - HepMC::GenParticle *g = index_to_particle[n]; - getStatsFromTuple( mo1,mo2,da1,da2,status,pid,n); - cout << setw(4) << n - << setw(8) << pid - << setw(5) << status - << setw(6) << mo1 - << setw(6) << mo2 - << setw(6) << da1 - << setw(6) << da2; + for (int n = 1; n <= evt_->particles_size(); n++) { + HepMC::GenParticle *g = index_to_particle[n]; + getStatsFromTuple(mo1, mo2, da1, da2, status, pid, n); + cout << setw(4) << n << setw(8) << pid << setw(5) << status << setw(6) << mo1 << setw(6) << mo2 << setw(6) << da1 + << setw(6) << da2; cout.setf(ios::fixed, ios::floatfield); cout.setf(ios::right, ios::adjustfield); cout << setw(10) << setprecision(2) << g->momentum().x(); cout << setw(8) << setprecision(2) << g->momentum().y(); cout << setw(10) << setprecision(2) << g->momentum().z(); cout << setw(8) << setprecision(2) << g->momentum().t(); - cout << setw(8) << setprecision(2) << g->generatedMass(); - // tau=L/(gamma*beta*c) + cout << setw(8) << setprecision(2) << g->generatedMass(); + // tau=L/(gamma*beta*c) if (g->production_vertex() != nullptr && g->end_vertex() != nullptr && status == 2) { - cout << setw(10) << setprecision(2) <production_vertex()->position().x(); - cout << setw(10) << setprecision(2) <production_vertex()->position().y(); - cout << setw(10) << setprecision(2) <production_vertex()->position().z(); - + cout << setw(10) << setprecision(2) << g->production_vertex()->position().x(); + cout << setw(10) << setprecision(2) << g->production_vertex()->position().y(); + cout << setw(10) << setprecision(2) << g->production_vertex()->position().z(); + double xm = g->production_vertex()->position().x(); double ym = g->production_vertex()->position().y(); double zm = g->production_vertex()->position().z(); double xd = g->end_vertex()->position().x(); double yd = g->end_vertex()->position().y(); double zd = g->end_vertex()->position().z(); - double decl = sqrt((xd-xm)*(xd-xm)+(yd-ym)*(yd-ym)+(zd-zm)*(zd-zm)); - double labTime = decl/c_light; + double decl = sqrt((xd - xm) * (xd - xm) + (yd - ym) * (yd - ym) + (zd - zm) * (zd - zm)); + double labTime = decl / c_light; // convert lab time to proper time - double properTime = labTime/g->momentum().rho()*(g->generatedMass() ); + double properTime = labTime / g->momentum().rho() * (g->generatedMass()); // set the proper time in nanoseconds - cout << setw(8) << setprecision(2) <particles_size()+1); +void HepMCFileReader::ReadStats() { + unsigned int particle_counter = 0; + index_to_particle.reserve(evt_->particles_size() + 1); index_to_particle[0] = nullptr; - for (HepMC::GenEvent::vertex_const_iterator v = evt_->vertices_begin(); - v != evt_->vertices_end(); ++v ) { + for (HepMC::GenEvent::vertex_const_iterator v = evt_->vertices_begin(); v != evt_->vertices_end(); ++v) { // making a list of incoming particles of the vertices // so that the mother indices in HEPEVT can be filled properly for (HepMC::GenVertex::particles_in_const_iterator p1 = (*v)->particles_in_const_begin(); - p1 != (*v)->particles_in_const_end(); ++p1 ) { + p1 != (*v)->particles_in_const_end(); + ++p1) { ++particle_counter; //particle_counter can be very large for heavy ions - if(particle_counter >= index_to_particle.size() ) { + if (particle_counter >= index_to_particle.size()) { //make it large enough to hold up to this index - index_to_particle.resize(particle_counter+1); - } + index_to_particle.resize(particle_counter + 1); + } index_to_particle[particle_counter] = *p1; particle_to_index[*p1] = particle_counter; - } + } // daughters are entered only if they aren't a mother of // another vertex for (HepMC::GenVertex::particles_out_const_iterator p2 = (*v)->particles_out_const_begin(); - p2 != (*v)->particles_out_const_end(); ++p2) { + p2 != (*v)->particles_out_const_end(); + ++p2) { if (!(*p2)->end_vertex()) { ++particle_counter; //particle_counter can be very large for heavy ions - if(particle_counter >= index_to_particle.size() ) { + if (particle_counter >= index_to_particle.size()) { //make it large enough to hold up to this index - index_to_particle.resize(particle_counter+1); - } + index_to_particle.resize(particle_counter + 1); + } index_to_particle[particle_counter] = *p2; particle_to_index[*p2] = particle_counter; } } - } + } } - //------------------------------------------------------------------------- -void HepMCFileReader::getStatsFromTuple(int &mo1, int &mo2, int &da1, int &da2, - int &status, int &pid, int j) const -{ +void HepMCFileReader::getStatsFromTuple(int &mo1, int &mo2, int &da1, int &da2, int &status, int &pid, int j) const { if (!evt_) { - cout << "HepMCFileReader: Got no event :-( Game over already ?" <status(); + status = index_to_particle[j]->status(); pid = index_to_particle[j]->pdg_id(); - if ( index_to_particle[j]->production_vertex() ) { - - //HepLorentzVector p = index_to_particle[j]-> - //production_vertex()->position(); - - int num_mothers = index_to_particle[j]->production_vertex()-> - particles_in_size(); - int first_mother = find_in_map( particle_to_index, - *(index_to_particle[j]-> - production_vertex()-> - particles_in_const_begin())); + if (index_to_particle[j]->production_vertex()) { + //HepLorentzVector p = index_to_particle[j]-> + //production_vertex()->position(); + + int num_mothers = index_to_particle[j]->production_vertex()->particles_in_size(); + int first_mother = + find_in_map(particle_to_index, *(index_to_particle[j]->production_vertex()->particles_in_const_begin())); int last_mother = first_mother + num_mothers - 1; - if ( first_mother == 0 ) last_mother = 0; - mo1=first_mother; - mo2=last_mother; + if (first_mother == 0) + last_mother = 0; + mo1 = first_mother; + mo2 = last_mother; } else { - mo1 =0; - mo2 =0; - } + mo1 = 0; + mo2 = 0; + } if (!index_to_particle[j]->end_vertex()) { //find # of 1. daughter - int first_daughter = find_in_map( particle_to_index, - *(index_to_particle[j]->end_vertex()->particles_begin(HepMC::children))); - //cout <<"first_daughter "<< first_daughter << "num_daughters " << num_daughters << endl; + int first_daughter = + find_in_map(particle_to_index, *(index_to_particle[j]->end_vertex()->particles_begin(HepMC::children))); + //cout <<"first_daughter "<< first_daughter << "num_daughters " << num_daughters << endl; HepMC::GenVertex::particle_iterator ic; - int last_daughter=0; + int last_daughter = 0; //find # of last daughter for (ic = index_to_particle[j]->end_vertex()->particles_begin(HepMC::children); - ic != index_to_particle[j]->end_vertex()->particles_end(HepMC::children); ++ic) - last_daughter= find_in_map( particle_to_index,*ic); - - if (first_daughter== 0) last_daughter = 0; - da1=first_daughter; - da2=last_daughter; + ic != index_to_particle[j]->end_vertex()->particles_end(HepMC::children); + ++ic) + last_daughter = find_in_map(particle_to_index, *ic); + + if (first_daughter == 0) + last_daughter = 0; + da1 = first_daughter; + da2 = last_daughter; } else { - da1=0; - da2=0; + da1 = 0; + da2 = 0; } - } -} - + } +} //------------------------------------------------------------------------- -int HepMCFileReader::find_in_map( const std::map& m, - HepMC::GenParticle *p) const -{ - std::map::const_iterator iter = m.find(p); +int HepMCFileReader::find_in_map(const std::map &m, HepMC::GenParticle *p) const { + std::map::const_iterator iter = m.find(p); return (iter == m.end()) ? 0 : iter->second; } - diff --git a/IOMC/Input/src/MCFileSource.cc b/IOMC/Input/src/MCFileSource.cc index 9f37ee848c0bb..34fc70988942c 100644 --- a/IOMC/Input/src/MCFileSource.cc +++ b/IOMC/Input/src/MCFileSource.cc @@ -9,56 +9,51 @@ #include #include - #include "FWCore/Framework/interface/Event.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "IOMC/Input/interface/HepMCFileReader.h" +#include "IOMC/Input/interface/HepMCFileReader.h" #include "IOMC/Input/interface/MCFileSource.h" #include "SimDataFormats/GeneratorProducts/interface/HepMCProduct.h" #include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h" namespace edm { -//------------------------------------------------------------------------- -MCFileSource::MCFileSource(const ParameterSet & pset, InputSourceDescription const& desc) : - ProducerSourceFromFiles(pset, desc, false), - reader_(HepMCFileReader::instance()), evt_(nullptr) -{ - LogInfo("MCFileSource") << "Reading HepMC file:" << fileNames()[0]; - std::string fileName = fileNames()[0]; - // strip the file: - if (fileName.find("file:") == 0){ - fileName.erase(0,5); - } - - reader_->initialize(fileName); - produces("generator"); - produces("generator"); -} - - -//------------------------------------------------------------------------- -MCFileSource::~MCFileSource(){ -} - -//------------------------------------------------------------------------- -bool MCFileSource::setRunAndEventInfo(EventID&, TimeValue_t&, EventAuxiliary::ExperimentType&) { - // Read one HepMC event - LogInfo("MCFileSource") << "Start Reading"; - evt_ = reader_->fillCurrentEventData(); - return(evt_ != nullptr); -} - -//------------------------------------------------------------------------- -void MCFileSource::produce(Event &e) { - // Store one HepMC event in the Event. - - auto bare_product = std::make_unique(); - bare_product->addHepMCData(evt_); - e.put(std::move(bare_product),"generator"); - std::unique_ptr info ( new GenEventInfoProduct( evt_ ) ); - e.put(std::move(info) ,"generator"); -} - -} + //------------------------------------------------------------------------- + MCFileSource::MCFileSource(const ParameterSet& pset, InputSourceDescription const& desc) + : ProducerSourceFromFiles(pset, desc, false), reader_(HepMCFileReader::instance()), evt_(nullptr) { + LogInfo("MCFileSource") << "Reading HepMC file:" << fileNames()[0]; + std::string fileName = fileNames()[0]; + // strip the file: + if (fileName.find("file:") == 0) { + fileName.erase(0, 5); + } + + reader_->initialize(fileName); + produces("generator"); + produces("generator"); + } + + //------------------------------------------------------------------------- + MCFileSource::~MCFileSource() {} + + //------------------------------------------------------------------------- + bool MCFileSource::setRunAndEventInfo(EventID&, TimeValue_t&, EventAuxiliary::ExperimentType&) { + // Read one HepMC event + LogInfo("MCFileSource") << "Start Reading"; + evt_ = reader_->fillCurrentEventData(); + return (evt_ != nullptr); + } + + //------------------------------------------------------------------------- + void MCFileSource::produce(Event& e) { + // Store one HepMC event in the Event. + + auto bare_product = std::make_unique(); + bare_product->addHepMCData(evt_); + e.put(std::move(bare_product), "generator"); + std::unique_ptr info(new GenEventInfoProduct(evt_)); + e.put(std::move(info), "generator"); + } + +} // namespace edm diff --git a/IOMC/Input/src/SealModule.cc b/IOMC/Input/src/SealModule.cc index 40de1e16bab7c..4e90c3a4647c0 100644 --- a/IOMC/Input/src/SealModule.cc +++ b/IOMC/Input/src/SealModule.cc @@ -1,8 +1,6 @@ #include "FWCore/Framework/interface/InputSourceMacros.h" #include "IOMC/Input/interface/MCFileSource.h" +using edm::MCFileSource; -using edm::MCFileSource; - DEFINE_FWK_INPUT_SOURCE(MCFileSource); - diff --git a/IOMC/RandomEngine/plugins/Module.cc b/IOMC/RandomEngine/plugins/Module.cc index 294aa6dbb25fd..a7e78c1ae8716 100644 --- a/IOMC/RandomEngine/plugins/Module.cc +++ b/IOMC/RandomEngine/plugins/Module.cc @@ -6,10 +6,10 @@ #include "IOMC/RandomEngine/src/RandomEngineStateProducer.h" #include "IOMC/RandomEngine/src/RandomFilter.h" -using edm::service::RandomNumberGeneratorService; using edm::RandomFilter; +using edm::service::RandomNumberGeneratorService; -typedef edm::serviceregistry::AllArgsMaker RandomMaker; +typedef edm::serviceregistry::AllArgsMaker RandomMaker; DEFINE_FWK_SERVICE_MAKER(RandomNumberGeneratorService, RandomMaker); DEFINE_FWK_MODULE(RandomEngineStateProducer); diff --git a/IOMC/RandomEngine/src/RandomEngineStateProducer.cc b/IOMC/RandomEngine/src/RandomEngineStateProducer.cc index 96b54680ab8d5..56f2393eda6b0 100644 --- a/IOMC/RandomEngine/src/RandomEngineStateProducer.cc +++ b/IOMC/RandomEngine/src/RandomEngineStateProducer.cc @@ -16,31 +16,28 @@ RandomEngineStateProducer::RandomEngineStateProducer(edm::ParameterSet const&) { produces(); } -RandomEngineStateProducer::~RandomEngineStateProducer() { -} +RandomEngineStateProducer::~RandomEngineStateProducer() {} -void -RandomEngineStateProducer::produce(edm::StreamID iID, edm::Event& ev, edm::EventSetup const&) const { +void RandomEngineStateProducer::produce(edm::StreamID iID, edm::Event& ev, edm::EventSetup const&) const { edm::Service randomService; - if(randomService.isAvailable()) { + if (randomService.isAvailable()) { auto states = std::make_unique(); states->setRandomEngineStates(randomService->getEventCache(ev.streamID())); ev.put(std::move(states)); } } -void -RandomEngineStateProducer::globalBeginLuminosityBlockProduce(edm::LuminosityBlock& lb, edm::EventSetup const&) const { +void RandomEngineStateProducer::globalBeginLuminosityBlockProduce(edm::LuminosityBlock& lb, + edm::EventSetup const&) const { edm::Service randomService; - if(randomService.isAvailable()) { + if (randomService.isAvailable()) { auto states = std::make_unique(); states->setRandomEngineStates(randomService->getLumiCache(lb.index())); lb.put(std::move(states), "beginLumi"); } } -void -RandomEngineStateProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { +void RandomEngineStateProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { edm::ParameterSetDescription desc; descriptions.add("randomEngineStateProducer", desc); } diff --git a/IOMC/RandomEngine/src/RandomEngineStateProducer.h b/IOMC/RandomEngine/src/RandomEngineStateProducer.h index df38893782896..3adb11bb1cc37 100644 --- a/IOMC/RandomEngine/src/RandomEngineStateProducer.h +++ b/IOMC/RandomEngine/src/RandomEngineStateProducer.h @@ -2,7 +2,7 @@ // // Package: RandomEngine // Class: RandomEngineStateProducer -// +// /** \class RandomEngineStateProducer Description: Gets the state of the random number engines from @@ -26,12 +26,12 @@ namespace edm { } class RandomEngineStateProducer : public edm::global::EDProducer { - public: - explicit RandomEngineStateProducer(edm::ParameterSet const& pset); - ~RandomEngineStateProducer() override; - static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); +public: + explicit RandomEngineStateProducer(edm::ParameterSet const& pset); + ~RandomEngineStateProducer() override; + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); - private: - void globalBeginLuminosityBlockProduce(edm::LuminosityBlock&, edm::EventSetup const&) const override; - void produce(edm::StreamID iID, edm::Event& ev, edm::EventSetup const& es) const override; +private: + void globalBeginLuminosityBlockProduce(edm::LuminosityBlock&, edm::EventSetup const&) const override; + void produce(edm::StreamID iID, edm::Event& ev, edm::EventSetup const& es) const override; }; diff --git a/IOMC/RandomEngine/src/RandomFilter.cc b/IOMC/RandomEngine/src/RandomFilter.cc index 332d5ebb184e6..a6bd7be6d25ff 100644 --- a/IOMC/RandomEngine/src/RandomFilter.cc +++ b/IOMC/RandomEngine/src/RandomFilter.cc @@ -11,24 +11,22 @@ using namespace edm; -RandomFilter::RandomFilter(edm::ParameterSet const& ps) : - acceptRate_(ps.getUntrackedParameter("acceptRate")) { +RandomFilter::RandomFilter(edm::ParameterSet const& ps) : acceptRate_(ps.getUntrackedParameter("acceptRate")) { Service rng; - if(!rng.isAvailable()) { - throw cms::Exception("Configuration") - << "RandomFilter requires the RandomNumberGeneratorService,\n" - "which is not present in the configuration file. You must add\n" - "the service in the configuration file or remove the modules that\n" - "require it.\n"; + if (!rng.isAvailable()) { + throw cms::Exception("Configuration") << "RandomFilter requires the RandomNumberGeneratorService,\n" + "which is not present in the configuration file. You must add\n" + "the service in the configuration file or remove the modules that\n" + "require it.\n"; } } -RandomFilter::~RandomFilter() { -} +RandomFilter::~RandomFilter() {} bool RandomFilter::filter(edm::Event& event, edm::EventSetup const&) { Service rng; CLHEP::HepRandomEngine& engine = rng->getEngine(event.streamID()); - if (engine.flat() < acceptRate_) return true; + if (engine.flat() < acceptRate_) + return true; return false; } diff --git a/IOMC/RandomEngine/src/RandomFilter.h b/IOMC/RandomEngine/src/RandomFilter.h index 24d51ba06ecce..106479e075029 100644 --- a/IOMC/RandomEngine/src/RandomFilter.h +++ b/IOMC/RandomEngine/src/RandomFilter.h @@ -2,7 +2,7 @@ // // Package: RandomEngine // Class: RandomFilter -// +// /**\class RandomFilter RandomFilter.h IOMC/RandomEngine/src/RandomFilter.h Description: The output of this module is used for test purposes. @@ -33,8 +33,7 @@ namespace edm { bool filter(edm::Event& e, edm::EventSetup const& c) override; private: - // value between 0 and 1 double acceptRate_; }; -} +} // namespace edm diff --git a/IOMC/RandomEngine/src/RandomNumberGeneratorService.cc b/IOMC/RandomEngine/src/RandomNumberGeneratorService.cc index 0af9d58792f0e..6686b4d2bb3d4 100644 --- a/IOMC/RandomEngine/src/RandomNumberGeneratorService.cc +++ b/IOMC/RandomEngine/src/RandomNumberGeneratorService.cc @@ -52,47 +52,45 @@ namespace edm { namespace service { - const std::vector::size_type RandomNumberGeneratorService::maxSeeds = 65536U; - const std::vector::size_type RandomNumberGeneratorService::maxStates = 65536U; - const std::uint32_t RandomNumberGeneratorService::maxSeedRanecu = 2147483647U; - const std::uint32_t RandomNumberGeneratorService::maxSeedHepJames = 900000000U; + const std::vector::size_type RandomNumberGeneratorService::maxSeeds = 65536U; + const std::vector::size_type RandomNumberGeneratorService::maxStates = 65536U; + const std::uint32_t RandomNumberGeneratorService::maxSeedRanecu = 2147483647U; + const std::uint32_t RandomNumberGeneratorService::maxSeedHepJames = 900000000U; const std::uint32_t RandomNumberGeneratorService::maxSeedTRandom3 = 4294967295U; RandomNumberGeneratorService::RandomNumberGeneratorService(ParameterSet const& pset, - ActivityRegistry& activityRegistry): - nStreams_(0), - saveFileName_(pset.getUntrackedParameter("saveFileName")), - saveFileNameRecorded_(false), - restoreFileName_(pset.getUntrackedParameter("restoreFileName")), - enableChecking_(pset.getUntrackedParameter("enableChecking")), - eventSeedOffset_(pset.getUntrackedParameter("eventSeedOffset")), - verbose_(pset.getUntrackedParameter("verbose")) { - - if(pset.exists("restoreStateTag")) { + ActivityRegistry& activityRegistry) + : nStreams_(0), + saveFileName_(pset.getUntrackedParameter("saveFileName")), + saveFileNameRecorded_(false), + restoreFileName_(pset.getUntrackedParameter("restoreFileName")), + enableChecking_(pset.getUntrackedParameter("enableChecking")), + eventSeedOffset_(pset.getUntrackedParameter("eventSeedOffset")), + verbose_(pset.getUntrackedParameter("verbose")) { + if (pset.exists("restoreStateTag")) { restoreStateTag_ = pset.getUntrackedParameter("restoreStateTag"); - if(restoreStateTag_.process().empty()) { + if (restoreStateTag_.process().empty()) { restoreStateTag_ = edm::InputTag(restoreStateTag_.label(), "", edm::InputTag::kSkipCurrentProcess); } } else { - restoreStateTag_ = edm::InputTag(pset.getUntrackedParameter("restoreStateLabel"), "", edm::InputTag::kSkipCurrentProcess); + restoreStateTag_ = edm::InputTag( + pset.getUntrackedParameter("restoreStateLabel"), "", edm::InputTag::kSkipCurrentProcess); } restoreStateBeginLumiTag_ = edm::InputTag(restoreStateTag_.label(), "beginLumi", restoreStateTag_.process()); - if(!restoreFileName_.empty() && !restoreStateTag_.label().empty()) { - throw Exception(errors::Configuration) - << "In the configuration for the RandomNumberGeneratorService both\n" - << "restoreFileName and restoreStateLabel were set to nonempty values\n" - << "which is illegal. It is impossible to restore the random engine\n" - << "states two different ways in the same process.\n"; + if (!restoreFileName_.empty() && !restoreStateTag_.label().empty()) { + throw Exception(errors::Configuration) << "In the configuration for the RandomNumberGeneratorService both\n" + << "restoreFileName and restoreStateLabel were set to nonempty values\n" + << "which is illegal. It is impossible to restore the random engine\n" + << "states two different ways in the same process.\n"; } // The saveFileName must correspond to a file name without any path specification. // Throw if that is not true. - if(!saveFileName_.empty() && (saveFileName_.find("/") != std::string::npos)) { + if (!saveFileName_.empty() && (saveFileName_.find("/") != std::string::npos)) { throw Exception(errors::Configuration) - << "The saveFileName parameter must be a simple file name with no path\n" - << "specification. In the configuration, it was given the value \"" - << saveFileName_ << "\"\n"; + << "The saveFileName parameter must be a simple file name with no path\n" + << "specification. In the configuration, it was given the value \"" << saveFileName_ << "\"\n"; } std::uint32_t initialSeed; @@ -100,91 +98,91 @@ namespace edm { std::string engineName; std::vector pSets = pset.getParameterNamesForType(); - for(auto const& label : pSets) { - + for (auto const& label : pSets) { ParameterSet const& modulePSet = pset.getParameterSet(label); engineName = modulePSet.getUntrackedParameter("engineName", std::string("HepJamesRandom")); bool initialSeedExists = modulePSet.exists("initialSeed"); bool initialSeedSetExists = modulePSet.exists("initialSeedSet"); - if(initialSeedExists && initialSeedSetExists) { - throw Exception(errors::Configuration) - << "For the module with the label \"" << label << "\",\n" - << "both the parameters \"initialSeed\" and \"initialSeedSet\"\n" - << "have been set in the configuration. You must set one or\n" - << "the other. It is illegal to set both.\n"; - } else if(!initialSeedExists && !initialSeedSetExists) { - throw Exception(errors::Configuration) - << "For the module with the label \"" << label << "\",\n" - << "neither the parameter \"initialSeed\" nor \"initialSeedSet\"\n" - << "has been set in the configuration. You must set one or\n" - << "the other.\n"; - } else if(initialSeedExists) { + if (initialSeedExists && initialSeedSetExists) { + throw Exception(errors::Configuration) << "For the module with the label \"" << label << "\",\n" + << "both the parameters \"initialSeed\" and \"initialSeedSet\"\n" + << "have been set in the configuration. You must set one or\n" + << "the other. It is illegal to set both.\n"; + } else if (!initialSeedExists && !initialSeedSetExists) { + throw Exception(errors::Configuration) << "For the module with the label \"" << label << "\",\n" + << "neither the parameter \"initialSeed\" nor \"initialSeedSet\"\n" + << "has been set in the configuration. You must set one or\n" + << "the other.\n"; + } else if (initialSeedExists) { initialSeed = modulePSet.getUntrackedParameter("initialSeed"); initialSeedSet.clear(); initialSeedSet.push_back(initialSeed); - } else if(initialSeedSetExists) { + } else if (initialSeedSetExists) { initialSeedSet = modulePSet.getUntrackedParameter("initialSeedSet"); } seedsAndNameMap_.insert(std::pair(label, SeedsAndName(initialSeedSet, engineName))); // For the CLHEP::RanecuEngine case, require a seed set containing exactly two seeds. - if(engineName == std::string("RanecuEngine")) { - if(initialSeedSet.size() != 2U) { + if (engineName == std::string("RanecuEngine")) { + if (initialSeedSet.size() != 2U) { throw Exception(errors::Configuration) - << "Random engines of type \"RanecuEngine\" require 2 seeds\n" - << "be specified with the parameter named \"initialSeedSet\".\n" - << "Either \"initialSeedSet\" was not in the configuration\n" - << "or its size was not 2 for the module with label \"" << label << "\".\n" ; + << "Random engines of type \"RanecuEngine\" require 2 seeds\n" + << "be specified with the parameter named \"initialSeedSet\".\n" + << "Either \"initialSeedSet\" was not in the configuration\n" + << "or its size was not 2 for the module with label \"" << label << "\".\n"; } - if(initialSeedSet[0] > maxSeedRanecu || - initialSeedSet[1] > maxSeedRanecu) { // They need to fit in a 31 bit integer + if (initialSeedSet[0] > maxSeedRanecu || + initialSeedSet[1] > maxSeedRanecu) { // They need to fit in a 31 bit integer throw Exception(errors::Configuration) - << "The RanecuEngine seeds should be in the range 0 to " << maxSeedRanecu << ".\n" - << "The seeds passed to the RandomNumberGenerationService from the\n" - "configuration file were " << initialSeedSet[0] << " and " << initialSeedSet[1] - << "\nThis was for the module with label \"" << label << "\".\n"; + << "The RanecuEngine seeds should be in the range 0 to " << maxSeedRanecu << ".\n" + << "The seeds passed to the RandomNumberGenerationService from the\n" + "configuration file were " + << initialSeedSet[0] << " and " << initialSeedSet[1] << "\nThis was for the module with label \"" + << label << "\".\n"; } } // For the other engines, one seed is required else { - if(initialSeedSet.size() != 1U) { + if (initialSeedSet.size() != 1U) { throw Exception(errors::Configuration) - << "Random engines of type \"HepJamesRandom\", \"TRandom3\" and \"MixMaxRng\" \n" - << "require exactly 1 seed be specified in the configuration.\n" - << "There were " << initialSeedSet.size() << " seeds set for the\n" - << "module with label \"" << label << "\".\n" ; + << "Random engines of type \"HepJamesRandom\", \"TRandom3\" and \"MixMaxRng\" \n" + << "require exactly 1 seed be specified in the configuration.\n" + << "There were " << initialSeedSet.size() << " seeds set for the\n" + << "module with label \"" << label << "\".\n"; } - if(engineName == "HepJamesRandom") { - if(initialSeedSet[0] > maxSeedHepJames) { + if (engineName == "HepJamesRandom") { + if (initialSeedSet[0] > maxSeedHepJames) { throw Exception(errors::Configuration) - << "The CLHEP::HepJamesRandom engine seed should be in the range 0 to " << maxSeedHepJames <<".\n" - << "The seed passed to the RandomNumberGenerationService from the\n" - "configuration file was " << initialSeedSet[0] << ". This was for \n" - << "the module with label " << label << ".\n"; + << "The CLHEP::HepJamesRandom engine seed should be in the range 0 to " << maxSeedHepJames << ".\n" + << "The seed passed to the RandomNumberGenerationService from the\n" + "configuration file was " + << initialSeedSet[0] << ". This was for \n" + << "the module with label " << label << ".\n"; } - } else if(engineName == "MixMaxRng") { - if(initialSeedSet[0] > maxSeedTRandom3) { + } else if (engineName == "MixMaxRng") { + if (initialSeedSet[0] > maxSeedTRandom3) { throw Exception(errors::Configuration) - << "The CLHEP::MixMaxRng engine seed should be in the range 0 to " << maxSeedTRandom3 << ".\n" - << "The seed passed to the RandomNumberGenerationService from the\n" - "configuration file was " << initialSeedSet[0] << ". This was for \n" - << "the module with label " << label << ".\n"; + << "The CLHEP::MixMaxRng engine seed should be in the range 0 to " << maxSeedTRandom3 << ".\n" + << "The seed passed to the RandomNumberGenerationService from the\n" + "configuration file was " + << initialSeedSet[0] << ". This was for \n" + << "the module with label " << label << ".\n"; } - } else if(engineName == "TRandom3") { - if(initialSeedSet[0] > maxSeedTRandom3) { + } else if (engineName == "TRandom3") { + if (initialSeedSet[0] > maxSeedTRandom3) { throw Exception(errors::Configuration) - << "The CLHEP::MixMaxRng engine seed should be in the range 0 to " << maxSeedTRandom3 << ".\n" - << "The seed passed to the RandomNumberGenerationService from the\n" - "configuration file was " << initialSeedSet[0] << ". This was for \n" - << "the module with label " << label << ".\n"; + << "The CLHEP::MixMaxRng engine seed should be in the range 0 to " << maxSeedTRandom3 << ".\n" + << "The seed passed to the RandomNumberGenerationService from the\n" + "configuration file was " + << initialSeedSet[0] << ". This was for \n" + << "the module with label " << label << ".\n"; } } else { throw Exception(errors::Configuration) - << "The random engine name, \"" << engineName - << "\", does not correspond to a supported engine.\n" - << "This engine was configured for the module with label \"" << label << "\""; + << "The random engine name, \"" << engineName << "\", does not correspond to a supported engine.\n" + << "This engine was configured for the module with label \"" << label << "\""; } } } @@ -192,8 +190,7 @@ namespace edm { activityRegistry.watchPreallocate(this, &RandomNumberGeneratorService::preallocate); - if(enableChecking_) { - + if (enableChecking_) { activityRegistry.watchPreModuleBeginStream(this, &RandomNumberGeneratorService::preModuleBeginStream); activityRegistry.watchPostModuleBeginStream(this, &RandomNumberGeneratorService::postModuleBeginStream); @@ -212,88 +209,79 @@ namespace edm { activityRegistry.watchPreModuleStreamEndLumi(this, &RandomNumberGeneratorService::preModuleStreamEndLumi); activityRegistry.watchPostModuleStreamEndLumi(this, &RandomNumberGeneratorService::postModuleStreamEndLumi); } - } - RandomNumberGeneratorService::~RandomNumberGeneratorService() { - } + RandomNumberGeneratorService::~RandomNumberGeneratorService() {} - void - RandomNumberGeneratorService::consumes(ConsumesCollector&& iC) const { - iC.consumes(restoreStateBeginLumiTag_); - iC.consumes(restoreStateTag_); + void RandomNumberGeneratorService::consumes(ConsumesCollector&& iC) const { + iC.consumes(restoreStateBeginLumiTag_); + iC.consumes(restoreStateTag_); } - CLHEP::HepRandomEngine& - RandomNumberGeneratorService::getEngine(StreamID const& streamID) { - + CLHEP::HepRandomEngine& RandomNumberGeneratorService::getEngine(StreamID const& streamID) { ModuleCallingContext const* mcc = CurrentModuleOnThread::getCurrentModuleOnThread(); - if(mcc == nullptr) { + if (mcc == nullptr) { throw Exception(errors::LogicError) - << "RandomNumberGeneratorService::getEngine\n" - "Requested a random number engine from the RandomNumberGeneratorService\n" - "when no module was active. ModuleCallingContext is null\n"; + << "RandomNumberGeneratorService::getEngine\n" + "Requested a random number engine from the RandomNumberGeneratorService\n" + "when no module was active. ModuleCallingContext is null\n"; } unsigned int moduleID = mcc->moduleDescription()->id(); std::vector& moduleIDVector = streamModuleIDToEngine_.at(streamID.value()); ModuleIDToEngine target(nullptr, moduleID); - std::vector::iterator iter = std::lower_bound(moduleIDVector.begin(), - moduleIDVector.end(), - target); - if(iter == moduleIDVector.end() || iter->moduleID() != moduleID) { + std::vector::iterator iter = + std::lower_bound(moduleIDVector.begin(), moduleIDVector.end(), target); + if (iter == moduleIDVector.end() || iter->moduleID() != moduleID) { throw Exception(errors::Configuration) - << "The module with label \"" - << mcc->moduleDescription()->moduleLabel() - << "\" requested a random number engine from the \n" - "RandomNumberGeneratorService, but that module was not configured\n" - "for random numbers. An engine is created only if a seed(s) is provided\n" - "in the configuration file. Please add the following PSet to the\n" - "configuration file for the RandomNumberGeneratorService:\n\n" - " " << mcc->moduleDescription()->moduleLabel() << " = cms.PSet(\n" - " initialSeed = cms.untracked.uint32(your_seed),\n" - " engineName = cms.untracked.string('TRandom3')\n" - " )\n" - "where you replace \"your_seed\" with a number and add a comma if necessary\n" - "The \"engineName\" parameter is optional. If absent the default is \"HepJamesRandom\".\n"; - + << "The module with label \"" << mcc->moduleDescription()->moduleLabel() + << "\" requested a random number engine from the \n" + "RandomNumberGeneratorService, but that module was not configured\n" + "for random numbers. An engine is created only if a seed(s) is provided\n" + "in the configuration file. Please add the following PSet to the\n" + "configuration file for the RandomNumberGeneratorService:\n\n" + " " + << mcc->moduleDescription()->moduleLabel() + << " = cms.PSet(\n" + " initialSeed = cms.untracked.uint32(your_seed),\n" + " engineName = cms.untracked.string('TRandom3')\n" + " )\n" + "where you replace \"your_seed\" with a number and add a comma if necessary\n" + "The \"engineName\" parameter is optional. If absent the default is \"HepJamesRandom\".\n"; } return *iter->labelAndEngine()->engine(); } - CLHEP::HepRandomEngine& - RandomNumberGeneratorService::getEngine(LuminosityBlockIndex const& lumiIndex) { - + CLHEP::HepRandomEngine& RandomNumberGeneratorService::getEngine(LuminosityBlockIndex const& lumiIndex) { ModuleCallingContext const* mcc = CurrentModuleOnThread::getCurrentModuleOnThread(); - if(mcc == nullptr) { + if (mcc == nullptr) { throw Exception(errors::LogicError) - << "RandomNumberGeneratorService::getEngine\n" - "Requested a random number engine from the RandomNumberGeneratorService\n" - "when no module was active. ModuleCallingContext is null\n"; + << "RandomNumberGeneratorService::getEngine\n" + "Requested a random number engine from the RandomNumberGeneratorService\n" + "when no module was active. ModuleCallingContext is null\n"; } unsigned int moduleID = mcc->moduleDescription()->id(); std::vector& moduleIDVector = lumiModuleIDToEngine_.at(lumiIndex.value()); ModuleIDToEngine target(nullptr, moduleID); - std::vector::iterator iter = std::lower_bound(moduleIDVector.begin(), - moduleIDVector.end(), - target); - if(iter == moduleIDVector.end() || iter->moduleID() != moduleID) { + std::vector::iterator iter = + std::lower_bound(moduleIDVector.begin(), moduleIDVector.end(), target); + if (iter == moduleIDVector.end() || iter->moduleID() != moduleID) { throw Exception(errors::Configuration) - << "The module with label \"" - << mcc->moduleDescription()->moduleLabel() - << "\" requested a random number engine from the \n" - "RandomNumberGeneratorService, but that module was not configured\n" - "for random numbers. An engine is created only if a seed(s) is provided\n" - "in the configuration file. Please add the following PSet to the\n" - "configuration file for the RandomNumberGeneratorService:\n\n" - " " << mcc->moduleDescription()->moduleLabel() << " = cms.PSet(\n" - " initialSeed = cms.untracked.uint32(your_seed),\n" - " engineName = cms.untracked.string('TRandom3')\n" - " )\n" - "where you replace \"your_seed\" with a number and add a comma if necessary\n" - "The \"engineName\" parameter is optional. If absent the default is \"HepJamesRandom\".\n"; - + << "The module with label \"" << mcc->moduleDescription()->moduleLabel() + << "\" requested a random number engine from the \n" + "RandomNumberGeneratorService, but that module was not configured\n" + "for random numbers. An engine is created only if a seed(s) is provided\n" + "in the configuration file. Please add the following PSet to the\n" + "configuration file for the RandomNumberGeneratorService:\n\n" + " " + << mcc->moduleDescription()->moduleLabel() + << " = cms.PSet(\n" + " initialSeed = cms.untracked.uint32(your_seed),\n" + " engineName = cms.untracked.string('TRandom3')\n" + " )\n" + "where you replace \"your_seed\" with a number and add a comma if necessary\n" + "The \"engineName\" parameter is optional. If absent the default is \"HepJamesRandom\".\n"; } return *iter->labelAndEngine()->engine(); } @@ -304,48 +292,47 @@ namespace edm { // to create your own engines using it. It is difficult to get the offsets // for streams/forking/offset parameters correct and almost certainly would break // replay. - std::uint32_t - RandomNumberGeneratorService::mySeed() const { + std::uint32_t RandomNumberGeneratorService::mySeed() const { std::string label; ModuleCallingContext const* mcc = CurrentModuleOnThread::getCurrentModuleOnThread(); - if(mcc == nullptr) { + if (mcc == nullptr) { throw Exception(errors::LogicError) - << "RandomNumberGeneratorService::getEngine()\n" - "Requested a random number engine from the RandomNumberGeneratorService\n" - "from an unallowed transition. ModuleCallingContext is null\n"; + << "RandomNumberGeneratorService::getEngine()\n" + "Requested a random number engine from the RandomNumberGeneratorService\n" + "from an unallowed transition. ModuleCallingContext is null\n"; } else { label = mcc->moduleDescription()->moduleLabel(); } std::map::const_iterator iter = seedsAndNameMap_.find(label); - if(iter == seedsAndNameMap_.end()) { + if (iter == seedsAndNameMap_.end()) { throw Exception(errors::Configuration) - << "The module with label \"" - << label - << "\" requested a random number seed from the \n" - "RandomNumberGeneratorService, but that module was not configured\n" - "for random numbers. An engine is created only if a seed(s) is provided\n" - "in the configuration file. Please add the following PSet to the\n" - "configuration file for the RandomNumberGeneratorService:\n\n" - " " << label << " = cms.PSet(\n" - " initialSeed = cms.untracked.uint32(your_seed),\n" - " engineName = cms.untracked.string('TRandom3')\n" - " )\n" - "where you replace \"your_seed\" with a number and add a comma if necessary\n" - "The \"engineName\" parameter is optional. If absent the default is \"HepJamesRandom\".\n"; + << "The module with label \"" << label + << "\" requested a random number seed from the \n" + "RandomNumberGeneratorService, but that module was not configured\n" + "for random numbers. An engine is created only if a seed(s) is provided\n" + "in the configuration file. Please add the following PSet to the\n" + "configuration file for the RandomNumberGeneratorService:\n\n" + " " + << label + << " = cms.PSet(\n" + " initialSeed = cms.untracked.uint32(your_seed),\n" + " engineName = cms.untracked.string('TRandom3')\n" + " )\n" + "where you replace \"your_seed\" with a number and add a comma if necessary\n" + "The \"engineName\" parameter is optional. If absent the default is \"HepJamesRandom\".\n"; } return iter->second.seeds()[0]; } - void - RandomNumberGeneratorService::fillDescriptions(ConfigurationDescriptions& descriptions) { + void RandomNumberGeneratorService::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; std::string emptyString; edm::InputTag emptyInputTag("", "", ""); - desc.addNode( edm::ParameterDescription("restoreStateTag", emptyInputTag, false) xor - edm::ParameterDescription("restoreStateLabel", emptyString, false) ); + desc.addNode(edm::ParameterDescription("restoreStateTag", emptyInputTag, false) xor + edm::ParameterDescription("restoreStateLabel", emptyString, false)); desc.addUntracked("saveFileName", emptyString); desc.addUntracked("restoreFileName", emptyString); @@ -365,28 +352,25 @@ namespace edm { descriptions.add("RandomNumberGeneratorService", desc); } - void - RandomNumberGeneratorService::preModuleConstruction(ModuleDescription const& description) { + void RandomNumberGeneratorService::preModuleConstruction(ModuleDescription const& description) { std::map::iterator iter = seedsAndNameMap_.find(description.moduleLabel()); - if(iter != seedsAndNameMap_.end()) { + if (iter != seedsAndNameMap_.end()) { iter->second.setModuleID(description.id()); } } - void - RandomNumberGeneratorService::preallocate(SystemBounds const& sb) { - + void RandomNumberGeneratorService::preallocate(SystemBounds const& sb) { nStreams_ = sb.maxNumberOfStreams(); assert(nStreams_ >= 1); - if(!restoreFileName_.empty() && nStreams_ != 1) { + if (!restoreFileName_.empty() && nStreams_ != 1) { throw Exception(errors::Configuration) - << "Configuration is illegal. The RandomNumberGeneratorService is configured\n" - << "to run replay using a text file to input the random engine states and\n" - << "the number of streams is greater than 1. Either set the\n" - << "parameter named \"restoreFileName\" in the RandomNumberGeneratorService\n" - << "to the empty string or set the parameter \"numberOfStreams\" in the top\n" - << "level options parameter set to 1. (Probably these are the default values\n" - << "and just not setting the parameters will also work)\n"; + << "Configuration is illegal. The RandomNumberGeneratorService is configured\n" + << "to run replay using a text file to input the random engine states and\n" + << "the number of streams is greater than 1. Either set the\n" + << "parameter named \"restoreFileName\" in the RandomNumberGeneratorService\n" + << "to the empty string or set the parameter \"numberOfStreams\" in the top\n" + << "level options parameter set to 1. (Probably these are the default values\n" + << "and just not setting the parameters will also work)\n"; } unsigned int nConcurrentLumis = sb.maxNumberOfConcurrentLuminosityBlocks(); @@ -398,37 +382,35 @@ namespace edm { lumiCache_.resize(nConcurrentLumis); outFiles_.resize(nStreams_); - for(unsigned int iStream = 0; iStream < nStreams_; ++iStream) { + for (unsigned int iStream = 0; iStream < nStreams_; ++iStream) { unsigned int seedOffset = iStream; createEnginesInVector(streamEngines_[iStream], seedOffset, eventSeedOffset_, streamModuleIDToEngine_[iStream]); - if(!saveFileName_.empty()) { - outFiles_[iStream] = std::make_shared(); // propagate_const has no reset() function + if (!saveFileName_.empty()) { + outFiles_[iStream] = std::make_shared(); // propagate_const has no reset() function } } - for(unsigned int iLumi = 0; iLumi < nConcurrentLumis; ++iLumi) { + for (unsigned int iLumi = 0; iLumi < nConcurrentLumis; ++iLumi) { unsigned int seedOffset = nStreams_; createEnginesInVector(lumiEngines_[iLumi], seedOffset, 0, lumiModuleIDToEngine_[iLumi]); snapShot(lumiEngines_[iLumi], lumiCache_[iLumi]); - if(!restoreFileName_.empty()) { + if (!restoreFileName_.empty()) { readLumiStatesFromTextFile(restoreFileName_, lumiCache_[iLumi]); } } - if(!restoreFileName_.empty()) { + if (!restoreFileName_.empty()) { // There is guaranteed to be one stream in this case snapShot(streamEngines_[0], eventCache_[0]); readEventStatesFromTextFile(restoreFileName_, eventCache_[0]); restoreFromCache(eventCache_[0], streamEngines_[0]); } - if(verbose_) { + if (verbose_) { print(std::cout); } } - void - RandomNumberGeneratorService::preBeginLumi(LuminosityBlock const& lumi) { - - if(!restoreStateTag_.label().empty()) { + void RandomNumberGeneratorService::preBeginLumi(LuminosityBlock const& lumi) { + if (!restoreStateTag_.label().empty()) { // Copy from a product in the LuminosityBlock to cache for a particular luminosityBlockIndex readFromLuminosityBlock(lumi); } @@ -436,11 +418,8 @@ namespace edm { restoreFromCache(lumiCache_[lumi.index()], lumiEngines_[lumi.index()]); } - void - RandomNumberGeneratorService::postEventRead(Event const& event) { - - if(!restoreStateTag_.label().empty()) { - + void RandomNumberGeneratorService::postEventRead(Event const& event) { + if (!restoreStateTag_.label().empty()) { // This initializes the cache before readFromEvent snapShot(streamEngines_[event.streamID()], eventCache_[event.streamID()]); @@ -456,10 +435,10 @@ namespace edm { } // if requested write text file from both caches - if(!saveFileName_.empty()) { + if (!saveFileName_.empty()) { saveStatesToFile(saveFileName_, event.streamID(), event.getLuminosityBlock().index()); bool expected = false; - if(saveFileNameRecorded_.compare_exchange_strong(expected, true)) { + if (saveFileNameRecorded_.compare_exchange_strong(expected, true)) { std::string fullName = constructSaveFileName(); Service reportSvc; reportSvc->reportRandomStateFile(fullName); @@ -467,87 +446,77 @@ namespace edm { } } - void - RandomNumberGeneratorService::preModuleBeginStream(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::preModuleBeginStream(StreamContext const& sc, ModuleCallingContext const& mcc) { preModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::postModuleBeginStream(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::postModuleBeginStream(StreamContext const& sc, ModuleCallingContext const& mcc) { postModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::preModuleEndStream(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::preModuleEndStream(StreamContext const& sc, ModuleCallingContext const& mcc) { preModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::postModuleEndStream(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::postModuleEndStream(StreamContext const& sc, ModuleCallingContext const& mcc) { postModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::preModuleStreamBeginRun(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::preModuleStreamBeginRun(StreamContext const& sc, + ModuleCallingContext const& mcc) { preModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::postModuleStreamBeginRun(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::postModuleStreamBeginRun(StreamContext const& sc, + ModuleCallingContext const& mcc) { postModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::preModuleStreamEndRun(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::preModuleStreamEndRun(StreamContext const& sc, ModuleCallingContext const& mcc) { preModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::postModuleStreamEndRun(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::postModuleStreamEndRun(StreamContext const& sc, + ModuleCallingContext const& mcc) { postModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::preModuleStreamBeginLumi(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::preModuleStreamBeginLumi(StreamContext const& sc, + ModuleCallingContext const& mcc) { preModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::postModuleStreamBeginLumi(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::postModuleStreamBeginLumi(StreamContext const& sc, + ModuleCallingContext const& mcc) { postModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::preModuleStreamEndLumi(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::preModuleStreamEndLumi(StreamContext const& sc, + ModuleCallingContext const& mcc) { preModuleStreamCheck(sc, mcc); } - void - RandomNumberGeneratorService::postModuleStreamEndLumi(StreamContext const& sc, ModuleCallingContext const& mcc) { + void RandomNumberGeneratorService::postModuleStreamEndLumi(StreamContext const& sc, + ModuleCallingContext const& mcc) { postModuleStreamCheck(sc, mcc); } - std::vector const& - RandomNumberGeneratorService::getLumiCache(LuminosityBlockIndex const& lumiIndex) const { + std::vector const& RandomNumberGeneratorService::getLumiCache( + LuminosityBlockIndex const& lumiIndex) const { return lumiCache_.at(lumiIndex.value()); } - std::vector const& - RandomNumberGeneratorService::getEventCache(StreamID const& streamID) const { + std::vector const& RandomNumberGeneratorService::getEventCache(StreamID const& streamID) const { return eventCache_.at(streamID.value()); } - void - RandomNumberGeneratorService::print(std::ostream& os) const { - + void RandomNumberGeneratorService::print(std::ostream& os) const { os << "\n\nRandomNumberGeneratorService dump\n\n"; os << " Contents of seedsAndNameMap (label moduleID engineType seeds)\n"; - for(auto const& entry : seedsAndNameMap_) { - os << " " << entry.first - << " " << entry.second.moduleID() - << " " << entry.second.engineName(); - for(auto val : entry.second.seeds()) { + for (auto const& entry : seedsAndNameMap_) { + os << " " << entry.first << " " << entry.second.moduleID() << " " << entry.second.engineName(); + for (auto val : entry.second.seeds()) { os << " " << val; } os << "\n"; @@ -564,17 +533,17 @@ namespace edm { os << "\n streamEngines_\n"; unsigned int iStream = 0; - for(auto const& k : streamEngines_) { + for (auto const& k : streamEngines_) { os << " Stream " << iStream << "\n"; - for(auto const& i : k) { + for (auto const& i : k) { os << " " << i.label(); - for(auto const& j : i.seeds()) { + for (auto const& j : i.seeds()) { os << " " << j; } os << " " << i.engine()->name(); - if(i.engine()->name() == std::string("HepJamesRandom")) { + if (i.engine()->name() == std::string("HepJamesRandom")) { os << " " << i.engine()->getSeed(); - } else if(i.engine()->name() == std::string("MixMaxRng")) { + } else if (i.engine()->name() == std::string("MixMaxRng")) { os << " " << i.engine()->getSeed(); } else { os << " engine does not know seeds"; @@ -585,17 +554,17 @@ namespace edm { } os << "\n lumiEngines_\n"; unsigned int iLumi = 0; - for(auto const& k : lumiEngines_) { + for (auto const& k : lumiEngines_) { os << " lumiIndex " << iLumi << "\n"; - for(auto const& i : k) { + for (auto const& i : k) { os << " " << i.label(); - for(auto const& j : i.seeds()) { + for (auto const& j : i.seeds()) { os << " " << j; } os << " " << i.engine()->name(); - if(i.engine()->name() == std::string("HepJamesRandom")) { + if (i.engine()->name() == std::string("HepJamesRandom")) { os << " " << i.engine()->getSeed(); - } else if(i.engine()->name() == std::string("MixMaxRng")) { + } else if (i.engine()->name() == std::string("MixMaxRng")) { os << " " << i.engine()->getSeed(); } else { os << " engine does not know seeds"; @@ -606,106 +575,97 @@ namespace edm { } } - void - RandomNumberGeneratorService::preModuleStreamCheck(StreamContext const& sc, ModuleCallingContext const& mcc) { - if(enableChecking_) { + void RandomNumberGeneratorService::preModuleStreamCheck(StreamContext const& sc, ModuleCallingContext const& mcc) { + if (enableChecking_) { unsigned int moduleID = mcc.moduleDescription()->id(); std::vector& moduleIDVector = streamModuleIDToEngine_.at(sc.streamID().value()); ModuleIDToEngine target(nullptr, moduleID); - std::vector::iterator iter = std::lower_bound(moduleIDVector.begin(), - moduleIDVector.end(), - target); - if(iter != moduleIDVector.end() && iter->moduleID() == moduleID) { + std::vector::iterator iter = + std::lower_bound(moduleIDVector.begin(), moduleIDVector.end(), target); + if (iter != moduleIDVector.end() && iter->moduleID() == moduleID) { LabelAndEngine* labelAndEngine = iter->labelAndEngine(); iter->setEngineState(labelAndEngine->engine()->put()); } } } - void - RandomNumberGeneratorService::postModuleStreamCheck(StreamContext const& sc, ModuleCallingContext const& mcc) { - if(enableChecking_) { + void RandomNumberGeneratorService::postModuleStreamCheck(StreamContext const& sc, ModuleCallingContext const& mcc) { + if (enableChecking_) { unsigned int moduleID = mcc.moduleDescription()->id(); std::vector& moduleIDVector = streamModuleIDToEngine_.at(sc.streamID().value()); ModuleIDToEngine target(nullptr, moduleID); - std::vector::iterator iter = std::lower_bound(moduleIDVector.begin(), - moduleIDVector.end(), - target); - if(iter != moduleIDVector.end() && iter->moduleID() == moduleID) { + std::vector::iterator iter = + std::lower_bound(moduleIDVector.begin(), moduleIDVector.end(), target); + if (iter != moduleIDVector.end() && iter->moduleID() == moduleID) { LabelAndEngine* labelAndEngine = iter->labelAndEngine(); - if(iter->engineState() != labelAndEngine->engine()->put()) { + if (iter->engineState() != labelAndEngine->engine()->put()) { throw Exception(errors::LogicError) - << "It is illegal to generate random numbers during beginStream, endStream,\n" - "beginRun, endRun, beginLumi, endLumi because that makes it very difficult\n" - "to replay the processing of individual events. Random numbers were\n" - "generated during one of these methods for the module with class name\n\"" - << mcc.moduleDescription()->moduleName() << "\" " - "and module label \"" << mcc.moduleDescription()->moduleLabel() << "\"\n"; + << "It is illegal to generate random numbers during beginStream, endStream,\n" + "beginRun, endRun, beginLumi, endLumi because that makes it very difficult\n" + "to replay the processing of individual events. Random numbers were\n" + "generated during one of these methods for the module with class name\n\"" + << mcc.moduleDescription()->moduleName() + << "\" " + "and module label \"" + << mcc.moduleDescription()->moduleLabel() << "\"\n"; } } } } - void - RandomNumberGeneratorService::readFromLuminosityBlock(LuminosityBlock const& lumi) { - + void RandomNumberGeneratorService::readFromLuminosityBlock(LuminosityBlock const& lumi) { Service tns; - if(tns.isAvailable()) { - if(tns->getProcessName() == restoreStateTag_.process()) { + if (tns.isAvailable()) { + if (tns->getProcessName() == restoreStateTag_.process()) { throw Exception(errors::Configuration) - << "In the configuration for the RandomNumberGeneratorService the\n" - << "restoreStateTag contains the current process which is illegal.\n" - << "The process name in the replay process should have been changed\n" - << "to be different than the original process name and the restoreStateTag\n" - << "should contain either the original process name or an empty process name.\n"; + << "In the configuration for the RandomNumberGeneratorService the\n" + << "restoreStateTag contains the current process which is illegal.\n" + << "The process name in the replay process should have been changed\n" + << "to be different than the original process name and the restoreStateTag\n" + << "should contain either the original process name or an empty process name.\n"; } } Handle states; lumi.getByLabel(restoreStateBeginLumiTag_, states); - if(!states.isValid()) { + if (!states.isValid()) { throw Exception(errors::ProductNotFound) - << "The RandomNumberGeneratorService is trying to restore\n" - << "the state of the random engines by reading a product from\n" - << "the LuminosityBlock with input tag \"" << restoreStateBeginLumiTag_ << "\".\n" - << "It could not find the product.\n" - << "Either the product in the LuminosityBlock was dropped or\n" - << "not produced or the configured input tag is incorrect or there is a bug somewhere\n"; + << "The RandomNumberGeneratorService is trying to restore\n" + << "the state of the random engines by reading a product from\n" + << "the LuminosityBlock with input tag \"" << restoreStateBeginLumiTag_ << "\".\n" + << "It could not find the product.\n" + << "Either the product in the LuminosityBlock was dropped or\n" + << "not produced or the configured input tag is incorrect or there is a bug somewhere\n"; return; } states->getRandomEngineStates(lumiCache_.at(lumi.index())); } - void - RandomNumberGeneratorService::readFromEvent(Event const& event) { - + void RandomNumberGeneratorService::readFromEvent(Event const& event) { Handle states; event.getByLabel(restoreStateTag_, states); - if(!states.isValid()) { + if (!states.isValid()) { throw Exception(errors::ProductNotFound) - << "The RandomNumberGeneratorService is trying to restore\n" - << "the state of the random engines by reading a product from\n" - << "the Event with input tag \"" << restoreStateTag_ << "\".\n" - << "It could not find the product.\n" - << "Either the product in the Event was dropped or\n" - << "not produced or the configured input tag is incorrect or there is a bug somewhere\n"; + << "The RandomNumberGeneratorService is trying to restore\n" + << "the state of the random engines by reading a product from\n" + << "the Event with input tag \"" << restoreStateTag_ << "\".\n" + << "It could not find the product.\n" + << "Either the product in the Event was dropped or\n" + << "not produced or the configured input tag is incorrect or there is a bug somewhere\n"; return; } states->getRandomEngineStates(eventCache_.at(event.streamID())); } - void - RandomNumberGeneratorService::snapShot(std::vector const& engines, std::vector& cache) { + void RandomNumberGeneratorService::snapShot(std::vector const& engines, + std::vector& cache) { cache.resize(engines.size()); std::vector::iterator state = cache.begin(); - for(std::vector::const_iterator iter = engines.begin(); - iter != engines.end(); - ++iter, ++state) { - + for (std::vector::const_iterator iter = engines.begin(); iter != engines.end(); ++iter, ++state) { std::string const& label = iter->label(); state->setLabel(label); state->setSeed(iter->seeds()); @@ -713,31 +673,29 @@ namespace edm { std::vector stateL = iter->engine()->put(); state->clearStateVector(); state->reserveStateVector(stateL.size()); - for(auto element : stateL) { + for (auto element : stateL) { state->push_back_stateVector(static_cast(element)); } } } - void - RandomNumberGeneratorService::restoreFromCache(std::vector const& cache, - std::vector& engines) { + void RandomNumberGeneratorService::restoreFromCache(std::vector const& cache, + std::vector& engines) { std::vector::iterator labelAndEngine = engines.begin(); - for(auto const& cachedState : cache) { - + for (auto const& cachedState : cache) { std::string const& engineLabel = cachedState.getLabel(); std::vector const& engineState = cachedState.getState(); std::vector engineStateL; engineStateL.reserve(engineState.size()); - for(auto const& value : engineState) { + for (auto const& value : engineState) { engineStateL.push_back(static_cast(value)); } std::vector const& engineSeeds = cachedState.getSeed(); std::vector engineSeedsL; engineSeedsL.reserve(engineSeeds.size()); - for(auto const& val : engineSeeds) { + for (auto const& val : engineSeeds) { long seedL = static_cast(val); engineSeedsL.push_back(seedL); @@ -757,8 +715,7 @@ namespace edm { // We need to handle each type of engine differently because each // has different requirements on the seed or seeds. - if(engineStateL[0] == CLHEP::engineIDulong()) { - + if (engineStateL[0] == CLHEP::engineIDulong()) { checkEngineType(engine->name(), std::string("HepJamesRandom"), engineLabel); // These two lines actually restore the seed and engine state. @@ -766,8 +723,7 @@ namespace edm { engine->get(engineStateL); labelAndEngine->setSeed(engineSeeds[0], 0); - } else if(engineStateL[0] == CLHEP::engineIDulong()) { - + } else if (engineStateL[0] == CLHEP::engineIDulong()) { checkEngineType(engine->name(), std::string("RanecuEngine"), engineLabel); // This line actually restores the engine state. @@ -775,8 +731,7 @@ namespace edm { labelAndEngine->setSeed(engineSeeds[0], 0); labelAndEngine->setSeed(engineSeeds[1], 1); - } else if(engineStateL[0] == CLHEP::engineIDulong()) { - + } else if (engineStateL[0] == CLHEP::engineIDulong()) { checkEngineType(engine->name(), std::string("MixMaxRng"), engineLabel); // This line actually restores the engine state. @@ -784,8 +739,7 @@ namespace edm { engine->get(engineStateL); labelAndEngine->setSeed(engineSeeds[0], 0); - } else if(engineStateL[0] == CLHEP::engineIDulong()) { - + } else if (engineStateL[0] == CLHEP::engineIDulong()) { checkEngineType(engine->name(), std::string("TRandom3"), engineLabel); // This line actually restores the engine state. @@ -797,59 +751,54 @@ namespace edm { // This should not be possible because this code should be able to restore // any kind of engine whose state can be saved. throw Exception(errors::Unknown) - << "The RandomNumberGeneratorService is trying to restore the state\n" - "of the random engines. The state in the event indicates an engine\n" - "of an unknown type. This should not be possible unless you are\n" - "running with an old code release on a new file that was created\n" - "with a newer release which had new engine types added. In this case\n" - "the only solution is to use a newer release. In any other case, notify\n" - "the EDM developers because this should not be possible\n"; + << "The RandomNumberGeneratorService is trying to restore the state\n" + "of the random engines. The state in the event indicates an engine\n" + "of an unknown type. This should not be possible unless you are\n" + "running with an old code release on a new file that was created\n" + "with a newer release which had new engine types added. In this case\n" + "the only solution is to use a newer release. In any other case, notify\n" + "the EDM developers because this should not be possible\n"; } ++labelAndEngine; } } - void - RandomNumberGeneratorService::checkEngineType(std::string const& typeFromConfig, - std::string const& typeFromEvent, - std::string const& engineLabel) const { - if(typeFromConfig != typeFromEvent) { + void RandomNumberGeneratorService::checkEngineType(std::string const& typeFromConfig, + std::string const& typeFromEvent, + std::string const& engineLabel) const { + if (typeFromConfig != typeFromEvent) { throw Exception(errors::Configuration) - << "The RandomNumberGeneratorService is trying to restore\n" - << "the state of the random engine for the module \"" - << engineLabel << "\". An\n" - << "error was detected because the type of the engine in the\n" - << "input file and the configuration file do not match.\n" - << "In the configuration file the type is \"" << typeFromConfig - << "\".\nIn the input file the type is \"" << typeFromEvent << "\". If\n" - << "you are not generating any random numbers in this module, then\n" - << "remove the line in the configuration file that gives it\n" - << "a seed and the error will go away. Otherwise, you must give\n" - << "this module the same engine type in the configuration file or\n" - << "stop trying to restore the random engine state.\n"; + << "The RandomNumberGeneratorService is trying to restore\n" + << "the state of the random engine for the module \"" << engineLabel << "\". An\n" + << "error was detected because the type of the engine in the\n" + << "input file and the configuration file do not match.\n" + << "In the configuration file the type is \"" << typeFromConfig << "\".\nIn the input file the type is \"" + << typeFromEvent << "\". If\n" + << "you are not generating any random numbers in this module, then\n" + << "remove the line in the configuration file that gives it\n" + << "a seed and the error will go away. Otherwise, you must give\n" + << "this module the same engine type in the configuration file or\n" + << "stop trying to restore the random engine state.\n"; } } - void - RandomNumberGeneratorService::saveStatesToFile(std::string const& fileName, - StreamID const& streamID, - LuminosityBlockIndex const& lumiIndex) { - + void RandomNumberGeneratorService::saveStatesToFile(std::string const& fileName, + StreamID const& streamID, + LuminosityBlockIndex const& lumiIndex) { std::ofstream& outFile = *outFiles_.at(streamID); - if(!outFile.is_open()) { + if (!outFile.is_open()) { std::stringstream file; file << fileName; - if(nStreams_ > 1) { + if (nStreams_ > 1) { file << "_" << streamID.value(); } outFile.open(file.str().c_str(), std::ofstream::out | std::ofstream::trunc); - if(!outFile) { + if (!outFile) { throw Exception(errors::Configuration) - << "Unable to open the file \"" - << file.str() << "\" to save the state of the random engines.\n"; + << "Unable to open the file \"" << file.str() << "\" to save the state of the random engines.\n"; } } @@ -868,10 +817,8 @@ namespace edm { outFile.flush(); } - void - RandomNumberGeneratorService::writeStates(std::vector const& v, - std::ofstream& outFile) { - for(auto & state : v) { + void RandomNumberGeneratorService::writeStates(std::vector const& v, std::ofstream& outFile) { + for (auto& state : v) { std::vector const& seedVector = state.getSeed(); std::vector::size_type seedVectorLength = seedVector.size(); @@ -880,24 +827,24 @@ namespace edm { outFile << "\n" << state.getLabel() << "\n\n"; - outFile << "\n" << seedVectorLength << "\n\n" ; + outFile << "\n" << seedVectorLength << "\n\n"; outFile << "\n"; writeVector(seedVector, outFile); outFile << "\n"; outFile << "\n" << stateVectorLength << "\n\n"; outFile << "\n"; writeVector(stateVector, outFile); - outFile << "\n"; + outFile << "\n"; } } - void - RandomNumberGeneratorService::writeVector(VUint32 const& v, - std::ofstream& outFile) { - if(v.empty()) return; + void RandomNumberGeneratorService::writeVector(VUint32 const& v, std::ofstream& outFile) { + if (v.empty()) + return; size_t numItems = v.size(); - for(size_t i = 0; i < numItems; ++i) { - if(i != 0 && i % 10 == 0) outFile << "\n"; + for (size_t i = 0; i < numItems; ++i) { + if (i != 0 && i % 10 == 0) + outFile << "\n"; outFile << std::setw(13) << v[i]; } outFile << "\n"; @@ -910,44 +857,39 @@ namespace edm { return fullName; } - void - RandomNumberGeneratorService::readEventStatesFromTextFile(std::string const& fileName, - std::vector& cache) { + void RandomNumberGeneratorService::readEventStatesFromTextFile(std::string const& fileName, + std::vector& cache) { std::string whichStates(""); readStatesFromFile(fileName, cache, whichStates); } - void - RandomNumberGeneratorService::readLumiStatesFromTextFile(std::string const& fileName, - std::vector& cache) { + void RandomNumberGeneratorService::readLumiStatesFromTextFile(std::string const& fileName, + std::vector& cache) { std::string whichStates(""); readStatesFromFile(fileName, cache, whichStates); } - - void - RandomNumberGeneratorService::readStatesFromFile(std::string const& fileName, - std::vector& cache, - std::string const& whichStates) { + void RandomNumberGeneratorService::readStatesFromFile(std::string const& fileName, + std::vector& cache, + std::string const& whichStates) { std::ifstream inFile; inFile.open(fileName.c_str(), std::ifstream::in); - if(!inFile) { + if (!inFile) { throw Exception(errors::Configuration) - << "Unable to open the file \"" - << fileName << "\" to restore the random engine states.\n"; + << "Unable to open the file \"" << fileName << "\" to restore the random engine states.\n"; } std::string text; inFile >> text; - if(!inFile.good() || text != std::string("")) { + if (!inFile.good() || text != std::string("")) { throw Exception(errors::Configuration) - << "Attempting to read file with random number engine states.\n" - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read the file header word.\n"; + << "Attempting to read file with random number engine states.\n" + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read the file header word.\n"; } bool saveToCache = false; - while(readEngineState(inFile, cache, whichStates, saveToCache)) {} + while (readEngineState(inFile, cache, whichStates, saveToCache)) { + } } bool RandomNumberGeneratorService::readEngineState(std::istream& is, @@ -967,26 +909,24 @@ namespace edm { // and end of the data for different sections. is >> leading; - if(!is.good()) { + if (!is.good()) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read next field and did not hit the end yet.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read next field and did not hit the end yet.\n"; } // This marks the end of the file. We are done. - if(leading == std::string("")) return false; + if (leading == std::string("")) + return false; // This marks the end of a section of the data - if(leading == std::string("") || - leading == std::string("")) { + if (leading == std::string("") || leading == std::string("")) { saveToCache = false; return true; } // This marks the beginning of a section - if(leading == std::string("") || - leading == std::string("")) { + if (leading == std::string("") || leading == std::string("")) { saveToCache = (leading == whichStates); return true; } @@ -994,102 +934,81 @@ namespace edm { // Process the next engine state is >> moduleLabel >> trailing; - if(!is.good() || - leading != std::string("") || - trailing != std::string("")) { + if (!is.good() || leading != std::string("") || trailing != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read a module label when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read a module label when restoring random engine states.\n"; } is >> leading >> seedVectorSize >> trailing; - if(!is.good() || - leading != std::string("") || - trailing != std::string("")) { + if (!is.good() || leading != std::string("") || trailing != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read seed vector length when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read seed vector length when restoring random engine states.\n"; } is >> leading; - if(!is.good() || - leading != std::string("")) { + if (!is.good() || leading != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read beginning of InitialSeeds when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read beginning of InitialSeeds when restoring random engine states.\n"; } - if(seedVectorSize > maxSeeds) { + if (seedVectorSize > maxSeeds) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "The number of seeds exceeds 64K.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "The number of seeds exceeds 64K.\n"; } readVector(is, seedVectorSize, seedVector); is >> trailing; - if(!is.good() || - trailing != std::string("")) { + if (!is.good() || trailing != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read end of InitialSeeds when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read end of InitialSeeds when restoring random engine states.\n"; } is >> leading >> stateVectorSize >> trailing; - if(!is.good() || - leading != std::string("") || - trailing != std::string("")) { + if (!is.good() || leading != std::string("") || trailing != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read state vector length when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read state vector length when restoring random engine states.\n"; } is >> leading; - if(!is.good() || - leading != std::string("")) { + if (!is.good() || leading != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read beginning of FullState when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read beginning of FullState when restoring random engine states.\n"; } - if(stateVectorSize > maxStates) { + if (stateVectorSize > maxStates) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "The number of states exceeds 64K.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "The number of states exceeds 64K.\n"; } readVector(is, stateVectorSize, stateVector); is >> trailing; - if(!is.good() || - trailing != std::string("")) { + if (!is.good() || trailing != std::string("")) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read end of FullState when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read end of FullState when restoring random engine states.\n"; } - if(saveToCache) { + if (saveToCache) { RandomEngineState randomEngineState; randomEngineState.setLabel(moduleLabel); std::vector::iterator state = - std::lower_bound(cache.begin(), cache.end(), randomEngineState); + std::lower_bound(cache.begin(), cache.end(), randomEngineState); - if(state != cache.end() && moduleLabel == state->getLabel()) { - if(seedVector.size() != state->getSeed().size() || - stateVector.size() != state->getState().size()) { + if (state != cache.end() && moduleLabel == state->getLabel()) { + if (seedVector.size() != state->getSeed().size() || stateVector.size() != state->getState().size()) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Vectors containing engine state are the incorrect size for the type of random engine.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Vectors containing engine state are the incorrect size for the type of random engine.\n"; } state->setSeed(seedVector); state->setState(stateVector); @@ -1098,41 +1017,38 @@ namespace edm { return true; } - void - RandomNumberGeneratorService::readVector(std::istream& is, unsigned numItems, std::vector& v) { + void RandomNumberGeneratorService::readVector(std::istream& is, unsigned numItems, std::vector& v) { v.clear(); v.reserve(numItems); std::uint32_t data; - for(unsigned i = 0; i < numItems; ++i) { + for (unsigned i = 0; i < numItems; ++i) { is >> data; - if(!is.good()) { + if (!is.good()) { throw Exception(errors::Configuration) - << "File \"" << restoreFileName_ - << "\" is ill-structured or otherwise corrupted.\n" - << "Cannot read vector when restoring random engine states.\n"; + << "File \"" << restoreFileName_ << "\" is ill-structured or otherwise corrupted.\n" + << "Cannot read vector when restoring random engine states.\n"; } v.push_back(data); } } - void - RandomNumberGeneratorService::createEnginesInVector(std::vector& engines, - unsigned int seedOffset, - unsigned int eventSeedOffset, - std::vector& moduleIDVector) { + void RandomNumberGeneratorService::createEnginesInVector(std::vector& engines, + unsigned int seedOffset, + unsigned int eventSeedOffset, + std::vector& moduleIDVector) { // The vectors we will fill here will be the same size as // or smaller than seedsAndNameMap_. engines.reserve(seedsAndNameMap_.size()); moduleIDVector.reserve(seedsAndNameMap_.size()); - for(auto const& i : seedsAndNameMap_) { + for (auto const& i : seedsAndNameMap_) { unsigned int moduleID = i.second.moduleID(); - if(moduleID != std::numeric_limits::max()) { + if (moduleID != std::numeric_limits::max()) { std::string const& label = i.first; std::string const& name = i.second.engineName(); VUint32 const& seeds = i.second.seeds(); - if(name == "RanecuEngine") { + if (name == "RanecuEngine") { std::shared_ptr engine = std::make_shared(); engines.emplace_back(label, seeds, engine); resetEngineSeeds(engines.back(), name, seeds, seedOffset, eventSeedOffset); @@ -1141,19 +1057,19 @@ namespace edm { else { long int seedL = static_cast(seeds[0]); - if(name == "HepJamesRandom") { + if (name == "HepJamesRandom") { std::shared_ptr engine = std::make_shared(seedL); engines.emplace_back(label, seeds, engine); - if(seedOffset != 0 || eventSeedOffset != 0) { + if (seedOffset != 0 || eventSeedOffset != 0) { resetEngineSeeds(engines.back(), name, seeds, seedOffset, eventSeedOffset); } - } else if(name == "MixMaxRng") { + } else if (name == "MixMaxRng") { std::shared_ptr engine = std::make_shared(seedL); engines.emplace_back(label, seeds, engine); - if(seedOffset != 0 || eventSeedOffset != 0) { + if (seedOffset != 0 || eventSeedOffset != 0) { resetEngineSeeds(engines.back(), name, seeds, seedOffset, eventSeedOffset); } - } else { // TRandom3, currently the only other possibility + } else { // TRandom3, currently the only other possibility // There is a dangerous conversion from std::uint32_t to long // that occurs above. In the next 2 lines we check the @@ -1167,25 +1083,23 @@ namespace edm { std::shared_ptr engine = std::make_shared(seedL); engines.emplace_back(label, seeds, engine); - if(seedOffset != 0 || eventSeedOffset != 0) { + if (seedOffset != 0 || eventSeedOffset != 0) { resetEngineSeeds(engines.back(), name, seeds, seedOffset, eventSeedOffset); } } } moduleIDVector.emplace_back(&engines.back(), moduleID); - } // if moduleID valid - } // loop over seedsAndMap + } // if moduleID valid + } // loop over seedsAndMap std::sort(moduleIDVector.begin(), moduleIDVector.end()); } - void - RandomNumberGeneratorService::resetEngineSeeds(LabelAndEngine& labelAndEngine, - std::string const& engineName, - VUint32 const& seeds, - std::uint32_t offset1, - std::uint32_t offset2) { - - if(engineName == "RanecuEngine") { + void RandomNumberGeneratorService::resetEngineSeeds(LabelAndEngine& labelAndEngine, + std::string const& engineName, + VUint32 const& seeds, + std::uint32_t offset1, + std::uint32_t offset2) { + if (engineName == "RanecuEngine") { assert(seeds.size() == 2U); // Wrap around if the offsets push the seed over the maximum allowed value std::uint32_t mod = maxSeedRanecu + 1U; @@ -1198,11 +1112,11 @@ namespace edm { long int seedL[2]; seedL[0] = static_cast(seed0); seedL[1] = static_cast(seeds[1]); - labelAndEngine.engine()->setSeeds(seedL,0); + labelAndEngine.engine()->setSeeds(seedL, 0); } else { assert(seeds.size() == 1U); - if(engineName == "HepJamesRandom" || engineName == "MixMaxRng") { + if (engineName == "HepJamesRandom" || engineName == "MixMaxRng") { // Wrap around if the offsets push the seed over the maximum allowed value std::uint32_t mod = maxSeedHepJames + 1U; offset1 %= mod; @@ -1220,12 +1134,12 @@ namespace edm { // the values 32 bits can hold std::uint32_t max32 = maxSeedTRandom3; std::uint32_t seed0 = seeds[0]; - if((max32 - seed0) >= offset1) { + if ((max32 - seed0) >= offset1) { seed0 += offset1; } else { seed0 = offset1 - (max32 - seed0) - 1U; } - if((max32 - seed0) >= offset2) { + if ((max32 - seed0) >= offset2) { seed0 += offset2; } else { seed0 = offset2 - (max32 - seed0) - 1U; @@ -1248,5 +1162,5 @@ namespace edm { } } } - } -} + } // namespace service +} // namespace edm diff --git a/IOMC/RandomEngine/src/RandomNumberGeneratorService.h b/IOMC/RandomEngine/src/RandomNumberGeneratorService.h index 7b0e0b52f0f2f..150c5d6054510 100644 --- a/IOMC/RandomEngine/src/RandomNumberGeneratorService.h +++ b/IOMC/RandomEngine/src/RandomNumberGeneratorService.h @@ -50,9 +50,7 @@ namespace edm { class SystemBounds; class RandomNumberGeneratorService : public RandomNumberGenerator { - public: - RandomNumberGeneratorService(ParameterSet const& pset, ActivityRegistry& activityRegistry); ~RandomNumberGeneratorService() override; @@ -116,18 +114,20 @@ namespace edm { void print(std::ostream& os) const override; private: - typedef std::vector VUint32; class LabelAndEngine { public: - LabelAndEngine(std::string const& theLabel, VUint32 const& theSeeds, std::shared_ptr const& theEngine) : - label_(theLabel), seeds_(theSeeds), engine_(theEngine) { } + LabelAndEngine(std::string const& theLabel, + VUint32 const& theSeeds, + std::shared_ptr const& theEngine) + : label_(theLabel), seeds_(theSeeds), engine_(theEngine) {} std::string const& label() const { return label_; } VUint32 const& seeds() const { return seeds_; } std::shared_ptr engine() const { return get_underlying_safe(engine_); } std::shared_ptr& engine() { return get_underlying_safe(engine_); } void setSeed(std::uint32_t v, unsigned int index) { seeds_.at(index) = v; } + private: std::string label_; VUint32 seeds_; @@ -139,8 +139,8 @@ namespace edm { // one to one association between LabelAndEngine objects and ModuleIDToEngine objects. class ModuleIDToEngine { public: - ModuleIDToEngine(LabelAndEngine* theLabelAndEngine, unsigned int theModuleID) : - engineState_(), labelAndEngine_(theLabelAndEngine), moduleID_(theModuleID) { } + ModuleIDToEngine(LabelAndEngine* theLabelAndEngine, unsigned int theModuleID) + : engineState_(), labelAndEngine_(theLabelAndEngine), moduleID_(theModuleID) {} std::vector const& engineState() const { return engineState_; } LabelAndEngine const* labelAndEngine() const { return get_underlying_safe(labelAndEngine_); } @@ -149,8 +149,9 @@ namespace edm { void setEngineState(std::vector const& v) { engineState_ = v; } // Used to sort so binary lookup can be used on a container of these. bool operator<(ModuleIDToEngine const& r) const { return moduleID() < r.moduleID(); } + private: - std::vector engineState_; // Used only for check in stream transitions + std::vector engineState_; // Used only for check in stream transitions edm::propagate_const labelAndEngine_; unsigned int moduleID_; }; @@ -165,8 +166,7 @@ namespace edm { void readFromEvent(Event const& event); void snapShot(std::vector const& engines, std::vector& cache); - void restoreFromCache(std::vector const& cache, - std::vector& engines); + void restoreFromCache(std::vector const& cache, std::vector& engines); void checkEngineType(std::string const& typeFromConfig, std::string const& typeFromEvent, @@ -175,16 +175,12 @@ namespace edm { void saveStatesToFile(std::string const& fileName, StreamID const& streamID, LuminosityBlockIndex const& lumiIndex); - void writeStates(std::vector const& v, - std::ofstream& outFile); - void writeVector(VUint32 const& v, - std::ofstream& outFile); + void writeStates(std::vector const& v, std::ofstream& outFile); + void writeVector(VUint32 const& v, std::ofstream& outFile); std::string constructSaveFileName() const; - void readEventStatesFromTextFile(std::string const& fileName, - std::vector& cache); - void readLumiStatesFromTextFile(std::string const& fileName, - std::vector& cache); + void readEventStatesFromTextFile(std::string const& fileName, std::vector& cache); + void readLumiStatesFromTextFile(std::string const& fileName, std::vector& cache); void readStatesFromFile(std::string const& fileName, std::vector& cache, std::string const& whichStates); @@ -211,12 +207,12 @@ namespace edm { // This exists because we can look things up faster using the moduleID // than using string comparisons with the moduleLabel - std::vector > streamModuleIDToEngine_; // streamID, sorted by moduleID - std::vector > lumiModuleIDToEngine_; // luminosityBlockIndex, sortedByModuleID + std::vector> streamModuleIDToEngine_; // streamID, sorted by moduleID + std::vector> lumiModuleIDToEngine_; // luminosityBlockIndex, sortedByModuleID // Holds the engines, plus the seeds and module label also - std::vector > streamEngines_; // streamID, sorted by label - std::vector > lumiEngines_; // luminosityBlockIndex, sorted by label + std::vector> streamEngines_; // streamID, sorted by label + std::vector> lumiEngines_; // luminosityBlockIndex, sorted by label // These hold the input tags needed to retrieve the states // of the random number engines stored in a previous process. @@ -225,8 +221,8 @@ namespace edm { edm::InputTag restoreStateTag_; edm::InputTag restoreStateBeginLumiTag_; - std::vector > eventCache_; // streamID, sorted by module label - std::vector > lumiCache_; // luminosityBlockIndex, sorted by module label + std::vector> eventCache_; // streamID, sorted by module label + std::vector> lumiCache_; // luminosityBlockIndex, sorted by module label // This is used to keep track of the seeds and engine name from // the configuration. The map key is the module label. @@ -234,12 +230,13 @@ namespace edm { // It is left as max unsigned if the module is never constructed and not in the process class SeedsAndName { public: - SeedsAndName(VUint32 const& theSeeds, std::string const& theEngineName) : - seeds_(theSeeds), engineName_(theEngineName), moduleID_(std::numeric_limits::max()) { } + SeedsAndName(VUint32 const& theSeeds, std::string const& theEngineName) + : seeds_(theSeeds), engineName_(theEngineName), moduleID_(std::numeric_limits::max()) {} VUint32 const& seeds() const { return seeds_; } std::string const& engineName() const { return engineName_; } unsigned int moduleID() const { return moduleID_; } void setModuleID(unsigned int v) { moduleID_ = v; } + private: VUint32 seeds_; std::string engineName_; @@ -253,7 +250,7 @@ namespace edm { // the save file name has been recorded in the job report. std::string saveFileName_; std::atomic saveFileNameRecorded_; - std::vector>> outFiles_; // streamID + std::vector>> outFiles_; // streamID // Keep the name of the file from which we restore the state // of all declared engines at the beginning of a run. A @@ -275,6 +272,6 @@ namespace edm { static const std::uint32_t maxSeedHepJames; static const std::uint32_t maxSeedTRandom3; }; - } -} + } // namespace service +} // namespace edm #endif diff --git a/IOMC/RandomEngine/src/TRandomAdaptor.cc b/IOMC/RandomEngine/src/TRandomAdaptor.cc index 3c9ace4e110bc..719fe277f4773 100644 --- a/IOMC/RandomEngine/src/TRandomAdaptor.cc +++ b/IOMC/RandomEngine/src/TRandomAdaptor.cc @@ -14,117 +14,113 @@ namespace edm { -TRandomAdaptor::TRandomAdaptor() : trand_(new TRandom3()) { - theSeed = trand_->GetSeed(); -} -TRandomAdaptor::TRandomAdaptor( long seed ) : trand_(new TRandom3(seed)) { - theSeed = trand_->GetSeed(); -} -TRandomAdaptor::TRandomAdaptor( int rowIndex, int colIndex ) : trand_(new TRandom3(rowIndex*colIndex-1)) { - theSeed = trand_->GetSeed(); -} - -TRandomAdaptor::TRandomAdaptor(std::istream&) { - Grumble(std::string("Cannot instantiate a TRandom engine from an istream")); -} - -TRandomAdaptor::~TRandomAdaptor() { -} - -std::ostream& TRandomAdaptor::put(std::ostream& os) const { - Grumble(std::string("put(std::ostream) not available for TRandom engines")); - return os; -} - -std::vector TRandomAdaptor::put() const { - std::vector v; - - int32_t itemSize = sizeof(uint32_t); - TBufferFile buffer(TBuffer::kWrite, 2048 * itemSize); - trand_->Streamer(buffer); - buffer.SetReadMode(); - char* bufferPtr = buffer.Buffer(); - int32_t numItems = (buffer.Length() + itemSize - 1) / itemSize; - v.reserve(numItems + 1); - v.push_back(CLHEP::engineIDulong()); - for(int i = 0; i < numItems; ++i) { - - // Here we do some ugly manipulations to the data to match the format - // of the output of the CLHEP put function (the whole point of this adaptor - // is to make TRandom3 work through the CLHEP interface as if it were a - // a CLHEP engine ...). In CLHEP, the vector returned by put contains - // unsigned long's, but these always contain only 32 bits of information. - // In the case of a 64 bit build the top 32 bits is only padding (all 0's). - - // Get the next 32 bits of data from the buffer - uint32_t value32 = *reinterpret_cast(bufferPtr + i * itemSize); - - if(i == numItems - 1) { - int nBytes = buffer.Length() % itemSize; - if(nBytes == 1) value32 &= 0xffu; - else if(nBytes == 2) value32 &= 0xffffu; - else if(nBytes == 3) value32 &= 0xffffffu; + TRandomAdaptor::TRandomAdaptor() : trand_(new TRandom3()) { theSeed = trand_->GetSeed(); } + TRandomAdaptor::TRandomAdaptor(long seed) : trand_(new TRandom3(seed)) { theSeed = trand_->GetSeed(); } + TRandomAdaptor::TRandomAdaptor(int rowIndex, int colIndex) : trand_(new TRandom3(rowIndex * colIndex - 1)) { + theSeed = trand_->GetSeed(); + } + + TRandomAdaptor::TRandomAdaptor(std::istream&) { + Grumble(std::string("Cannot instantiate a TRandom engine from an istream")); + } + + TRandomAdaptor::~TRandomAdaptor() {} + + std::ostream& TRandomAdaptor::put(std::ostream& os) const { + Grumble(std::string("put(std::ostream) not available for TRandom engines")); + return os; + } + + std::vector TRandomAdaptor::put() const { + std::vector v; + + int32_t itemSize = sizeof(uint32_t); + TBufferFile buffer(TBuffer::kWrite, 2048 * itemSize); + trand_->Streamer(buffer); + buffer.SetReadMode(); + char* bufferPtr = buffer.Buffer(); + int32_t numItems = (buffer.Length() + itemSize - 1) / itemSize; + v.reserve(numItems + 1); + v.push_back(CLHEP::engineIDulong()); + for (int i = 0; i < numItems; ++i) { + // Here we do some ugly manipulations to the data to match the format + // of the output of the CLHEP put function (the whole point of this adaptor + // is to make TRandom3 work through the CLHEP interface as if it were a + // a CLHEP engine ...). In CLHEP, the vector returned by put contains + // unsigned long's, but these always contain only 32 bits of information. + // In the case of a 64 bit build the top 32 bits is only padding (all 0's). + + // Get the next 32 bits of data from the buffer + uint32_t value32 = *reinterpret_cast(bufferPtr + i * itemSize); + + if (i == numItems - 1) { + int nBytes = buffer.Length() % itemSize; + if (nBytes == 1) + value32 &= 0xffu; + else if (nBytes == 2) + value32 &= 0xffffu; + else if (nBytes == 3) + value32 &= 0xffffffu; + } + + // Push it into the vector in an unsigned long which may be 32 or 64 bits + v.push_back(static_cast(value32)); } + return v; + } - // Push it into the vector in an unsigned long which may be 32 or 64 bits - v.push_back(static_cast(value32)); + void TRandomAdaptor::setSeed(long seed, int) { + trand_->SetSeed(seed); + theSeed = trand_->GetSeed(); } - return v; -} - -void TRandomAdaptor::setSeed(long seed, int) { - trand_->SetSeed(seed); - theSeed = trand_->GetSeed(); -} - -// Sets the state of the algorithm according to the zero terminated -// array of seeds. It is allowed to ignore one or many seeds in this array. -void TRandomAdaptor::setSeeds(long const* seeds, int) { - trand_->SetSeed(seeds[0]); - theSeed = trand_->GetSeed(); -} - - -std::istream& TRandomAdaptor::get(std::istream& is) { - Grumble(std::string("get(std::istream) not available for TRandom engines")); - return getState(is); -} - -std::istream& TRandomAdaptor::getState(std::istream& is) { - Grumble(std::string("getState(std::istream) not available for TRandom engines")); - return is; -} - -bool TRandomAdaptor::get(std::vector const& v) { - if(v.empty()) return false; - if(v[0] != CLHEP::engineIDulong()) return false; - int32_t numItems = v.size()-1; - - int32_t itemSize = sizeof(uint32_t); - TBufferFile buffer(TBuffer::kRead, numItems * itemSize + 1024); - char* bufferPtr = buffer.Buffer(); - for(int32_t i = 0; i < numItems; ++i) { - - *reinterpret_cast(bufferPtr + i * itemSize) = static_cast(v[i+1] & 0xffffffff); + + // Sets the state of the algorithm according to the zero terminated + // array of seeds. It is allowed to ignore one or many seeds in this array. + void TRandomAdaptor::setSeeds(long const* seeds, int) { + trand_->SetSeed(seeds[0]); + theSeed = trand_->GetSeed(); + } + + std::istream& TRandomAdaptor::get(std::istream& is) { + Grumble(std::string("get(std::istream) not available for TRandom engines")); + return getState(is); + } + + std::istream& TRandomAdaptor::getState(std::istream& is) { + Grumble(std::string("getState(std::istream) not available for TRandom engines")); + return is; } - // Note that this will fail if the TRandom3 version (specifically the TStreamerInfo) - // has changed between the time the state was saved and the time the following call - // is made. Because we are manually calling the Streamer function, the original - // TStreamerInfo is not saved anywhere. Normally ROOT saves the TStreamerInfo - // automatically. - trand_->Streamer(buffer); + bool TRandomAdaptor::get(std::vector const& v) { + if (v.empty()) + return false; + if (v[0] != CLHEP::engineIDulong()) + return false; + int32_t numItems = v.size() - 1; + + int32_t itemSize = sizeof(uint32_t); + TBufferFile buffer(TBuffer::kRead, numItems * itemSize + 1024); + char* bufferPtr = buffer.Buffer(); + for (int32_t i = 0; i < numItems; ++i) { + *reinterpret_cast(bufferPtr + i * itemSize) = static_cast(v[i + 1] & 0xffffffff); + } - return true; -} + // Note that this will fail if the TRandom3 version (specifically the TStreamerInfo) + // has changed between the time the state was saved and the time the following call + // is made. Because we are manually calling the Streamer function, the original + // TStreamerInfo is not saved anywhere. Normally ROOT saves the TStreamerInfo + // automatically. + trand_->Streamer(buffer); -void TRandomAdaptor::Grumble(std::string const& errortext) const { + return true; + } -// Throw an edm::Exception for unimplemented functions - std::ostringstream sstr; - sstr << "Unimplemented Feature: " << errortext << '\n'; - edm::Exception except(edm::errors::UnimplementedFeature, sstr.str()); - throw except; -} + void TRandomAdaptor::Grumble(std::string const& errortext) const { + // Throw an edm::Exception for unimplemented functions + std::ostringstream sstr; + sstr << "Unimplemented Feature: " << errortext << '\n'; + edm::Exception except(edm::errors::UnimplementedFeature, sstr.str()); + throw except; + } } // namespace edm diff --git a/IOMC/RandomEngine/src/TRandomAdaptor.h b/IOMC/RandomEngine/src/TRandomAdaptor.h index a06cfab7108c5..3602d0dca0d05 100644 --- a/IOMC/RandomEngine/src/TRandomAdaptor.h +++ b/IOMC/RandomEngine/src/TRandomAdaptor.h @@ -11,22 +11,21 @@ namespace edm { class TRandomAdaptor : public CLHEP::HepRandomEngine { - public: typedef value_ptr TRandom3Ptr; // Constructors and destructor. TRandomAdaptor(); - TRandomAdaptor( long seed ); - TRandomAdaptor( int rowIndex, int colIndex ); - TRandomAdaptor( std::istream & is ); + TRandomAdaptor(long seed); + TRandomAdaptor(int rowIndex, int colIndex); + TRandomAdaptor(std::istream& is); ~TRandomAdaptor() override; // Returns a pseudo random number in ]0,1[ (i. e., excluding the end points). double flat() override { return trand_->Rndm(); } // Fills an array "vect" of specified size with flat random values. - void flatArray(int const size, double* vect) override { trand_->RndmArray(size,vect); } + void flatArray(int const size, double* vect) override { trand_->RndmArray(size, vect); } // Sets the state of the algorithm according to seed. void setSeed(long seed, int) override; @@ -39,7 +38,7 @@ namespace edm { void saveStatus(char const filename[] = "TRandom.conf") const override { trand_->WriteRandom(filename); } // Reads from named file the the last saved engine status and restores it. - void restoreStatus(char const filename[] = "TRandom.conf" ) override { trand_->ReadRandom(filename); } + void restoreStatus(char const filename[] = "TRandom.conf") override { trand_->ReadRandom(filename); } // Dumps the current engine status on the screen. void showStatus() const override { trand_->Dump(); } @@ -47,33 +46,32 @@ namespace edm { // Returns a float flat ]0,1[ operator float() override { return (float)(trand_->Rndm()); } - // Returns an unsigned int (32-bit) flat - operator unsigned int() override { return (unsigned int)((trand_->Rndm())*exponent_bit_32()); } + // Returns an unsigned int (32-bit) flat + operator unsigned int() override { return (unsigned int)((trand_->Rndm()) * exponent_bit_32()); } - std::ostream & put(std::ostream & os) const override; - std::istream & get(std::istream & is) override; - std::string beginTag ( ) { return std::string(trand_->GetName())+std::string("-begin"); } - std::istream & getState ( std::istream & is ) override; + std::ostream& put(std::ostream& os) const override; + std::istream& get(std::istream& is) override; + std::string beginTag() { return std::string(trand_->GetName()) + std::string("-begin"); } + std::istream& getState(std::istream& is) override; // Returns the engine name as a string - std::string name() const override { return std::string("T")+std::string(trand_->GetName()); } + std::string name() const override { return std::string("T") + std::string(trand_->GetName()); } static std::string engineName() { return std::string("TRandomAdaptor"); } - std::vector put () const override; - bool get (std::vector const& v) override; - bool getState (std::vector const& v) override { return get(v); } + std::vector put() const override; + bool get(std::vector const& v) override; + bool getState(std::vector const& v) override { return get(v); } // In case all else fails, let the user talk directly to the engine TRandom3* getRootEngine() { return trand_.operator->(); } private: - void Grumble(std::string const& errortext) const; mutable TRandom3Ptr trand_; - }; // TRandomAdaptor + }; // TRandomAdaptor } // namespace edm -#endif // IOMC_RandomEngine_TRandomAdaptor_h +#endif // IOMC_RandomEngine_TRandomAdaptor_h diff --git a/IOMC/RandomEngine/test/TestRandomNumberServiceGlobal.cc b/IOMC/RandomEngine/test/TestRandomNumberServiceGlobal.cc index b1428c63d5f00..05f9f867e64dd 100644 --- a/IOMC/RandomEngine/test/TestRandomNumberServiceGlobal.cc +++ b/IOMC/RandomEngine/test/TestRandomNumberServiceGlobal.cc @@ -88,9 +88,7 @@ namespace { class TestRandomNumberServiceStreamCache { public: - TestRandomNumberServiceStreamCache() : - serviceEngine_(nullptr), - countEvents_(0) { } + TestRandomNumberServiceStreamCache() : serviceEngine_(nullptr), countEvents_(0) {} edm::propagate_const serviceEngine_; unsigned int countEvents_; std::ofstream outFile_; @@ -102,13 +100,14 @@ class TestRandomNumberServiceStreamCache { class TestRandomNumberServiceLumiCache { public: - TestRandomNumberServiceLumiCache() { } + TestRandomNumberServiceLumiCache() {} edm::propagate_const> referenceEngine_; std::vector referenceRandomNumbers_; }; -class TestRandomNumberServiceGlobal : public edm::global::EDAnalyzer, - edm::LuminosityBlockCache > { +class TestRandomNumberServiceGlobal + : public edm::global::EDAnalyzer, + edm::LuminosityBlockCache> { public: explicit TestRandomNumberServiceGlobal(edm::ParameterSet const&); ~TestRandomNumberServiceGlobal(); @@ -117,11 +116,10 @@ class TestRandomNumberServiceGlobal : public edm::global::EDAnalyzer globalBeginLuminosityBlock(edm::LuminosityBlock const&, - edm::EventSetup const&) const override; + virtual std::shared_ptr globalBeginLuminosityBlock( + edm::LuminosityBlock const&, edm::EventSetup const&) const override; - virtual void globalEndLuminosityBlock(edm::LuminosityBlock const&, - edm::EventSetup const&) const override { } + virtual void globalEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) const override {} virtual std::unique_ptr beginStream(edm::StreamID) const override; virtual void endStream(edm::StreamID) const override; @@ -129,11 +127,14 @@ class TestRandomNumberServiceGlobal : public edm::global::EDAnalyzer seeds_; unsigned int offset_; @@ -154,48 +155,47 @@ class TestRandomNumberServiceGlobal : public edm::global::EDAnalyzer("engineName")), - seeds_(pset.getUntrackedParameter >("seeds")), - offset_(pset.getUntrackedParameter("offset")), - maxEvents_(pset.getUntrackedParameter("maxEvents")), - nStreams_(pset.getUntrackedParameter("nStreams")), - skippedEvents_(pset.getUntrackedParameter >("skippedEvents", std::vector(1, 0))), - seedByLumi_(pset.getUntrackedParameter >("seedByLumi", std::vector(1, 0))), - multiStreamReplay_(pset.getUntrackedParameter("multiStreamReplay", false)), - dump_(pset.getUntrackedParameter("dump", false)) { - - if(dump_) { +TestRandomNumberServiceGlobal::TestRandomNumberServiceGlobal(edm::ParameterSet const& pset) + : engineName_(pset.getUntrackedParameter("engineName")), + seeds_(pset.getUntrackedParameter>("seeds")), + offset_(pset.getUntrackedParameter("offset")), + maxEvents_(pset.getUntrackedParameter("maxEvents")), + nStreams_(pset.getUntrackedParameter("nStreams")), + skippedEvents_( + pset.getUntrackedParameter>("skippedEvents", std::vector(1, 0))), + seedByLumi_(pset.getUntrackedParameter>("seedByLumi", std::vector(1, 0))), + multiStreamReplay_(pset.getUntrackedParameter("multiStreamReplay", false)), + dump_(pset.getUntrackedParameter("dump", false)) { + if (dump_) { edm::Service rng; bool exceptionThrown = true; try { - unsigned int mySeed = rng->mySeed(); - std::cout << "*** TestRandomNumberServiceGlobal constructor " << mySeed << "\n"; - exceptionThrown = false; - } catch( cms::Exception const&) { + unsigned int mySeed = rng->mySeed(); + std::cout << "*** TestRandomNumberServiceGlobal constructor " << mySeed << "\n"; + exceptionThrown = false; + } catch (cms::Exception const&) { } - if(not exceptionThrown) { - throw cms::Exception("FailedToThrow")<<"RandomNunberGenerator::mySeed did not throw"; + if (not exceptionThrown) { + throw cms::Exception("FailedToThrow") << "RandomNunberGenerator::mySeed did not throw"; } } } -TestRandomNumberServiceGlobal::~TestRandomNumberServiceGlobal() { -} - -void -TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, edm::Event const& event, edm::EventSetup const&) const { +TestRandomNumberServiceGlobal::~TestRandomNumberServiceGlobal() {} +void TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, + edm::Event const& event, + edm::EventSetup const&) const { // Add some sleep to encourage all the streams to get events to process. - if(nStreams_ > 1) { + if (nStreams_ > 1) { usleep(25000); } - if(dump_) { + if (dump_) { edm::Service rng; std::cout << "*** TestRandomNumberServiceGlobal analyze " << rng->mySeed() << " " - << rng->getEngine(streamID).name() << " streamID= " << streamID - << " multiStreamReplay: " << multiStreamReplay_ << "\n"; + << rng->getEngine(streamID).name() << " streamID= " << streamID + << " multiStreamReplay: " << multiStreamReplay_ << "\n"; } TestRandomNumberServiceStreamCache* cache = streamCache(streamID); @@ -205,9 +205,9 @@ TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, edm::Event const& edm::Service rng; CLHEP::HepRandomEngine& engine = rng->getEngine(streamID); - if(cache->serviceEngine_ != &engine){ + if (cache->serviceEngine_ != &engine) { throw cms::Exception("TestRandomNumberService") - << "TestRandomNumberServiceGlobal::analyze: Engines from event and beginStream are not the same"; + << "TestRandomNumberServiceGlobal::analyze: Engines from event and beginStream are not the same"; } // Generate some random numbers for tests purposes @@ -220,9 +220,9 @@ TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, edm::Event const& double mean = 10.0; // Mean of the exponential double randomNumberEvent3_ = expDist.fire(mean); - if(dump_) { - std::cout << " " << engine.name() << " " << randomNumberEvent0_<< " " << randomNumberEvent1_<< " " - << randomNumberEvent2_<< " " << randomNumberEvent3_ << std::endl; + if (dump_) { + std::cout << " " << engine.name() << " " << randomNumberEvent0_ << " " << randomNumberEvent1_ << " " + << randomNumberEvent2_ << " " << randomNumberEvent3_ << std::endl; } // Write them to a text file @@ -232,24 +232,28 @@ TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, edm::Event const& cache->outFile_ << randomNumberEvent2_ << "\n"; cache->outFile_ << randomNumberEvent3_ << "\n"; - if(!multiStreamReplay_) { + if (!multiStreamReplay_) { // Compare with the reference numbers when not skipping events at the beginning - if(skippedEvents_.size() == 1 && skippedEvents_[0] == 0) { - if(randomNumberEvent0_ != cache->referenceRandomNumbers_.at(0 + 4 * cache->countEvents_) || - randomNumberEvent1_ != cache->referenceRandomNumbers_.at(1 + 4 * cache->countEvents_) || - randomNumberEvent2_ != cache->referenceRandomNumbers_.at(2 + 4 * cache->countEvents_) || - randomNumberEvent3_ != cache->referenceRandomNumbers_.at(3 + 4 * cache->countEvents_)) { + if (skippedEvents_.size() == 1 && skippedEvents_[0] == 0) { + if (randomNumberEvent0_ != cache->referenceRandomNumbers_.at(0 + 4 * cache->countEvents_) || + randomNumberEvent1_ != cache->referenceRandomNumbers_.at(1 + 4 * cache->countEvents_) || + randomNumberEvent2_ != cache->referenceRandomNumbers_.at(2 + 4 * cache->countEvents_) || + randomNumberEvent3_ != cache->referenceRandomNumbers_.at(3 + 4 * cache->countEvents_)) { throw cms::Exception("TestRandomNumberService") - << "TestRandomNumberServiceGlobal::analyze: Random sequence does not match expected sequence"; + << "TestRandomNumberServiceGlobal::analyze: Random sequence does not match expected sequence"; } - // Now compare when skipping events - } else if(cache->countEvents_ < skippedEvents_.size()) { - if(randomNumberEvent0_ != cache->referenceRandomNumbers_.at(0 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_))) || - randomNumberEvent1_ != cache->referenceRandomNumbers_.at(1 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_))) || - randomNumberEvent2_ != cache->referenceRandomNumbers_.at(2 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_))) || - randomNumberEvent3_ != cache->referenceRandomNumbers_.at(3 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_)))) { + // Now compare when skipping events + } else if (cache->countEvents_ < skippedEvents_.size()) { + if (randomNumberEvent0_ != cache->referenceRandomNumbers_.at( + 0 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_))) || + randomNumberEvent1_ != cache->referenceRandomNumbers_.at( + 1 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_))) || + randomNumberEvent2_ != cache->referenceRandomNumbers_.at( + 2 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_))) || + randomNumberEvent3_ != cache->referenceRandomNumbers_.at( + 3 + 4 * (cache->countEvents_ + skippedEvents_.at(cache->countEvents_)))) { throw cms::Exception("TestRandomNumberService") - << "TestRandomNumberServiceGlobal::analyze: Random sequence does not match expected sequence"; + << "TestRandomNumberServiceGlobal::analyze: Random sequence does not match expected sequence"; } } } @@ -268,15 +272,15 @@ TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, edm::Event const& cache->lastEventRandomNumbers_ = ss.str(); // Print the numbers to a file for each event - if(nStreams_ > 1) { + if (nStreams_ > 1) { { std::lock_guard lock(write_mutex); std::ostringstream ss; - if(multiStreamReplay_) { + if (multiStreamReplay_) { ss << "replay"; } - ss << "testRandomServiceL" << event.eventAuxiliary().luminosityBlock() - << "E" << event.eventAuxiliary().event() << ".txt"; + ss << "testRandomServiceL" << event.eventAuxiliary().luminosityBlock() << "E" << event.eventAuxiliary().event() + << ".txt"; std::string filename = ss.str(); std::ofstream outFile; @@ -299,42 +303,40 @@ TestRandomNumberServiceGlobal::analyze(edm::StreamID streamID, edm::Event const& } void TestRandomNumberServiceGlobal::beginJob() { - if(dump_) { + if (dump_) { bool exceptionThrown = true; try { - edm::Service rng; - unsigned int mySeed = rng->mySeed(); - std::cout << "*** TestRandomNumberServiceGlobal beginJob " << mySeed << "\n"; - exceptionThrown = false; - } catch( cms::Exception const&) { + edm::Service rng; + unsigned int mySeed = rng->mySeed(); + std::cout << "*** TestRandomNumberServiceGlobal beginJob " << mySeed << "\n"; + exceptionThrown = false; + } catch (cms::Exception const&) { } - if(not exceptionThrown) { - throw cms::Exception("FailedToThrow")<<"RandomNunberGenerator::mySeed did not throw"; + if (not exceptionThrown) { + throw cms::Exception("FailedToThrow") << "RandomNunberGenerator::mySeed did not throw"; } } } void TestRandomNumberServiceGlobal::endJob() { - if(dump_) { + if (dump_) { bool exceptionThrown = true; try { - edm::Service rng; - unsigned int mySeed = rng->mySeed(); - std::cout << "*** TestRandomNumberServiceGlobal endJob " << mySeed << "\n"; - exceptionThrown = false; - } catch( cms::Exception const&) { + edm::Service rng; + unsigned int mySeed = rng->mySeed(); + std::cout << "*** TestRandomNumberServiceGlobal endJob " << mySeed << "\n"; + exceptionThrown = false; + } catch (cms::Exception const&) { } - if(not exceptionThrown) { - throw cms::Exception("FailedToThrow")<<"RandomNunberGenerator::mySeed did not throw"; + if (not exceptionThrown) { + throw cms::Exception("FailedToThrow") << "RandomNunberGenerator::mySeed did not throw"; } } } -std::shared_ptr -TestRandomNumberServiceGlobal::globalBeginLuminosityBlock(edm::LuminosityBlock const& lumi, - edm::EventSetup const& iES) const { - - if(dump_) { +std::shared_ptr TestRandomNumberServiceGlobal::globalBeginLuminosityBlock( + edm::LuminosityBlock const& lumi, edm::EventSetup const& iES) const { + if (dump_) { edm::Service rng; std::cout << "*** TestRandomNumberServiceGlobal beginLuminosityBlock " << rng->mySeed() << " " << rng->getEngine(lumi.index()).name() << "\n"; @@ -343,25 +345,28 @@ TestRandomNumberServiceGlobal::globalBeginLuminosityBlock(edm::LuminosityBlock c auto lumiCache = std::make_shared(); unsigned int seed0 = seeds_.at(0) + nStreams_; - if(lumi.luminosityBlockAuxiliary().luminosityBlock() < seedByLumi_.size()) { + if (lumi.luminosityBlockAuxiliary().luminosityBlock() < seedByLumi_.size()) { seed0 = seedByLumi_.at(lumi.luminosityBlockAuxiliary().luminosityBlock()); } - if(engineName_ == "RanecuEngine") { - lumiCache->referenceEngine_ = std::shared_ptr(new CLHEP::RanecuEngine()); // propagate_const has no reset() function + if (engineName_ == "RanecuEngine") { + lumiCache->referenceEngine_ = std::shared_ptr( + new CLHEP::RanecuEngine()); // propagate_const has no reset() function long int seedL[2]; seedL[0] = static_cast(seed0); seedL[1] = static_cast(seeds_.at(1)); - lumiCache->referenceEngine_->setSeeds(seedL,0); - } - else { + lumiCache->referenceEngine_->setSeeds(seedL, 0); + } else { long int seedL = static_cast(seed0); - if(engineName_ == "HepJamesRandom") { - lumiCache->referenceEngine_ = std::shared_ptr(new CLHEP::HepJamesRandom(seedL)); // propagate_const has no reset() function - } else if(engineName_ == "MixMaxRng") { - lumiCache->referenceEngine_ = std::shared_ptr(new CLHEP::MixMaxRng(seedL)); // propagate_const has no reset() function + if (engineName_ == "HepJamesRandom") { + lumiCache->referenceEngine_ = std::shared_ptr( + new CLHEP::HepJamesRandom(seedL)); // propagate_const has no reset() function + } else if (engineName_ == "MixMaxRng") { + lumiCache->referenceEngine_ = std::shared_ptr( + new CLHEP::MixMaxRng(seedL)); // propagate_const has no reset() function } else { - lumiCache->referenceEngine_ = std::shared_ptr(new edm::TRandomAdaptor(seedL)); // propagate_const has no reset() function + lumiCache->referenceEngine_ = std::shared_ptr( + new edm::TRandomAdaptor(seedL)); // propagate_const has no reset() function } } @@ -373,21 +378,18 @@ TestRandomNumberServiceGlobal::globalBeginLuminosityBlock(edm::LuminosityBlock c double x1 = engine.flat(); double x2 = engine.flat(); - if(x1 != y1 || x2 != y2) { + if (x1 != y1 || x2 != y2) { throw cms::Exception("TestRandomNumberService") - << "TestRandomNumberServiceGlobal::globalBeginLuminosityBlock: " - << " x1= " << x1 << " y1= " << y1 << " x2= " << x2 << " y2= " << y2 << " " - << engine.name() << " " << lumiCache->referenceEngine_->name() - << " seed0= " << seed0 << " nStream= " << nStreams_; + << "TestRandomNumberServiceGlobal::globalBeginLuminosityBlock: " + << " x1= " << x1 << " y1= " << y1 << " x2= " << x2 << " y2= " << y2 << " " << engine.name() << " " + << lumiCache->referenceEngine_->name() << " seed0= " << seed0 << " nStream= " << nStreams_; } return lumiCache; } - -std::unique_ptr -TestRandomNumberServiceGlobal::beginStream(edm::StreamID streamID) const { - +std::unique_ptr TestRandomNumberServiceGlobal::beginStream( + edm::StreamID streamID) const { auto streamCache = std::make_unique(); edm::Service rng; @@ -404,25 +406,28 @@ TestRandomNumberServiceGlobal::beginStream(edm::StreamID streamID) const { std::string outFileName = std::string("testRandomService") + suffix.str() + std::string(".txt"); streamCache->outFile_.open(outFileName.c_str(), std::ofstream::out); - if(engineName_ == "RanecuEngine") { - streamCache->referenceEngine_ = std::shared_ptr(new CLHEP::RanecuEngine()); // propagate_const has no reset() function + if (engineName_ == "RanecuEngine") { + streamCache->referenceEngine_ = std::shared_ptr( + new CLHEP::RanecuEngine()); // propagate_const has no reset() function long int seedL[2]; seedL[0] = static_cast(seeds_.at(0) + streamID.value() + offset_); seedL[1] = static_cast(seeds_.at(1)); - streamCache->referenceEngine_->setSeeds(seedL,0); - } - else { + streamCache->referenceEngine_->setSeeds(seedL, 0); + } else { long int seedL = static_cast(seeds_.at(0) + streamID.value() + offset_); - if(engineName_ == "HepJamesRandom") { - streamCache->referenceEngine_ = std::shared_ptr(new CLHEP::HepJamesRandom(seedL)); // propagate_const has no reset() function - } else if(engineName_ == "MixMaxRng") { - streamCache->referenceEngine_ = std::shared_ptr(new CLHEP::MixMaxRng(seedL)); // propagate_const has no reset() function + if (engineName_ == "HepJamesRandom") { + streamCache->referenceEngine_ = std::shared_ptr( + new CLHEP::HepJamesRandom(seedL)); // propagate_const has no reset() function + } else if (engineName_ == "MixMaxRng") { + streamCache->referenceEngine_ = std::shared_ptr( + new CLHEP::MixMaxRng(seedL)); // propagate_const has no reset() function } else { - streamCache->referenceEngine_ = std::shared_ptr(new edm::TRandomAdaptor(seedL)); // propagate_const has no reset() function + streamCache->referenceEngine_ = std::shared_ptr( + new edm::TRandomAdaptor(seedL)); // propagate_const has no reset() function } } - for(unsigned int i = 0; i < maxEvents_; ++i) { + for (unsigned int i = 0; i < maxEvents_; ++i) { streamCache->referenceRandomNumbers_.push_back(streamCache->referenceEngine_->flat()); streamCache->referenceRandomNumbers_.push_back(streamCache->referenceEngine_->flat()); streamCache->referenceRandomNumbers_.push_back(streamCache->referenceEngine_->flat()); @@ -433,9 +438,7 @@ TestRandomNumberServiceGlobal::beginStream(edm::StreamID streamID) const { return streamCache; } -void -TestRandomNumberServiceGlobal::endStream(edm::StreamID streamID) const { - +void TestRandomNumberServiceGlobal::endStream(edm::StreamID streamID) const { TestRandomNumberServiceStreamCache* cache = streamCache(streamID); std::ostringstream ss; @@ -460,8 +463,9 @@ TestRandomNumberServiceGlobal::endStream(edm::StreamID streamID) const { // engine.flat(); } -void -TestRandomNumberServiceGlobal::streamBeginRun(edm::StreamID streamID, edm::Run const&, edm::EventSetup const&) const { +void TestRandomNumberServiceGlobal::streamBeginRun(edm::StreamID streamID, + edm::Run const&, + edm::EventSetup const&) const { // Running this will causing the checking code to throw an exception // Uncomment to verify the checking works // edm::Service rng; @@ -469,8 +473,9 @@ TestRandomNumberServiceGlobal::streamBeginRun(edm::StreamID streamID, edm::Run c // engine.flat(); } -void -TestRandomNumberServiceGlobal::streamEndRun(edm::StreamID streamID, edm::Run const&, edm::EventSetup const&) const { +void TestRandomNumberServiceGlobal::streamEndRun(edm::StreamID streamID, + edm::Run const&, + edm::EventSetup const&) const { // Running this will causing the checking code to throw an exception // Uncomment to verify the checking works // edm::Service rng; @@ -478,8 +483,9 @@ TestRandomNumberServiceGlobal::streamEndRun(edm::StreamID streamID, edm::Run con // engine.flat(); } -void -TestRandomNumberServiceGlobal::streamBeginLuminosityBlock(edm::StreamID streamID, edm::LuminosityBlock const&, edm::EventSetup const&) const { +void TestRandomNumberServiceGlobal::streamBeginLuminosityBlock(edm::StreamID streamID, + edm::LuminosityBlock const&, + edm::EventSetup const&) const { // Running this will causing the checking code to throw an exception // Uncomment to verify the checking works // edm::Service rng; @@ -487,8 +493,9 @@ TestRandomNumberServiceGlobal::streamBeginLuminosityBlock(edm::StreamID streamID // engine.flat(); } -void -TestRandomNumberServiceGlobal::streamEndLuminosityBlock(edm::StreamID streamID, edm::LuminosityBlock const&, edm::EventSetup const&) const { +void TestRandomNumberServiceGlobal::streamEndLuminosityBlock(edm::StreamID streamID, + edm::LuminosityBlock const&, + edm::EventSetup const&) const { // Running this will causing the checking code to throw an exception // Uncomment to verify the checking works // edm::Service rng; diff --git a/IgTools/IgProf/plugins/IgProfModule.cc b/IgTools/IgProf/plugins/IgProfModule.cc index 1a78b178380d2..9d02de0ab3150 100644 --- a/IgTools/IgProf/plugins/IgProfModule.cc +++ b/IgTools/IgProf/plugins/IgProfModule.cc @@ -14,117 +14,104 @@ #include #include -class IgProfModule : public edm::EDAnalyzer -{ +class IgProfModule : public edm::EDAnalyzer { public: IgProfModule(const edm::ParameterSet &ps) - : dump_(nullptr), - prescale_(0), - nrecord_(0), - nevent_(0), - nrun_(0), - nlumi_(0), - nfile_(0) - { - // Removing the __extension__ gives a warning which - // is acknowledged as a language problem in the C++ Standard Core - // Language Defect Report - // - // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#195 - // - // since the suggested decision seems to be that the syntax should - // actually be "Conditionally-Supported Behavior" in some - // future C++ standard I simply silence the warning. - if (void *sym = dlsym(nullptr, "igprof_dump_now")) - dump_ = __extension__ (void(*)(const char *)) sym; - else - edm::LogWarning("IgProfModule") - << "IgProfModule requested but application is not" - << " currently being profiled with igprof\n"; - - prescale_ = ps.getUntrackedParameter("reportEventInterval", prescale_); - atBeginJob_ = ps.getUntrackedParameter("reportToFileAtBeginJob", atBeginJob_); - atEndJob_ = ps.getUntrackedParameter("reportToFileAtEndJob", atEndJob_); - atBeginLumi_ = ps.getUntrackedParameter("reportToFileAtBeginLumi", atBeginLumi_); - atEndLumi_ = ps.getUntrackedParameter("reportToFileAtEndLumi", atEndLumi_); - atInputFile_ = ps.getUntrackedParameter("reportToFileAtInputFile", atInputFile_); - atEvent_ = ps.getUntrackedParameter("reportToFileAtEvent", atEvent_); - } - - void beginJob() override - { makeDump(atBeginJob_); } - - void endJob(void) override - { makeDump(atEndJob_); } - - void analyze(const edm::Event &e, const edm::EventSetup &) override - { - nevent_ = e.id().event(); - if (prescale_ > 0 && (++nrecord_ % prescale_) == 1) - makeDump(atEvent_); - } - - void beginRun(const edm::Run &r, const edm::EventSetup &) override - { nrun_ = r.run(); makeDump(atBeginRun_); } - - void endRun(const edm::Run &, const edm::EventSetup &) override - { makeDump(atEndRun_); } - - void beginLuminosityBlock(const edm::LuminosityBlock &l, const edm::EventSetup &) override - { nlumi_ = l.luminosityBlock(); makeDump(atBeginLumi_); } - - void endLuminosityBlock(const edm::LuminosityBlock &l, const edm::EventSetup &) override - { makeDump(atEndLumi_); } - - void respondToOpenInputFile(const edm::FileBlock &) override - { ++nfile_; makeDump(atInputFile_); } + : dump_(nullptr), prescale_(0), nrecord_(0), nevent_(0), nrun_(0), nlumi_(0), nfile_(0) { + // Removing the __extension__ gives a warning which + // is acknowledged as a language problem in the C++ Standard Core + // Language Defect Report + // + // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#195 + // + // since the suggested decision seems to be that the syntax should + // actually be "Conditionally-Supported Behavior" in some + // future C++ standard I simply silence the warning. + if (void *sym = dlsym(nullptr, "igprof_dump_now")) + dump_ = __extension__(void (*)(const char *)) sym; + else + edm::LogWarning("IgProfModule") << "IgProfModule requested but application is not" + << " currently being profiled with igprof\n"; + + prescale_ = ps.getUntrackedParameter("reportEventInterval", prescale_); + atBeginJob_ = ps.getUntrackedParameter("reportToFileAtBeginJob", atBeginJob_); + atEndJob_ = ps.getUntrackedParameter("reportToFileAtEndJob", atEndJob_); + atBeginLumi_ = ps.getUntrackedParameter("reportToFileAtBeginLumi", atBeginLumi_); + atEndLumi_ = ps.getUntrackedParameter("reportToFileAtEndLumi", atEndLumi_); + atInputFile_ = ps.getUntrackedParameter("reportToFileAtInputFile", atInputFile_); + atEvent_ = ps.getUntrackedParameter("reportToFileAtEvent", atEvent_); + } + + void beginJob() override { makeDump(atBeginJob_); } + + void endJob(void) override { makeDump(atEndJob_); } + + void analyze(const edm::Event &e, const edm::EventSetup &) override { + nevent_ = e.id().event(); + if (prescale_ > 0 && (++nrecord_ % prescale_) == 1) + makeDump(atEvent_); + } + + void beginRun(const edm::Run &r, const edm::EventSetup &) override { + nrun_ = r.run(); + makeDump(atBeginRun_); + } + + void endRun(const edm::Run &, const edm::EventSetup &) override { makeDump(atEndRun_); } + + void beginLuminosityBlock(const edm::LuminosityBlock &l, const edm::EventSetup &) override { + nlumi_ = l.luminosityBlock(); + makeDump(atBeginLumi_); + } + + void endLuminosityBlock(const edm::LuminosityBlock &l, const edm::EventSetup &) override { makeDump(atEndLumi_); } + + void respondToOpenInputFile(const edm::FileBlock &) override { + ++nfile_; + makeDump(atInputFile_); + } private: - void makeDump(const std::string &format) - { - if (! dump_ || format.empty()) - return; - - std::string final(format); - final = replace(final, "%I", nrecord_); - final = replaceU64(final, "%E", nevent_); - final = replaceU64(final, "%R", nrun_); - final = replaceU64(final, "%L", nlumi_); - final = replace(final, "%F", nfile_); - dump_(final.c_str()); + void makeDump(const std::string &format) { + if (!dump_ || format.empty()) + return; + + std::string final(format); + final = replace(final, "%I", nrecord_); + final = replaceU64(final, "%E", nevent_); + final = replaceU64(final, "%R", nrun_); + final = replaceU64(final, "%L", nlumi_); + final = replace(final, "%F", nfile_); + dump_(final.c_str()); + } + + static std::string replace(const std::string &s, const char *pat, int val) { + size_t pos = 0; + size_t patlen = strlen(pat); + std::string result = s; + while ((pos = result.find(pat, pos)) != std::string::npos) { + char buf[64]; + int n = sprintf(buf, "%d", val); + result.replace(pos, patlen, buf); + pos = pos - patlen + n; } - static std::string replace(const std::string &s, const char *pat, int val) - { - size_t pos = 0; - size_t patlen = strlen(pat); - std::string result = s; - while ((pos = result.find(pat, pos)) != std::string::npos) - { - char buf[64]; - int n = sprintf(buf, "%d", val); - result.replace(pos, patlen, buf); - pos = pos - patlen + n; - } - - return result; + return result; + } + + static std::string replaceU64(const std::string &s, const char *pat, unsigned long long val) { + size_t pos = 0; + size_t patlen = strlen(pat); + std::string result = s; + while ((pos = result.find(pat, pos)) != std::string::npos) { + char buf[64]; + int n = sprintf(buf, "%llu", val); + result.replace(pos, patlen, buf); + pos = pos - patlen + n; } - static std::string replaceU64(const std::string &s, const char *pat, unsigned long long val) - { - size_t pos = 0; - size_t patlen = strlen(pat); - std::string result = s; - while ((pos = result.find(pat, pos)) != std::string::npos) - { - char buf[64]; - int n = sprintf(buf, "%llu", val); - result.replace(pos, patlen, buf); - pos = pos - patlen + n; - } - - return result; - } + return result; + } void (*dump_)(const char *); std::string atBeginJob_; diff --git a/IgTools/IgProf/plugins/IgProfService.cc b/IgTools/IgProf/plugins/IgProfService.cc index c8efb95ee342f..60f9ec5e414e2 100644 --- a/IgTools/IgProf/plugins/IgProfService.cc +++ b/IgTools/IgProf/plugins/IgProfService.cc @@ -1,6 +1,6 @@ // -// Description: FWK service to implement hook for igprof memory profile +// Description: FWK service to implement hook for igprof memory profile // dump functionality // // Peter Elmer, Princeton University 18 Nov, 2008 @@ -22,65 +22,48 @@ using namespace edm::service; -IgProfService::IgProfService(ParameterSet const& ps, - ActivityRegistry&iRegistry) - : dump_(nullptr), - mineventrecord_(1), - prescale_(1), - nrecord_(0), - nevent_(0), - nrun_(0), - nlumi_(0), - nfileopened_(0), - nfileclosed_(0) { - - - // Removing the __extension__ gives a warning which - // is acknowledged as a language problem in the C++ Standard Core - // Language Defect Report - // - // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#195 - // - // since the suggested decision seems to be that the syntax should - // actually be "Conditionally-Supported Behavior" in some - // future C++ standard I simply silence the warning. - if (void *sym = dlsym(nullptr, "igprof_dump_now")) { - dump_ = __extension__ (void(*)(const char *)) sym; - } else - edm::LogWarning("IgProfModule") - << "IgProfModule requested but application is not" - << " currently being profiled with igprof\n"; +IgProfService::IgProfService(ParameterSet const &ps, ActivityRegistry &iRegistry) + : dump_(nullptr), + mineventrecord_(1), + prescale_(1), + nrecord_(0), + nevent_(0), + nrun_(0), + nlumi_(0), + nfileopened_(0), + nfileclosed_(0) { + // Removing the __extension__ gives a warning which + // is acknowledged as a language problem in the C++ Standard Core + // Language Defect Report + // + // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#195 + // + // since the suggested decision seems to be that the syntax should + // actually be "Conditionally-Supported Behavior" in some + // future C++ standard I simply silence the warning. + if (void *sym = dlsym(nullptr, "igprof_dump_now")) { + dump_ = __extension__(void (*)(const char *)) sym; + } else + edm::LogWarning("IgProfModule") << "IgProfModule requested but application is not" + << " currently being profiled with igprof\n"; // Get the configuration - prescale_ - = ps.getUntrackedParameter("reportEventInterval", prescale_); - mineventrecord_ - = ps.getUntrackedParameter("reportFirstEvent", mineventrecord_); - - atPostBeginJob_ - = ps.getUntrackedParameter("reportToFileAtPostBeginJob", atPostBeginJob_); - atPostBeginRun_ - = ps.getUntrackedParameter("reportToFileAtPostBeginRun", atPostBeginRun_); - atPostBeginLumi_ - = ps.getUntrackedParameter("reportToFileAtPostBeginLumi", atPostBeginLumi_); - - atPreEvent_ - = ps.getUntrackedParameter("reportToFileAtPreEvent", atPreEvent_); - atPostEvent_ - = ps.getUntrackedParameter("reportToFileAtPostEvent", atPostEvent_); - - atPostEndLumi_ - = ps.getUntrackedParameter("reportToFileAtPostEndLumi", atPostEndLumi_); - atPostEndRun_ - = ps.getUntrackedParameter("reportToFileAtPostEndRun", atPostEndRun_); - atPostEndJob_ - = ps.getUntrackedParameter("reportToFileAtPostEndJob", atPostEndJob_); - - atPostOpenFile_ - = ps.getUntrackedParameter("reportToFileAtPostOpenFile", atPostOpenFile_); - atPostCloseFile_ - = ps.getUntrackedParameter("reportToFileAtPostCloseFile", atPostCloseFile_); + prescale_ = ps.getUntrackedParameter("reportEventInterval", prescale_); + mineventrecord_ = ps.getUntrackedParameter("reportFirstEvent", mineventrecord_); + atPostBeginJob_ = ps.getUntrackedParameter("reportToFileAtPostBeginJob", atPostBeginJob_); + atPostBeginRun_ = ps.getUntrackedParameter("reportToFileAtPostBeginRun", atPostBeginRun_); + atPostBeginLumi_ = ps.getUntrackedParameter("reportToFileAtPostBeginLumi", atPostBeginLumi_); + + atPreEvent_ = ps.getUntrackedParameter("reportToFileAtPreEvent", atPreEvent_); + atPostEvent_ = ps.getUntrackedParameter("reportToFileAtPostEvent", atPostEvent_); + + atPostEndLumi_ = ps.getUntrackedParameter("reportToFileAtPostEndLumi", atPostEndLumi_); + atPostEndRun_ = ps.getUntrackedParameter("reportToFileAtPostEndRun", atPostEndRun_); + atPostEndJob_ = ps.getUntrackedParameter("reportToFileAtPostEndJob", atPostEndJob_); + + atPostOpenFile_ = ps.getUntrackedParameter("reportToFileAtPostOpenFile", atPostOpenFile_); + atPostCloseFile_ = ps.getUntrackedParameter("reportToFileAtPostCloseFile", atPostCloseFile_); // Register for the framework signals iRegistry.watchPostBeginJob(this, &IgProfService::postBeginJob); @@ -96,60 +79,51 @@ IgProfService::IgProfService(ParameterSet const& ps, iRegistry.watchPostOpenFile(this, &IgProfService::postOpenFile); iRegistry.watchPostCloseFile(this, &IgProfService::postCloseFile); - } -void IgProfService::postBeginJob() { - makeDump(atPostBeginJob_); -} +void IgProfService::postBeginJob() { makeDump(atPostBeginJob_); } -void IgProfService::postBeginRun(GlobalContext const& gc) { - nrun_ = gc.luminosityBlockID().run(); makeDump(atPostBeginRun_); +void IgProfService::postBeginRun(GlobalContext const &gc) { + nrun_ = gc.luminosityBlockID().run(); + makeDump(atPostBeginRun_); } -void IgProfService::postBeginLumi(GlobalContext const& gc) { - nlumi_ = gc.luminosityBlockID().luminosityBlock(); makeDump(atPostBeginLumi_); +void IgProfService::postBeginLumi(GlobalContext const &gc) { + nlumi_ = gc.luminosityBlockID().luminosityBlock(); + makeDump(atPostBeginLumi_); } -void IgProfService::preEvent(StreamContext const& iStream) { - ++nrecord_; // count before events +void IgProfService::preEvent(StreamContext const &iStream) { + ++nrecord_; // count before events nevent_ = iStream.eventID().event(); - if ((prescale_ > 0) && - (nrecord_ >= mineventrecord_) && - (((nrecord_ - mineventrecord_)% prescale_) == 0)) makeDump(atPreEvent_); + if ((prescale_ > 0) && (nrecord_ >= mineventrecord_) && (((nrecord_ - mineventrecord_) % prescale_) == 0)) + makeDump(atPreEvent_); } -void IgProfService::postEvent(StreamContext const& iStream) { +void IgProfService::postEvent(StreamContext const &iStream) { nevent_ = iStream.eventID().event(); - if ((prescale_ > 0) && - (nrecord_ >= mineventrecord_) && - (((nrecord_ - mineventrecord_)% prescale_) == 0)) makeDump(atPostEvent_); + if ((prescale_ > 0) && (nrecord_ >= mineventrecord_) && (((nrecord_ - mineventrecord_) % prescale_) == 0)) + makeDump(atPostEvent_); } -void IgProfService::postEndLumi(GlobalContext const &) { - makeDump(atPostEndLumi_); -} +void IgProfService::postEndLumi(GlobalContext const &) { makeDump(atPostEndLumi_); } -void IgProfService::postEndRun(GlobalContext const &) { - makeDump(atPostEndRun_); -} +void IgProfService::postEndRun(GlobalContext const &) { makeDump(atPostEndRun_); } -void IgProfService::postEndJob() { - makeDump(atPostEndJob_); -} +void IgProfService::postEndJob() { makeDump(atPostEndJob_); } -void IgProfService::postOpenFile (std::string const&, bool) { - ++nfileopened_; +void IgProfService::postOpenFile(std::string const &, bool) { + ++nfileopened_; makeDump(atPostOpenFile_); -} +} -void IgProfService::postCloseFile (std::string const&, bool) { - ++nfileclosed_; +void IgProfService::postCloseFile(std::string const &, bool) { + ++nfileclosed_; makeDump(atPostCloseFile_); -} +} void IgProfService::makeDump(const std::string &format) { - if (! dump_ || format.empty()) + if (!dump_ || format.empty()) return; std::string final(format); @@ -162,13 +136,11 @@ void IgProfService::makeDump(const std::string &format) { dump_(final.c_str()); } -std::string -IgProfService::replace(const std::string &s, const char *pat, int val) { +std::string IgProfService::replace(const std::string &s, const char *pat, int val) { size_t pos = 0; size_t patlen = strlen(pat); std::string result = s; - while ((pos = result.find(pat, pos)) != std::string::npos) - { + while ((pos = result.find(pat, pos)) != std::string::npos) { char buf[64]; int n = sprintf(buf, "%d", val); result.replace(pos, patlen, buf); @@ -178,13 +150,11 @@ IgProfService::replace(const std::string &s, const char *pat, int val) { return result; } -std::string -IgProfService::replaceU64(const std::string &s, const char *pat, unsigned long long val) { +std::string IgProfService::replaceU64(const std::string &s, const char *pat, unsigned long long val) { size_t pos = 0; size_t patlen = strlen(pat); std::string result = s; - while ((pos = result.find(pat, pos)) != std::string::npos) - { + while ((pos = result.find(pat, pos)) != std::string::npos) { char buf[64]; int n = sprintf(buf, "%llu", val); result.replace(pos, patlen, buf); @@ -195,4 +165,3 @@ IgProfService::replaceU64(const std::string &s, const char *pat, unsigned long l } DEFINE_FWK_SERVICE(IgProfService); - diff --git a/IgTools/IgProf/plugins/IgProfService.h b/IgTools/IgProf/plugins/IgProfService.h index 3a592d84911f8..0adc1faa4f87e 100644 --- a/IgTools/IgProf/plugins/IgProfService.h +++ b/IgTools/IgProf/plugins/IgProfService.h @@ -2,7 +2,7 @@ #define IgTools_IgProf_IgProfService_h // -// Description: FWK service to implement hook for igprof memory profile +// Description: FWK service to implement hook for igprof memory profile // dump functionality // // Peter Elmer, Princeton University 18 Nov, 2008 @@ -15,17 +15,15 @@ namespace edm { class GlobalContext; class StreamContext; - + namespace service { class IgProfService { - public: - - IgProfService(const ParameterSet&,ActivityRegistry&); + IgProfService(const ParameterSet &, ActivityRegistry &); void postBeginJob(); - void postBeginRun(GlobalContext const& gc); + void postBeginRun(GlobalContext const &gc); void postBeginLumi(GlobalContext const &gc); @@ -38,22 +36,16 @@ namespace edm { void postEndJob(); - void postOpenFile(std::string const&, bool); + void postOpenFile(std::string const &, bool); - void postCloseFile(std::string const&, bool); + void postCloseFile(std::string const &, bool); - inline - bool isProcessWideService(IgProfService const*) { - return true; - } + inline bool isProcessWideService(IgProfService const *) { return true; } private: - void makeDump(const std::string &format); - static std::string replace(const std::string &s, - const char *pat, int val); - static std::string replaceU64(const std::string &s, - const char *pat, unsigned long long val); + static std::string replace(const std::string &s, const char *pat, int val); + static std::string replaceU64(const std::string &s, const char *pat, unsigned long long val); void (*dump_)(const char *); @@ -73,16 +65,14 @@ namespace edm { int mineventrecord_; int prescale_; - int nrecord_; // counter + int nrecord_; // counter edm::EventNumber_t nevent_; edm::RunNumber_t nrun_; edm::LuminosityBlockNumber_t nlumi_; int nfileopened_; // counter of files opened thus far int nfileclosed_; // counter of files closed thus far - }; - } -} - + } // namespace service +} // namespace edm #endif diff --git a/PerfTools/Callgrind/interface/ProfilerService.h b/PerfTools/Callgrind/interface/ProfilerService.h index db33a9d3fd84a..993cbb2ceb233 100644 --- a/PerfTools/Callgrind/interface/ProfilerService.h +++ b/PerfTools/Callgrind/interface/ProfilerService.h @@ -1,7 +1,6 @@ #ifndef ProfilerService_H #define ProfilerService_H - //FIXME only forward declarations??? #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" @@ -14,7 +13,7 @@ namespace test { class TestProfilerService; struct CheckPaths; -} +} // namespace test /** \class ProfilerService * A Service to start and stop calgrind profiling on demand... @@ -27,18 +26,14 @@ class ProfilerService { friend struct test::CheckPaths; public: - /// Standard Service Constructor - ProfilerService(edm::ParameterSet const& pset, - edm::ActivityRegistry& activity); + ProfilerService(edm::ParameterSet const& pset, edm::ActivityRegistry& activity); /// Destructor ~ProfilerService(); - // ---- Public Interface ----- - /// start instrumentation if not active. true if started now bool startInstrumentation(); @@ -58,50 +53,40 @@ class ProfilerService { void dumpStat() const; /// true if the current event has to be instrumented - bool doEvent() const { return m_doEvent;} + bool doEvent() const { return m_doEvent; } /// true if instrumentation is active - bool active() const { return m_active>0;} - + bool active() const { return m_active > 0; } // ---- Service Interface: to be called only by the Framework ---- - - void preSourceI(edm::StreamID) { - fullEvent(); - } - void beginEventI(edm::StreamContext const& stream) { - beginEvent(); - } + void preSourceI(edm::StreamID) { fullEvent(); } - void endEventI(edm::StreamContext const& stream) { - endEvent(); - } - void beginPathI(edm::StreamContext const& stream, edm::PathContext const& path) { - beginPath(path.pathName()); - } + void beginEventI(edm::StreamContext const& stream) { beginEvent(); } + + void endEventI(edm::StreamContext const& stream) { endEvent(); } + void beginPathI(edm::StreamContext const& stream, edm::PathContext const& path) { beginPath(path.pathName()); } void endPathI(edm::StreamContext const& stream, edm::PathContext const& path, edm::HLTPathStatus const&) { endPath(path.pathName()); } private: - void fullEvent(); void beginEvent(); void endEvent(); - + void beginPath(std::string const& path); void endPath(std::string const& path); void newEvent(); // configurable - int m_firstEvent; + int m_firstEvent; int m_lastEvent; int m_dumpInterval; - std::vector m_paths; - std::vector m_excludedPaths; + std::vector m_paths; + std::vector m_excludedPaths; bool m_allPaths; // internal state @@ -111,7 +96,6 @@ class ProfilerService { int m_active; bool m_paused; std::string m_activePath; +}; -}; - -#endif // ProfilerService_H +#endif // ProfilerService_H diff --git a/PerfTools/Callgrind/plugins/CallgrindAnalyzer.cc b/PerfTools/Callgrind/plugins/CallgrindAnalyzer.cc index 69b04db898b27..016e997e30414 100644 --- a/PerfTools/Callgrind/plugins/CallgrindAnalyzer.cc +++ b/PerfTools/Callgrind/plugins/CallgrindAnalyzer.cc @@ -2,7 +2,7 @@ // // Package: Profiler // Class: Profiler -// +// /**\class Profiler Profiler.cc PerfTools/Callgrind/plugins/Profiler.cc Description: @@ -16,7 +16,6 @@ // // - // system include files #include @@ -29,7 +28,6 @@ #include "FWCore/ParameterSet/interface/ParameterSet.h" - #include "valgrind/callgrind.h" // // class declaration @@ -41,17 +39,16 @@ class Profiler : public edm::EDAnalyzer { explicit Profiler(const edm::ParameterSet&); ~Profiler() override; - private: - void beginJob() override ; + void beginJob() override; void analyze(const edm::Event&, const edm::EventSetup&) override; - void endJob() override ; - + void endJob() override; + // ----------member data --------------------------- - int m_firstEvent; - int m_lastEvent; - int m_action; - int m_evtCount; + int m_firstEvent; + int m_lastEvent; + int m_action; + int m_evtCount; }; // @@ -65,26 +62,19 @@ class Profiler : public edm::EDAnalyzer { // // constructors and destructor // -Profiler::Profiler(const edm::ParameterSet& parameters) -{ - //now do what ever initialization is needed - m_firstEvent = parameters.getParameter("firstEvent"); - m_lastEvent = parameters.getParameter("lastEvent"); - m_action = parameters.getParameter("action"); - m_evtCount=0; - +Profiler::Profiler(const edm::ParameterSet& parameters) { + //now do what ever initialization is needed + m_firstEvent = parameters.getParameter("firstEvent"); + m_lastEvent = parameters.getParameter("lastEvent"); + m_action = parameters.getParameter("action"); + m_evtCount = 0; } - -Profiler::~Profiler() -{ - - // do anything here that needs to be done at desctruction time - // (e.g. close files, deallocate resources etc.) - +Profiler::~Profiler() { + // do anything here that needs to be done at desctruction time + // (e.g. close files, deallocate resources etc.) } - // // member functions // @@ -92,45 +82,33 @@ Profiler::~Profiler() // ------------ method called to for each event ------------ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-but-set-variable" -void -Profiler::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) -{ -m_evtCount++; -if(m_evtCount >= m_firstEvent && (m_evtCount <= m_lastEvent || m_lastEvent == -1)) - { - switch (m_action) - { - case 0: - CALLGRIND_STOP_INSTRUMENTATION; - cout << "Stop Instr" << endl; - break; - case 1: - CALLGRIND_START_INSTRUMENTATION; - CALLGRIND_DUMP_STATS; - cout << "Start Instr" << endl; - break; - case 2: - CALLGRIND_DUMP_STATS; - cout << "Dump stat" << endl; - break; - +void Profiler::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) { + m_evtCount++; + if (m_evtCount >= m_firstEvent && (m_evtCount <= m_lastEvent || m_lastEvent == -1)) { + switch (m_action) { + case 0: + CALLGRIND_STOP_INSTRUMENTATION; + cout << "Stop Instr" << endl; + break; + case 1: + CALLGRIND_START_INSTRUMENTATION; + CALLGRIND_DUMP_STATS; + cout << "Start Instr" << endl; + break; + case 2: + CALLGRIND_DUMP_STATS; + cout << "Dump stat" << endl; + break; + } } - - } } #pragma GCC diagnostic pop - // ------------ method called once each job just before starting event loop ------------ -void -Profiler::beginJob() -{ -} +void Profiler::beginJob() {} // ------------ method called once each job just after ending the event loop ------------ -void -Profiler::endJob() { -} +void Profiler::endJob() {} //define this as a plug-in DEFINE_FWK_MODULE(Profiler); diff --git a/PerfTools/Callgrind/plugins/ProfilerAnalyzer.cc b/PerfTools/Callgrind/plugins/ProfilerAnalyzer.cc index 06c90a5be5154..19b9d4bc828e9 100644 --- a/PerfTools/Callgrind/plugins/ProfilerAnalyzer.cc +++ b/PerfTools/Callgrind/plugins/ProfilerAnalyzer.cc @@ -2,7 +2,7 @@ // // Package: PerfTools/Callgrind // Class: ProfilerAnalyzer -// +// /**\class ProfilerAnalyzer ProfilerAnalyzer.cc PerfTools/Callgrind/plugins/ProfilerAnalyzer.cc Description: an Module that either start or stop profiling @@ -15,15 +15,12 @@ \author Vincenzo Innocente */ -// +// // Original Author: Andrea Rizzi // Created: Thu Jan 18 10:34:18 CET 2007 - - // system include files - // user include files #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/EDAnalyzer.h" @@ -41,38 +38,30 @@ class ProfilerAnalyzer : public edm::EDAnalyzer { explicit ProfilerAnalyzer(const edm::ParameterSet&); ~ProfilerAnalyzer() override; - private: void beginJob() override; - void analyze(const edm::Event&, const edm::EventSetup&) override =0; - void endJob() override ; - + void analyze(const edm::Event&, const edm::EventSetup&) override = 0; + void endJob() override; }; class StartProfilerAnalyzer : public ProfilerAnalyzer { public: - explicit StartProfilerAnalyzer(const edm::ParameterSet & pset) : - ProfilerAnalyzer(pset) {} - ~StartProfilerAnalyzer() override{} - + explicit StartProfilerAnalyzer(const edm::ParameterSet& pset) : ProfilerAnalyzer(pset) {} + ~StartProfilerAnalyzer() override {} private: void analyze(const edm::Event&, const edm::EventSetup&) override; - }; class StopProfilerAnalyzer : public ProfilerAnalyzer { public: - explicit StopProfilerAnalyzer(const edm::ParameterSet & pset) : - ProfilerAnalyzer(pset) {} - ~StopProfilerAnalyzer() override{} + explicit StopProfilerAnalyzer(const edm::ParameterSet& pset) : ProfilerAnalyzer(pset) {} + ~StopProfilerAnalyzer() override {} private: void analyze(const edm::Event&, const edm::EventSetup&) override; - }; - // // constants, enums and typedefs // @@ -84,47 +73,30 @@ class StopProfilerAnalyzer : public ProfilerAnalyzer { // // constructors and destructor // -ProfilerAnalyzer::ProfilerAnalyzer(const edm::ParameterSet&) -{ -} - - -ProfilerAnalyzer::~ProfilerAnalyzer() -{ - - // do anything here that needs to be done at desctruction time - // (e.g. close files, deallocate resources etc.) +ProfilerAnalyzer::ProfilerAnalyzer(const edm::ParameterSet&) {} +ProfilerAnalyzer::~ProfilerAnalyzer() { + // do anything here that needs to be done at desctruction time + // (e.g. close files, deallocate resources etc.) } - // // member functions // // ------------ method called to for each event ------------ -void -StartProfilerAnalyzer::analyze(const edm::Event&, const edm::EventSetup&) -{ +void StartProfilerAnalyzer::analyze(const edm::Event&, const edm::EventSetup&) { edm::Service()->startInstrumentation(); } -void -StopProfilerAnalyzer::analyze(const edm::Event&, const edm::EventSetup&) -{ +void StopProfilerAnalyzer::analyze(const edm::Event&, const edm::EventSetup&) { edm::Service()->stopInstrumentation(); } - // ------------ method called once each job just before starting event loop ------------ -void -ProfilerAnalyzer::beginJob() -{ -} +void ProfilerAnalyzer::beginJob() {} // ------------ method called once each job just after ending the event loop ------------ -void -ProfilerAnalyzer::endJob() { -} +void ProfilerAnalyzer::endJob() {} //define this as a plug-in DEFINE_FWK_MODULE(StartProfilerAnalyzer); diff --git a/PerfTools/Callgrind/plugins/ProfilerServicePlugin.cc b/PerfTools/Callgrind/plugins/ProfilerServicePlugin.cc index f97243294e72f..c8a34179e3428 100644 --- a/PerfTools/Callgrind/plugins/ProfilerServicePlugin.cc +++ b/PerfTools/Callgrind/plugins/ProfilerServicePlugin.cc @@ -1,3 +1,3 @@ #include "PerfTools/Callgrind/interface/ProfilerService.h" #include "FWCore/ServiceRegistry/interface/ServiceMaker.h" -DEFINE_FWK_SERVICE (ProfilerService); +DEFINE_FWK_SERVICE(ProfilerService); diff --git a/PerfTools/Callgrind/src/ProfilerService.cc b/PerfTools/Callgrind/src/ProfilerService.cc index 826f8e42b5763..2ea2c1128010f 100644 --- a/PerfTools/Callgrind/src/ProfilerService.cc +++ b/PerfTools/Callgrind/src/ProfilerService.cc @@ -9,91 +9,94 @@ #include "valgrind/callgrind.h" -ProfilerService::ProfilerService(edm::ParameterSet const& pset, - edm::ActivityRegistry& activity) : - - m_firstEvent(pset.getUntrackedParameter("firstEvent",0 )), - m_lastEvent(pset.getUntrackedParameter("lastEvent",std::numeric_limits::max())), - m_dumpInterval(pset.getUntrackedParameter("dumpInterval",100)), - m_paths(pset.getUntrackedParameter >("paths",std::vector() )), - m_excludedPaths(pset.getUntrackedParameter >("excludePaths",std::vector() )), - m_allPaths(false), - m_evtCount(0), - m_counts(0), - m_doEvent(false), - m_active(0), - m_paused(false) { +ProfilerService::ProfilerService(edm::ParameterSet const& pset, edm::ActivityRegistry& activity) + : + + m_firstEvent(pset.getUntrackedParameter("firstEvent", 0)), + m_lastEvent(pset.getUntrackedParameter("lastEvent", std::numeric_limits::max())), + m_dumpInterval(pset.getUntrackedParameter("dumpInterval", 100)), + m_paths(pset.getUntrackedParameter >("paths", std::vector())), + m_excludedPaths( + pset.getUntrackedParameter >("excludePaths", std::vector())), + m_allPaths(false), + m_evtCount(0), + m_counts(0), + m_doEvent(false), + m_active(0), + m_paused(false) { static std::string const allPaths("ALL"); - m_allPaths = std::find(m_paths.begin(),m_paths.end(),allPaths) != m_paths.end(); - + m_allPaths = std::find(m_paths.begin(), m_paths.end(), allPaths) != m_paths.end(); + // either FullEvent or selected path static std::string const fullEvent("FullEvent"); - if (std::find(m_paths.begin(),m_paths.end(),fullEvent) != m_paths.end()) - activity.watchPostSourceEvent(this,&ProfilerService::preSourceI); + if (std::find(m_paths.begin(), m_paths.end(), fullEvent) != m_paths.end()) + activity.watchPostSourceEvent(this, &ProfilerService::preSourceI); else { - activity.watchPreEvent(this,&ProfilerService::beginEventI); - activity.watchPostEvent(this,&ProfilerService::endEventI); - activity.watchPrePathEvent(this,&ProfilerService::beginPathI); - activity.watchPostPathEvent(this,&ProfilerService::endPathI); + activity.watchPreEvent(this, &ProfilerService::beginEventI); + activity.watchPostEvent(this, &ProfilerService::endEventI); + activity.watchPrePathEvent(this, &ProfilerService::beginPathI); + activity.watchPostPathEvent(this, &ProfilerService::endPathI); } } -ProfilerService::~ProfilerService(){ - dumpStat(); -} +ProfilerService::~ProfilerService() { dumpStat(); } #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-but-set-variable" -bool ProfilerService::startInstrumentation(){ +bool ProfilerService::startInstrumentation() { // FIXME here or in client? - if (!doEvent()) return false; - + if (!doEvent()) + return false; - if (m_active==0) { + if (m_active == 0) { CALLGRIND_START_INSTRUMENTATION; - if (m_counts%m_dumpInterval==0) dumpStat(); + if (m_counts % m_dumpInterval == 0) + dumpStat(); ++m_counts; } // support nested start/stop ++m_active; - return m_active==1; + return m_active == 1; } bool ProfilerService::stopInstrumentation() { - if (m_active==0) return false; + if (m_active == 0) + return false; --m_active; - if (m_active==0) + if (m_active == 0) CALLGRIND_STOP_INSTRUMENTATION; - return m_active==0; + return m_active == 0; } bool ProfilerService::forceStopInstrumentation() { - if (m_active==0) return false; + if (m_active == 0) + return false; // FIXME report something if appens; CALLGRIND_STOP_INSTRUMENTATION; - m_active=0; + m_active = 0; return true; } bool ProfilerService::pauseInstrumentation() { - if (m_active==0) return false; - CALLGRIND_STOP_INSTRUMENTATION; - m_paused=true; - return true; -} + if (m_active == 0) + return false; + CALLGRIND_STOP_INSTRUMENTATION; + m_paused = true; + return true; +} bool ProfilerService::resumeInstrumentation() { - if (m_active==0 || (!m_paused)) return false; + if (m_active == 0 || (!m_paused)) + return false; CALLGRIND_START_INSTRUMENTATION; - if (m_counts%m_dumpInterval==0) dumpStat(); + if (m_counts % m_dumpInterval == 0) + dumpStat(); ++m_counts; - m_paused=false; + m_paused = false; return true; } -void ProfilerService::dumpStat() const { - CALLGRIND_DUMP_STATS; -} +void ProfilerService::dumpStat() const { CALLGRIND_DUMP_STATS; } #pragma GCC diagnostic pop void ProfilerService::newEvent() { @@ -101,12 +104,11 @@ void ProfilerService::newEvent() { m_doEvent = m_evtCount >= m_firstEvent && m_evtCount <= m_lastEvent; } - void ProfilerService::fullEvent() { newEvent(); - if(m_doEvent&&m_active==0) + if (m_doEvent && m_active == 0) startInstrumentation(); - if ( (!m_doEvent) && m_active!=0) { + if ((!m_doEvent) && m_active != 0) { stopInstrumentation(); // force, a nested instrumentation may fail to close in presence of filters forceStopInstrumentation(); @@ -114,38 +116,40 @@ void ProfilerService::fullEvent() { } } -void ProfilerService::beginEvent() { +void ProfilerService::beginEvent() { newEvent(); // static std::string const fullEvent("FullEvent"); // if (std::find(m_paths.begin(),m_paths.end(),fullEvent) != m_paths.end()) - if (m_allPaths) + if (m_allPaths) startInstrumentation(); } -void ProfilerService::endEvent() { +void ProfilerService::endEvent() { stopInstrumentation(); // force, a nested instrumentation may fail to close in presence of filters forceStopInstrumentation(); } -void ProfilerService::beginPath(std::string const& path) { - if (!doEvent()) return; +void ProfilerService::beginPath(std::string const& path) { + if (!doEvent()) + return; // assume less than 5-6 path to instrument or to exclude - if (std::find(m_excludedPaths.begin(),m_excludedPaths.end(),path) != m_excludedPaths.end()) { + if (std::find(m_excludedPaths.begin(), m_excludedPaths.end(), path) != m_excludedPaths.end()) { pauseInstrumentation(); - return; + return; } - if (std::find(m_paths.begin(),m_paths.end(),path) == m_paths.end()) return; - m_activePath=path; + if (std::find(m_paths.begin(), m_paths.end(), path) == m_paths.end()) + return; + m_activePath = path; startInstrumentation(); } -void ProfilerService::endPath(std::string const& path) { +void ProfilerService::endPath(std::string const& path) { resumeInstrumentation(); - if (m_activePath==path) { + if (m_activePath == path) { stopInstrumentation(); m_activePath.clear(); } - // do to force, a nested instrumentation may fail to close in presence of filters + // do to force, a nested instrumentation may fail to close in presence of filters // forceStopInstrumentation(); } diff --git a/PerfTools/Callgrind/test/ProfilerServiceTest.cppunit.cc b/PerfTools/Callgrind/test/ProfilerServiceTest.cppunit.cc index 51f5e5f97f783..57ec54008a8c5 100644 --- a/PerfTools/Callgrind/test/ProfilerServiceTest.cppunit.cc +++ b/PerfTools/Callgrind/test/ProfilerServiceTest.cppunit.cc @@ -10,29 +10,30 @@ using namespace boost::assign; #include #include -#include -#include -#include - +#include +#include +#include #include namespace { std::string gS; double gD; - void doSomething(std::string const & name) { + void doSomething(std::string const& name) { static std::string const local("p1"); - if (name==local) gS=name; + if (name == local) + gS = name; if (!name.empty()) - gD += std::sqrt( double(name[0])); + gD += std::sqrt(double(name[0])); } - void doSomethingElse(std::string const & name) { + void doSomethingElse(std::string const& name) { static std::string const local("p1"); - if (name==local) gS=name; + if (name == local) + gS = name; if (!name.empty()) - gD += std::sqrt( double(name[0])); + gD += std::sqrt(double(name[0])); } -} +} // namespace namespace test { class TestProfilerService : public CppUnit::TestFixture { @@ -47,7 +48,7 @@ namespace test { // CPPUNIT_TEST(check_AllPaths); CPPUNIT_TEST(check_Nesting); CPPUNIT_TEST_SUITE_END(); - + public: TestProfilerService(); void setUp(); @@ -62,371 +63,360 @@ namespace test { void check_AllPaths(); void check_Nesting(); }; - + CPPUNIT_TEST_SUITE_REGISTRATION(TestProfilerService); - - TestProfilerService::TestProfilerService(){} - + + TestProfilerService::TestProfilerService() {} + void TestProfilerService::setUp() {} - + void TestProfilerService::tearDown() {} - + void TestProfilerService::check_constr() { edm::ParameterSet pset; edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - CPPUNIT_ASSERT(ps.m_firstEvent==0); - CPPUNIT_ASSERT(ps.m_lastEvent==std::numeric_limits::max()); - CPPUNIT_ASSERT(ps.m_dumpInterval==100); + ProfilerService ps(pset, activity); + CPPUNIT_ASSERT(ps.m_firstEvent == 0); + CPPUNIT_ASSERT(ps.m_lastEvent == std::numeric_limits::max()); + CPPUNIT_ASSERT(ps.m_dumpInterval == 100); CPPUNIT_ASSERT(ps.m_excludedPaths.empty()); CPPUNIT_ASSERT(ps.m_paths.empty()); } - + void TestProfilerService::check_config() { - int fe=2; - int le=10; - int di=5; - std::vector paths; - paths += "p1","p2","p3"; - std::vector ep; - ep += "e1","e2","e3"; + int fe = 2; + int le = 10; + int di = 5; + std::vector paths; + paths += "p1", "p2", "p3"; + std::vector ep; + ep += "e1", "e2", "e3"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter("dumpInterval",di); - pset.addUntrackedParameter >("paths",paths); - pset.addUntrackedParameter >("excludePaths",ep); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter("dumpInterval", di); + pset.addUntrackedParameter >("paths", paths); + pset.addUntrackedParameter >("excludePaths", ep); edm::ActivityRegistry activity; { - ProfilerService ps(pset,activity); - CPPUNIT_ASSERT(ps.m_firstEvent==fe); - CPPUNIT_ASSERT(ps.m_lastEvent==le); - CPPUNIT_ASSERT(ps.m_dumpInterval==di); - CPPUNIT_ASSERT(ps.m_excludedPaths==ep); - CPPUNIT_ASSERT(ps.m_paths==paths); + ProfilerService ps(pset, activity); + CPPUNIT_ASSERT(ps.m_firstEvent == fe); + CPPUNIT_ASSERT(ps.m_lastEvent == le); + CPPUNIT_ASSERT(ps.m_dumpInterval == di); + CPPUNIT_ASSERT(ps.m_excludedPaths == ep); + CPPUNIT_ASSERT(ps.m_paths == paths); CPPUNIT_ASSERT(!ps.m_allPaths); } paths += "ALL"; - pset.addUntrackedParameter >("paths",paths); + pset.addUntrackedParameter >("paths", paths); { - ProfilerService ps(pset,activity); + ProfilerService ps(pset, activity); CPPUNIT_ASSERT(ps.m_allPaths); } - } - + void TestProfilerService::check_Instrumentation() { - int fe=2; - int le=10; - int di=5; + int fe = 2; + int le = 10; + int di = 5; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter("dumpInterval",di); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter("dumpInterval", di); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); + ProfilerService ps(pset, activity); ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(!ps.doEvent()); - CPPUNIT_ASSERT(ps.m_counts==0); + CPPUNIT_ASSERT(ps.m_counts == 0); CPPUNIT_ASSERT(!ps.startInstrumentation()); doSomethingElse("bha"); CPPUNIT_ASSERT(!ps.stopInstrumentation()); CPPUNIT_ASSERT(!ps.forceStopInstrumentation()); - + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - CPPUNIT_ASSERT(ps.m_counts==0); + CPPUNIT_ASSERT(ps.m_active == 0); + CPPUNIT_ASSERT(ps.m_counts == 0); CPPUNIT_ASSERT(ps.doEvent()); CPPUNIT_ASSERT(ps.startInstrumentation()); - CPPUNIT_ASSERT(ps.m_counts==1); + CPPUNIT_ASSERT(ps.m_counts == 1); doSomething("bha"); CPPUNIT_ASSERT(ps.stopInstrumentation()); CPPUNIT_ASSERT(!ps.forceStopInstrumentation()); - + CPPUNIT_ASSERT(ps.startInstrumentation()); CPPUNIT_ASSERT(!ps.startInstrumentation()); doSomething("bha"); CPPUNIT_ASSERT(!ps.stopInstrumentation()); CPPUNIT_ASSERT(ps.stopInstrumentation()); CPPUNIT_ASSERT(!ps.stopInstrumentation()); - - - for(int i=2;i<10;i++) { + + for (int i = 2; i < 10; i++) { ps.beginEvent(); doSomething("bha"); } - CPPUNIT_ASSERT(ps.m_evtCount==10); + CPPUNIT_ASSERT(ps.m_evtCount == 10); CPPUNIT_ASSERT(ps.doEvent()); CPPUNIT_ASSERT(ps.startInstrumentation()); - CPPUNIT_ASSERT(ps.m_counts==3); + CPPUNIT_ASSERT(ps.m_counts == 3); doSomething("bha"); CPPUNIT_ASSERT(ps.stopInstrumentation()); ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_evtCount==11); + CPPUNIT_ASSERT(ps.m_evtCount == 11); CPPUNIT_ASSERT(!ps.doEvent()); CPPUNIT_ASSERT(!ps.startInstrumentation()); doSomethingElse("bha"); CPPUNIT_ASSERT(!ps.stopInstrumentation()); - CPPUNIT_ASSERT(ps.m_counts==3); - + CPPUNIT_ASSERT(ps.m_counts == 3); } - + void TestProfilerService::check_FullEvent() { - int fe=2; - int le=10; - std::vector paths; + int fe = 2; + int le = 10; + std::vector paths; paths += "FullEvent"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter >("paths",paths); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter >("paths", paths); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - + ProfilerService ps(pset, activity); + ps.fullEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(!ps.doEvent()); doSomethingElse("bha"); - + ps.fullEvent(); - CPPUNIT_ASSERT(ps.m_active==1); + CPPUNIT_ASSERT(ps.m_active == 1); CPPUNIT_ASSERT(ps.doEvent()); doSomething("bha"); - for(int i=2;i<10;i++) { + for (int i = 2; i < 10; i++) { ps.fullEvent(); doSomething("bha"); } - CPPUNIT_ASSERT(ps.m_evtCount==10); - CPPUNIT_ASSERT(ps.doEvent()); // who cares? - + CPPUNIT_ASSERT(ps.m_evtCount == 10); + CPPUNIT_ASSERT(ps.doEvent()); // who cares? + ps.fullEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(!ps.doEvent()); doSomethingElse("bha"); ps.fullEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - - + CPPUNIT_ASSERT(ps.m_active == 0); } - + // now used ALL paths.... void TestProfilerService::check_Event() { - int fe=2; - int le=10; - std::vector paths; + int fe = 2; + int le = 10; + std::vector paths; paths += "ALL"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter >("paths",paths); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter >("paths", paths); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - + ProfilerService ps(pset, activity); + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(!ps.doEvent()); doSomethingElse("bha"); ps.endEvent(); - + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==1); + CPPUNIT_ASSERT(ps.m_active == 1); CPPUNIT_ASSERT(ps.doEvent()); doSomething("bha"); ps.endEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - for(int i=2;i<10;i++) { + CPPUNIT_ASSERT(ps.m_active == 0); + for (int i = 2; i < 10; i++) { ps.beginEvent(); doSomething("bha"); ps.endEvent(); } - CPPUNIT_ASSERT(ps.m_evtCount==10); - CPPUNIT_ASSERT(ps.doEvent()); // who cares? - + CPPUNIT_ASSERT(ps.m_evtCount == 10); + CPPUNIT_ASSERT(ps.doEvent()); // who cares? + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(!ps.doEvent()); doSomethingElse("bha"); ps.endEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - - + CPPUNIT_ASSERT(ps.m_active == 0); } - + struct CheckPaths { - CheckPaths(ProfilerService & ips, std::vector const & iselpaths, int ibase=0, bool iexc=false ) : - ps(ips), selpaths(iselpaths),base(ibase), exc(iexc), done(0) {} - ProfilerService & ps; - std::vector const & selpaths; + CheckPaths(ProfilerService& ips, std::vector const& iselpaths, int ibase = 0, bool iexc = false) + : ps(ips), selpaths(iselpaths), base(ibase), exc(iexc), done(0) {} + ProfilerService& ps; + std::vector const& selpaths; int base; bool exc; - + mutable int done; - - void operator()(std::string const & path) const { - bool found = std::find(selpaths.begin(),selpaths.end(),path)!= selpaths.end(); - bool ok = ps.doEvent() && ( exc ? !found : found); + + void operator()(std::string const& path) const { + bool found = std::find(selpaths.begin(), selpaths.end(), path) != selpaths.end(); + bool ok = ps.doEvent() && (exc ? !found : found); noselPath(true); ps.beginPath(path); - if (ok) selPath(path); - else noselPath(!ps.doEvent()); + if (ok) + selPath(path); + else + noselPath(!ps.doEvent()); ps.endPath(path); noselPath(true); } - - void selPath(const std::string & path) const { - CPPUNIT_ASSERT(ps.m_active==(base+ (exc? 0:1))); - CPPUNIT_ASSERT(exc||ps.m_activePath==path); + + void selPath(const std::string& path) const { + CPPUNIT_ASSERT(ps.m_active == (base + (exc ? 0 : 1))); + CPPUNIT_ASSERT(exc || ps.m_activePath == path); CPPUNIT_ASSERT(!ps.m_paused); ++done; doSomething(path); } - - void noselPath(bool f=false) const { - CPPUNIT_ASSERT(ps.m_active==base); - CPPUNIT_ASSERT(ps.m_activePath.empty()); - CPPUNIT_ASSERT(f||(!exc)||ps.m_paused); - CPPUNIT_ASSERT(!(f&&ps.m_paused)); - doSomethingElse("else"); + + void noselPath(bool f = false) const { + CPPUNIT_ASSERT(ps.m_active == base); + CPPUNIT_ASSERT(ps.m_activePath.empty()); + CPPUNIT_ASSERT(f || (!exc) || ps.m_paused); + CPPUNIT_ASSERT(!(f && ps.m_paused)); + doSomethingElse("else"); } - }; - + void TestProfilerService::check_Path() { - int fe=2; - int le=10; - std::vector paths; - paths += "p1","p2","p3"; + int fe = 2; + int le = 10; + std::vector paths; + paths += "p1", "p2", "p3"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter >("paths",paths); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter >("paths", paths); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - - std::vector allPaths; - allPaths += "p1","p21","p22","p3"; - CheckPaths cp(ps,paths); - CPPUNIT_ASSERT(std::find(paths.begin(),paths.end(),allPaths[0]) != paths.end()); - CPPUNIT_ASSERT(std::find(paths.begin(),paths.end(),allPaths[1]) == paths.end()); - CPPUNIT_ASSERT(std::find(paths.begin(),paths.end(),allPaths[2]) == paths.end()); - CPPUNIT_ASSERT(std::find(paths.begin(),paths.end(),allPaths[3]) != paths.end()); - + ProfilerService ps(pset, activity); + + std::vector allPaths; + allPaths += "p1", "p21", "p22", "p3"; + CheckPaths cp(ps, paths); + CPPUNIT_ASSERT(std::find(paths.begin(), paths.end(), allPaths[0]) != paths.end()); + CPPUNIT_ASSERT(std::find(paths.begin(), paths.end(), allPaths[1]) == paths.end()); + CPPUNIT_ASSERT(std::find(paths.begin(), paths.end(), allPaths[2]) == paths.end()); + CPPUNIT_ASSERT(std::find(paths.begin(), paths.end(), allPaths[3]) != paths.end()); + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - CPPUNIT_ASSERT(!ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp).done==0); + CPPUNIT_ASSERT(ps.m_active == 0); + CPPUNIT_ASSERT(!ps.doEvent()); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp).done == 0); ps.endEvent(); - + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp).done==2); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp).done == 2); ps.endEvent(); - } void TestProfilerService::check_ExcludedPath() { - int fe=2; - int le=10; - std::vector paths; + int fe = 2; + int le = 10; + std::vector paths; paths += "ALL"; - std::vector expaths; + std::vector expaths; expaths += "p21"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter >("paths",paths); - pset.addUntrackedParameter >("excludePaths",expaths); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter >("paths", paths); + pset.addUntrackedParameter >("excludePaths", expaths); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - - std::vector allPaths; - allPaths += "p1","p21","p22","p3"; - CheckPaths cp0(ps,expaths,0,true); - CheckPaths cp1(ps,expaths,1,true); - CPPUNIT_ASSERT(std::find(expaths.begin(),expaths.end(),allPaths[0]) == expaths.end()); - CPPUNIT_ASSERT(std::find(expaths.begin(),expaths.end(),allPaths[1]) != expaths.end()); - CPPUNIT_ASSERT(std::find(expaths.begin(),expaths.end(),allPaths[2]) == expaths.end()); - CPPUNIT_ASSERT(std::find(expaths.begin(),expaths.end(),allPaths[3]) == expaths.end()); - + ProfilerService ps(pset, activity); + + std::vector allPaths; + allPaths += "p1", "p21", "p22", "p3"; + CheckPaths cp0(ps, expaths, 0, true); + CheckPaths cp1(ps, expaths, 1, true); + CPPUNIT_ASSERT(std::find(expaths.begin(), expaths.end(), allPaths[0]) == expaths.end()); + CPPUNIT_ASSERT(std::find(expaths.begin(), expaths.end(), allPaths[1]) != expaths.end()); + CPPUNIT_ASSERT(std::find(expaths.begin(), expaths.end(), allPaths[2]) == expaths.end()); + CPPUNIT_ASSERT(std::find(expaths.begin(), expaths.end(), allPaths[3]) == expaths.end()); + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - CPPUNIT_ASSERT(!ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp0).done==0); + CPPUNIT_ASSERT(ps.m_active == 0); + CPPUNIT_ASSERT(!ps.doEvent()); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp0).done == 0); ps.endEvent(); - + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active!=0); + CPPUNIT_ASSERT(ps.m_active != 0); CPPUNIT_ASSERT(ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp1).done==3); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp1).done == 3); ps.endEvent(); - } - + // same as nesting, removed.... void TestProfilerService::check_AllPaths() { - int fe=2; - int le=10; - std::vector paths; - paths += "ALL","p2","p3"; + int fe = 2; + int le = 10; + std::vector paths; + paths += "ALL", "p2", "p3"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter >("paths",paths); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter >("paths", paths); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - - std::vector allPaths; - allPaths += "p1","p21","p22","p3"; - CheckPaths cp(ps,paths); - + ProfilerService ps(pset, activity); + + std::vector allPaths; + allPaths += "p1", "p21", "p22", "p3"; + CheckPaths cp(ps, paths); + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - CPPUNIT_ASSERT(!ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp).done==0); + CPPUNIT_ASSERT(ps.m_active == 0); + CPPUNIT_ASSERT(!ps.doEvent()); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp).done == 0); ps.endEvent(); - + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==1); + CPPUNIT_ASSERT(ps.m_active == 1); CPPUNIT_ASSERT(ps.doEvent()); // CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp).done==4); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp).done==1); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp).done == 1); ps.endEvent(); - } - + void TestProfilerService::check_Nesting() { - int fe=2; - int le=10; - std::vector paths; - paths += "ALL", "p1","p2","p3"; + int fe = 2; + int le = 10; + std::vector paths; + paths += "ALL", "p1", "p2", "p3"; edm::ParameterSet pset; - pset.addUntrackedParameter("firstEvent",fe); - pset.addUntrackedParameter("lastEvent",le); - pset.addUntrackedParameter >("paths",paths); + pset.addUntrackedParameter("firstEvent", fe); + pset.addUntrackedParameter("lastEvent", le); + pset.addUntrackedParameter >("paths", paths); edm::ActivityRegistry activity; - ProfilerService ps(pset,activity); - - std::vector allPaths; - allPaths += "p1","p21","p22","p3"; - - - CheckPaths cp0(ps,paths,0); - CheckPaths cp1(ps,paths,1); - + ProfilerService ps(pset, activity); + + std::vector allPaths; + allPaths += "p1", "p21", "p22", "p3"; + + CheckPaths cp0(ps, paths, 0); + CheckPaths cp1(ps, paths, 1); + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==0); + CPPUNIT_ASSERT(ps.m_active == 0); CPPUNIT_ASSERT(!ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp0).done==0); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp0).done == 0); ps.endEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - + CPPUNIT_ASSERT(ps.m_active == 0); + ps.beginEvent(); - CPPUNIT_ASSERT(ps.m_active==1); + CPPUNIT_ASSERT(ps.m_active == 1); CPPUNIT_ASSERT(ps.doEvent()); - CPPUNIT_ASSERT(std::for_each(allPaths.begin(),allPaths.end(),cp1).done==2); + CPPUNIT_ASSERT(std::for_each(allPaths.begin(), allPaths.end(), cp1).done == 2); ps.endEvent(); - CPPUNIT_ASSERT(ps.m_active==0); - + CPPUNIT_ASSERT(ps.m_active == 0); } -} +} // namespace test diff --git a/PerfTools/EdmEvent/bin/edmEventSize.cpp b/PerfTools/EdmEvent/bin/edmEventSize.cpp old mode 100755 new mode 100644 index 773c44f7c182c..e34743c5cecf8 --- a/PerfTools/EdmEvent/bin/edmEventSize.cpp +++ b/PerfTools/EdmEvent/bin/edmEventSize.cpp @@ -5,7 +5,6 @@ #include "PerfTools/EdmEvent/interface/EdmEventSize.h" - #include #include #include @@ -16,105 +15,101 @@ #include #include "FWCore/FWLite/interface/FWLiteEnabler.h" -static const char * const kHelpOpt = "help"; -static const char * const kHelpCommandOpt = "help,h"; -static const char * const kDataFileOpt = "data-file"; -static const char * const kDataFileCommandOpt = "data-file,d"; -static const char * const kTreeNameOpt = "tree-name"; -static const char * const kTreeNameCommandOpt = "tree-name,n"; -static const char * const kOutputOpt = "output"; -static const char * const kOutputCommandOpt = "output,o"; -static const char * const kAutoLoadOpt ="auto-loader"; -static const char * const kAutoLoadCommandOpt ="auto-loader,a"; -static const char * const kPlotOpt ="plot"; -static const char * const kPlotCommandOpt ="plot,p"; -static const char * const kSavePlotOpt ="save-plot"; -static const char * const kSavePlotCommandOpt ="save-plot,s"; -static const char * const kPlotTopOpt ="plot-top"; -static const char * const kPlotTopCommandOpt ="plot-top,t"; -static const char * const kVerboseOpt = "verbose"; -static const char * const kVerboseCommandOpt = "verbose,v"; -static const char * const kAlphabeticOrderOpt ="alphabetic-order"; -static const char * const kAlphabeticOrderCommandOpt ="alphabetic-order,A"; -static const char * const kFormatNamesOpt ="format-names"; -static const char * const kFormatNamesCommandOpt ="format-names,F"; - -int main( int argc, char * argv[] ) { +static const char* const kHelpOpt = "help"; +static const char* const kHelpCommandOpt = "help,h"; +static const char* const kDataFileOpt = "data-file"; +static const char* const kDataFileCommandOpt = "data-file,d"; +static const char* const kTreeNameOpt = "tree-name"; +static const char* const kTreeNameCommandOpt = "tree-name,n"; +static const char* const kOutputOpt = "output"; +static const char* const kOutputCommandOpt = "output,o"; +static const char* const kAutoLoadOpt = "auto-loader"; +static const char* const kAutoLoadCommandOpt = "auto-loader,a"; +static const char* const kPlotOpt = "plot"; +static const char* const kPlotCommandOpt = "plot,p"; +static const char* const kSavePlotOpt = "save-plot"; +static const char* const kSavePlotCommandOpt = "save-plot,s"; +static const char* const kPlotTopOpt = "plot-top"; +static const char* const kPlotTopCommandOpt = "plot-top,t"; +static const char* const kVerboseOpt = "verbose"; +static const char* const kVerboseCommandOpt = "verbose,v"; +static const char* const kAlphabeticOrderOpt = "alphabetic-order"; +static const char* const kAlphabeticOrderCommandOpt = "alphabetic-order,A"; +static const char* const kFormatNamesOpt = "format-names"; +static const char* const kFormatNamesCommandOpt = "format-names,F"; + +int main(int argc, char* argv[]) { using namespace boost::program_options; using namespace std; - string programName( argv[ 0 ] ); - string descString( programName ); + string programName(argv[0]); + string descString(programName); descString += " [options] "; descString += "data_file \nAllowed options"; - options_description desc( descString ); - - desc.add_options() - ( kHelpCommandOpt, "produce help message" ) - ( kAutoLoadCommandOpt, "automatic library loading (avoid root warnings)" ) - ( kDataFileCommandOpt, value(), "data file" ) - ( kTreeNameCommandOpt, value(), "tree name (default \"Events\")" ) - ( kOutputCommandOpt, value(), "output file" ) - ( kAlphabeticOrderCommandOpt, "sort by alphabetic order (default: sort by size)" ) - ( kFormatNamesCommandOpt, "format product name as \"product:label (type)\" (default: use full branch name)" ) - ( kPlotCommandOpt, value(), "produce a summary plot" ) - ( kPlotTopCommandOpt, value(), "plot only the top size branches" ) - ( kSavePlotCommandOpt, value(), "save plot into root file " ) - ( kVerboseCommandOpt, "verbose printout" ); + options_description desc(descString); + + desc.add_options()(kHelpCommandOpt, "produce help message")(kAutoLoadCommandOpt, + "automatic library loading (avoid root warnings)")( + kDataFileCommandOpt, value(), "data file")( + kTreeNameCommandOpt, value(), "tree name (default \"Events\")")( + kOutputCommandOpt, value(), "output file")(kAlphabeticOrderCommandOpt, + "sort by alphabetic order (default: sort by size)")( + kFormatNamesCommandOpt, "format product name as \"product:label (type)\" (default: use full branch name)")( + kPlotCommandOpt, value(), "produce a summary plot")( + kPlotTopCommandOpt, value(), "plot only the top size branches")( + kSavePlotCommandOpt, value(), "save plot into root file ")(kVerboseCommandOpt, "verbose printout"); positional_options_description p; - p.add( kDataFileOpt, -1 ); + p.add(kDataFileOpt, -1); variables_map vm; try { - store( command_line_parser(argc,argv).options(desc).positional(p).run(), vm ); - notify( vm ); - } catch( const error& ) { + store(command_line_parser(argc, argv).options(desc).positional(p).run(), vm); + notify(vm); + } catch (const error&) { return 7000; } - if( vm.count( kHelpOpt ) ) { - cout << desc <SetBatch(); - if( vm.count( kAutoLoadOpt ) != 0 ) { - gSystem->Load( "libFWCoreFWLite" ); + if (vm.count(kAutoLoadOpt) != 0) { + gSystem->Load("libFWCoreFWLite"); FWLiteEnabler::enable(); - } - else - gErrorIgnoreLevel = kError; - - bool verbose = vm.count( kVerboseOpt ) > 0; + } else + gErrorIgnoreLevel = kError; + bool verbose = vm.count(kVerboseOpt) > 0; std::string fileName = vm[kDataFileOpt].as(); std::string treeName = "Events"; - if ( vm.count( kTreeNameOpt) ) - treeName=vm[kTreeNameOpt].as(); + if (vm.count(kTreeNameOpt)) + treeName = vm[kTreeNameOpt].as(); perftools::EdmEventSize me; - + try { - me.parseFile(fileName,treeName); - } catch(perftools::EdmEventSize::Error const & error) { - std::cerr << programName << ":" << error.descr << std::endl; + me.parseFile(fileName, treeName); + } catch (perftools::EdmEventSize::Error const& error) { + std::cerr << programName << ":" << error.descr << std::endl; return error.code; - } + } - if ( vm.count( kFormatNamesOpt) ) + if (vm.count(kFormatNamesOpt)) me.formatNames(); - if ( vm.count( kAlphabeticOrderOpt ) ) + if (vm.count(kAlphabeticOrderOpt)) me.sortAlpha(); if (verbose) { @@ -123,24 +118,25 @@ int main( int argc, char * argv[] ) { std::cout << std::endl; } - if (vm.count( kOutputOpt )) { + if (vm.count(kOutputOpt)) { std::ofstream of(vm[kOutputOpt].as().c_str()); - me.dump(of); of << std::endl; + me.dump(of); + of << std::endl; } - bool plot = ( vm.count( kPlotOpt ) > 0 ); - bool save = ( vm.count( kSavePlotOpt ) > 0 ); - if (plot||save) { - + bool plot = (vm.count(kPlotOpt) > 0); + bool save = (vm.count(kSavePlotOpt) > 0); + if (plot || save) { std::string plotName; - std::string histName; - if( plot ) plotName = vm[kPlotOpt].as(); - if( save ) histName = vm[kSavePlotOpt].as(); - int top=0; - if( vm.count( kPlotTopOpt ) > 0 ) top = vm[ kPlotTopOpt ].as(); - me.produceHistos(plotName,histName,top); - - + std::string histName; + if (plot) + plotName = vm[kPlotOpt].as(); + if (save) + histName = vm[kSavePlotOpt].as(); + int top = 0; + if (vm.count(kPlotTopOpt) > 0) + top = vm[kPlotTopOpt].as(); + me.produceHistos(plotName, histName, top); } return 0; diff --git a/PerfTools/EdmEvent/interface/EdmEventSize.h b/PerfTools/EdmEvent/interface/EdmEventSize.h index eae2b931239f3..ae4ea9ad5485c 100644 --- a/PerfTools/EdmEvent/interface/EdmEventSize.h +++ b/PerfTools/EdmEvent/interface/EdmEventSize.h @@ -1,9 +1,9 @@ #ifndef PerfTools_EdmEventSize_H #define PerfTools_EdmEventSize_H -#include -#include -#include +#include +#include +#include namespace perftools { @@ -23,24 +23,18 @@ namespace perftools { */ class EdmEventSize { public: - /// generic exception struct Error { - Error(std::string const & idescr, int icode) : - descr(idescr), code(icode){} + Error(std::string const& idescr, int icode) : descr(idescr), code(icode) {} std::string descr; int code; }; /// the information for each branch struct BranchRecord { - BranchRecord() : - compr_size(0.), - uncompr_size(0.) {} - BranchRecord(std::string const & iname, - double compr, double uncompr) : - fullName(iname), name(iname), - compr_size(compr), uncompr_size(uncompr){} + BranchRecord() : compr_size(0.), uncompr_size(0.) {} + BranchRecord(std::string const& iname, double compr, double uncompr) + : fullName(iname), name(iname), compr_size(compr), uncompr_size(uncompr) {} std::string fullName; std::string name; double compr_size; @@ -51,31 +45,30 @@ namespace perftools { /// Constructor EdmEventSize(); - /// Constructor and parse - explicit EdmEventSize(std::string const & fileName, std::string const & treeName="Events"); - + /// Constructor and parse + explicit EdmEventSize(std::string const& fileName, std::string const& treeName = "Events"); + /// read file, compute branch size, sort by size - void parseFile(std::string const & fileName, std::string const & treeName="Events"); + void parseFile(std::string const& fileName, std::string const& treeName = "Events"); /// sort by name void sortAlpha(); - + /// transform Branch names in "formatted" prodcut identifiers void formatNames(); /// dump the ascii table on "co" - void dump(std::ostream & co, bool header=true) const; + void dump(std::ostream& co, bool header = true) const; /// produce histograms and optionally write them in "file" or as "plot" - void produceHistos(std::string const & plot, std::string const & file, int top=0) const; + void produceHistos(std::string const& plot, std::string const& file, int top = 0) const; private: std::string m_fileName; int m_nEvents; Branches m_branches; - }; -} +} // namespace perftools -#endif // PerfTools_EdmEventSize_H +#endif // PerfTools_EdmEventSize_H diff --git a/PerfTools/EdmEvent/src/EdmEventSize.cc b/PerfTools/EdmEvent/src/EdmEventSize.cc index 9ba228a4db63b..436a9bf91c234 100644 --- a/PerfTools/EdmEvent/src/EdmEventSize.cc +++ b/PerfTools/EdmEvent/src/EdmEventSize.cc @@ -26,236 +26,229 @@ namespace { - enum Indices {kUncompressed,kCompressed}; + enum Indices { kUncompressed, kCompressed }; - typedef std::valarray size_type; + typedef std::valarray size_type; - size_type getBasketSize( TBranch *); - - size_type getBasketSize( TObjArray * branches) { - size_type result(static_cast(0),2); + size_type getBasketSize(TBranch*); + + size_type getBasketSize(TObjArray* branches) { + size_type result(static_cast(0), 2); size_t n = branches->GetEntries(); - for( size_t i = 0; i < n; ++ i ) { - TBranch * b = dynamic_cast( branches->At( i ) ); - assert( b != nullptr ); + for (size_t i = 0; i < n; ++i) { + TBranch* b = dynamic_cast(branches->At(i)); + assert(b != nullptr); result += getBasketSize(b); } return result; } - - size_type getBasketSize( TBranch * b) { - size_type result(static_cast(0),2); - if ( b != nullptr ) { - if ( b->GetZipBytes() > 0 ) { - result[kUncompressed] = b->GetTotBytes(); result[kCompressed] = b->GetZipBytes(); + + size_type getBasketSize(TBranch* b) { + size_type result(static_cast(0), 2); + if (b != nullptr) { + if (b->GetZipBytes() > 0) { + result[kUncompressed] = b->GetTotBytes(); + result[kCompressed] = b->GetZipBytes(); } else { - result[kUncompressed] = b->GetTotalSize(); result[kCompressed] = b->GetTotalSize(); + result[kUncompressed] = b->GetTotalSize(); + result[kCompressed] = b->GetTotalSize(); } - result += getBasketSize( b->GetListOfBranches() ); + result += getBasketSize(b->GetListOfBranches()); } return result; } - - size_type getTotalSize( TBranch * br) { - TBufferFile buf( TBuffer::kWrite, 10000 ); - TBranch::Class()->WriteBuffer( buf, br ); + size_type getTotalSize(TBranch* br) { + TBufferFile buf(TBuffer::kWrite, 10000); + TBranch::Class()->WriteBuffer(buf, br); size_type size = getBasketSize(br); - if ( br->GetZipBytes() > 0 ) + if (br->GetZipBytes() > 0) size[kUncompressed] += buf.Length(); return size; } -} +} // namespace namespace perftools { - EdmEventSize::EdmEventSize() : - m_nEvents(0) {} - - EdmEventSize::EdmEventSize(std::string const & fileName, std::string const & treeName ) : - m_nEvents(0) { + EdmEventSize::EdmEventSize() : m_nEvents(0) {} + + EdmEventSize::EdmEventSize(std::string const& fileName, std::string const& treeName) : m_nEvents(0) { parseFile(fileName); } - - void EdmEventSize::parseFile(std::string const & fileName, std::string const & treeName) { + + void EdmEventSize::parseFile(std::string const& fileName, std::string const& treeName) { m_fileName = fileName; m_branches.clear(); - TFile * file = TFile::Open( fileName.c_str() ); - if( file==nullptr || ( !(*file).IsOpen() ) ) - throw Error( "unable to open data file " + fileName, 7002); - - TObject * o = file->Get(treeName.c_str() ); - if ( o == nullptr ) + TFile* file = TFile::Open(fileName.c_str()); + if (file == nullptr || (!(*file).IsOpen())) + throw Error("unable to open data file " + fileName, 7002); + + TObject* o = file->Get(treeName.c_str()); + if (o == nullptr) throw Error("no object \"" + treeName + "\" found in file: " + fileName, 7003); - - TTree * events = dynamic_cast (o); - if ( events == nullptr ) + + TTree* events = dynamic_cast(o); + if (events == nullptr) throw Error("object \"" + treeName + "\" is not a TTree in file: " + fileName, 7004); - + m_nEvents = events->GetEntries(); - if ( m_nEvents == 0 ) + if (m_nEvents == 0) throw Error("tree \"" + treeName + "\" in file " + fileName + " contains no Events", 7005); + TObjArray* branches = events->GetListOfBranches(); + if (branches == nullptr) + throw Error("tree \"" + treeName + "\" in file " + fileName + " contains no branches", 7006); - TObjArray * branches = events->GetListOfBranches(); - if ( branches == nullptr ) - throw Error("tree \"" + treeName+ "\" in file " + fileName + " contains no branches", 7006); - - const size_t n = branches->GetEntries(); + const size_t n = branches->GetEntries(); m_branches.reserve(n); - for( size_t i = 0; i < n; ++i ) { - TBranch * b = dynamic_cast( branches->At( i ) ); - if (b==nullptr) continue; - std::string const name( b->GetName() ); - if ( name == "EventAux" ) continue; + for (size_t i = 0; i < n; ++i) { + TBranch* b = dynamic_cast(branches->At(i)); + if (b == nullptr) + continue; + std::string const name(b->GetName()); + if (name == "EventAux") + continue; size_type s = getTotalSize(b); - m_branches.push_back( BranchRecord(name, double(s[kCompressed])/double(m_nEvents), double(s[kUncompressed])/double(m_nEvents)) ); + m_branches.push_back( + BranchRecord(name, double(s[kCompressed]) / double(m_nEvents), double(s[kUncompressed]) / double(m_nEvents))); } - std::sort(m_branches.begin(),m_branches.end(), - boost::bind(std::greater(), - boost::bind(&BranchRecord::compr_size,_1), - boost::bind(&BranchRecord::compr_size,_2)) - ); - + std::sort(m_branches.begin(), + m_branches.end(), + boost::bind(std::greater(), + boost::bind(&BranchRecord::compr_size, _1), + boost::bind(&BranchRecord::compr_size, _2))); } - - void EdmEventSize::sortAlpha() { - std::sort(m_branches.begin(),m_branches.end(), - boost::bind(std::less(), - boost::bind(&BranchRecord::name,_1), - boost::bind(&BranchRecord::name,_2)) - ); + void EdmEventSize::sortAlpha() { + std::sort( + m_branches.begin(), + m_branches.end(), + boost::bind( + std::less(), boost::bind(&BranchRecord::name, _1), boost::bind(&BranchRecord::name, _2))); } namespace detail { // format as product:label (type) - void shorterName(EdmEventSize::BranchRecord & br) { + void shorterName(EdmEventSize::BranchRecord& br) { size_t b = br.fullName.find('_'); size_t e = br.fullName.rfind('_'); - if (b==e) br.name=br.fullName; + if (b == e) + br.name = br.fullName; else { - // remove type and process - br.name = br.fullName.substr(b+1,e-b-1); - // change label separator in : - e = br.name.rfind('_'); - if (e!=std::string::npos) br.name.replace(e,1,":"); - // add the type name - br.name.append(" ("+br.fullName.substr(0,b)+")"); + // remove type and process + br.name = br.fullName.substr(b + 1, e - b - 1); + // change label separator in : + e = br.name.rfind('_'); + if (e != std::string::npos) + br.name.replace(e, 1, ":"); + // add the type name + br.name.append(" (" + br.fullName.substr(0, b) + ")"); } } - } - - void EdmEventSize::formatNames() { - std::for_each(m_branches.begin(),m_branches.end(), - &detail::shorterName); - } - + } // namespace detail + void EdmEventSize::formatNames() { std::for_each(m_branches.begin(), m_branches.end(), &detail::shorterName); } namespace detail { - void dump(std::ostream& co, EdmEventSize::BranchRecord const & br) { - co << br.name << " " << br.uncompr_size << " " << br.compr_size << "\n"; + void dump(std::ostream& co, EdmEventSize::BranchRecord const& br) { + co << br.name << " " << br.uncompr_size << " " << br.compr_size << "\n"; } - } + } // namespace detail - - void EdmEventSize::dump(std::ostream & co, bool header) const { + void EdmEventSize::dump(std::ostream& co, bool header) const { if (header) { co << "File " << m_fileName << " Events " << m_nEvents << "\n"; - co <<"Branch Name | Average Uncompressed Size (Bytes/Event) | Average Compressed Size (Bytes/Event) \n"; + co << "Branch Name | Average Uncompressed Size (Bytes/Event) | Average Compressed Size (Bytes/Event) \n"; } - std::for_each(m_branches.begin(),m_branches.end(), - boost::bind(detail::dump,boost::ref(co),_1)); + std::for_each(m_branches.begin(), m_branches.end(), boost::bind(detail::dump, boost::ref(co), _1)); } namespace detail { struct Hist { - - explicit Hist(int itop) : - top(itop), - uncompressed( "uncompressed", "branch sizes", top, -0.5, - 0.5 + top ), - compressed( "compressed", "branch sizes", top, -0.5, - 0.5 + top ), - cxAxis(compressed.GetXaxis()), - uxAxis(uncompressed.GetXaxis()), - x(0) {} - - void fill(EdmEventSize::BranchRecord const & br) { - if ( x < top ) { - cxAxis->SetBinLabel( x + 1, br.name.c_str() ); - uxAxis->SetBinLabel( x + 1, br.name.c_str() ); - compressed.Fill( x, br.compr_size ); - uncompressed.Fill( x, br.uncompr_size ); - x++; - } + explicit Hist(int itop) + : top(itop), + uncompressed("uncompressed", "branch sizes", top, -0.5, -0.5 + top), + compressed("compressed", "branch sizes", top, -0.5, -0.5 + top), + cxAxis(compressed.GetXaxis()), + uxAxis(uncompressed.GetXaxis()), + x(0) {} + + void fill(EdmEventSize::BranchRecord const& br) { + if (x < top) { + cxAxis->SetBinLabel(x + 1, br.name.c_str()); + uxAxis->SetBinLabel(x + 1, br.name.c_str()); + compressed.Fill(x, br.compr_size); + uncompressed.Fill(x, br.uncompr_size); + x++; + } } void finalize() { - double mn = std::numeric_limits::max(); - for( int i = 1; i <= top; ++i ) { - double cm = compressed.GetMinimum( i ), um = uncompressed.GetMinimum( i ); - if ( cm > 0 && cm < mn ) mn = cm; - if ( um > 0 && um < mn ) mn = um; - } - mn *= 0.8; - double mx = std::max( compressed.GetMaximum(), uncompressed.GetMaximum() ); - mx *= 1.2; - uncompressed.SetMinimum( mn ); - uncompressed.SetMaximum( mx ); - compressed.SetMinimum( mn ); - // compressed.SetMaximum( mx ); - cxAxis->SetLabelOffset( -0.32 ); - cxAxis->LabelsOption( "v" ); - cxAxis->SetLabelSize( 0.03 ); - uxAxis->SetLabelOffset( -0.32 ); - uxAxis->LabelsOption( "v" ); - uxAxis->SetLabelSize( 0.03 ); - compressed.GetYaxis()->SetTitle( "Bytes" ); - compressed.SetFillColor( kBlue ); - compressed.SetLineWidth( 2 ); - uncompressed.GetYaxis()->SetTitle( "Bytes" ); - uncompressed.SetFillColor( kRed ); - uncompressed.SetLineWidth( 2 ); - + double mn = std::numeric_limits::max(); + for (int i = 1; i <= top; ++i) { + double cm = compressed.GetMinimum(i), um = uncompressed.GetMinimum(i); + if (cm > 0 && cm < mn) + mn = cm; + if (um > 0 && um < mn) + mn = um; + } + mn *= 0.8; + double mx = std::max(compressed.GetMaximum(), uncompressed.GetMaximum()); + mx *= 1.2; + uncompressed.SetMinimum(mn); + uncompressed.SetMaximum(mx); + compressed.SetMinimum(mn); + // compressed.SetMaximum( mx ); + cxAxis->SetLabelOffset(-0.32); + cxAxis->LabelsOption("v"); + cxAxis->SetLabelSize(0.03); + uxAxis->SetLabelOffset(-0.32); + uxAxis->LabelsOption("v"); + uxAxis->SetLabelSize(0.03); + compressed.GetYaxis()->SetTitle("Bytes"); + compressed.SetFillColor(kBlue); + compressed.SetLineWidth(2); + uncompressed.GetYaxis()->SetTitle("Bytes"); + uncompressed.SetFillColor(kRed); + uncompressed.SetLineWidth(2); } - + int top; TH1F uncompressed; TH1F compressed; - TAxis * cxAxis; - TAxis * uxAxis; - + TAxis* cxAxis; + TAxis* uxAxis; + int x; }; - - } - - void EdmEventSize::produceHistos(std::string const & plot, std::string const & file, int top) const { - if (top==0) top = m_branches.size(); + + } // namespace detail + + void EdmEventSize::produceHistos(std::string const& plot, std::string const& file, int top) const { + if (top == 0) + top = m_branches.size(); detail::Hist h(top); - std::for_each(m_branches.begin(),m_branches.end(), - boost::bind(&detail::Hist::fill,boost::ref(h),_1)); + std::for_each(m_branches.begin(), m_branches.end(), boost::bind(&detail::Hist::fill, boost::ref(h), _1)); h.finalize(); - if( !plot.empty() ) { - gROOT->SetStyle( "Plain" ); - gStyle->SetOptStat( kFALSE ); + if (!plot.empty()) { + gROOT->SetStyle("Plain"); + gStyle->SetOptStat(kFALSE); gStyle->SetOptLogy(); TCanvas c; h.uncompressed.Draw(); - h.compressed.Draw( "same" ); - c.SaveAs( plot.c_str() ); + h.compressed.Draw("same"); + c.SaveAs(plot.c_str()); } - if ( !file.empty() ) { - TFile f( file.c_str(), "RECREATE" ); + if (!file.empty()) { + TFile f(file.c_str(), "RECREATE"); h.compressed.Write(); h.uncompressed.Write(); f.Close(); } - } -} +} // namespace perftools diff --git a/PerfTools/EdmEvent/test/branchSizes.cpp b/PerfTools/EdmEvent/test/branchSizes.cpp old mode 100755 new mode 100644 index a9b573e1b0569..143f07436257d --- a/PerfTools/EdmEvent/test/branchSizes.cpp +++ b/PerfTools/EdmEvent/test/branchSizes.cpp @@ -26,22 +26,22 @@ using namespace std; -static const char * const kHelpOpt = "help"; -static const char * const kHelpCommandOpt = "help,h"; -static const char * const kDataFileOpt = "data-file"; -static const char * const kDataFileCommandOpt = "data-file,d"; -static const char * const kAutoLoadOpt ="auto-loader"; -static const char * const kAutoLoadCommandOpt ="auto-loader,a"; -static const char * const kPlotOpt ="plot"; -static const char * const kPlotCommandOpt ="plot,p"; -static const char * const kSavePlotOpt ="save-plot"; -static const char * const kSavePlotCommandOpt ="save-plot,s"; -static const char * const kPlotTopOpt ="plot-top"; -static const char * const kPlotTopCommandOpt ="plot-top,t"; -static const char * const kVerboseOpt = "verbose"; -static const char * const kVerboseCommandOpt = "verbose,v"; -static const char * const kAlphabeticOrderOpt ="alphabetic-order"; -static const char * const kAlphabeticOrderCommandOpt ="alphabetic-order,A"; +static const char *const kHelpOpt = "help"; +static const char *const kHelpCommandOpt = "help,h"; +static const char *const kDataFileOpt = "data-file"; +static const char *const kDataFileCommandOpt = "data-file,d"; +static const char *const kAutoLoadOpt = "auto-loader"; +static const char *const kAutoLoadCommandOpt = "auto-loader,a"; +static const char *const kPlotOpt = "plot"; +static const char *const kPlotCommandOpt = "plot,p"; +static const char *const kSavePlotOpt = "save-plot"; +static const char *const kSavePlotCommandOpt = "save-plot,s"; +static const char *const kPlotTopOpt = "plot-top"; +static const char *const kPlotTopCommandOpt = "plot-top,t"; +static const char *const kVerboseOpt = "verbose"; +static const char *const kVerboseCommandOpt = "verbose,v"; +static const char *const kAlphabeticOrderOpt = "alphabetic-order"; +static const char *const kAlphabeticOrderCommandOpt = "alphabetic-order,A"; typedef pair size_type; @@ -49,257 +49,255 @@ typedef pair BranchRecord; typedef vector BranchVector; -ostream & operator << ( ostream & out, const size_type & s ) { +ostream &operator<<(ostream &out, const size_type &s) { out << s.first << '/' << s.second << " bytes"; - if ( s.second != 0 ) - out << " (compr: " << double( s.first ) / double ( s.second ) << ")"; + if (s.second != 0) + out << " (compr: " << double(s.first) / double(s.second) << ")"; return out; } -size_type & operator+=( size_type & s1, const size_type & s2 ) { +size_type &operator+=(size_type &s1, const size_type &s2) { s1.first += s2.first; s1.second += s2.second; return s1; } -size_type GetTotalSize( TBranch *, bool verbose ); +size_type GetTotalSize(TBranch *, bool verbose); -size_type GetBasketSize( TBranch *, bool verbose ); +size_type GetBasketSize(TBranch *, bool verbose); -size_type GetBasketSize( TObjArray * branches, bool verbose ) { - size_type result = make_pair( 0, 0 ); +size_type GetBasketSize(TObjArray *branches, bool verbose) { + size_type result = make_pair(0, 0); size_t n = branches->GetEntries(); - for( size_t i = 0; i < n; ++ i ) { - TBranch * b = dynamic_cast( branches->At( i ) ); - assert( b != 0 ); - result += GetBasketSize( b, verbose ); + for (size_t i = 0; i < n; ++i) { + TBranch *b = dynamic_cast(branches->At(i)); + assert(b != 0); + result += GetBasketSize(b, verbose); } return result; } -size_type GetBasketSize( TBranch * b, bool verbose ) { - size_type result = make_pair( 0, 0 ); - if ( b != 0 ) { - if ( b->GetZipBytes() > 0 ) { - result = make_pair( b->GetTotBytes(), b->GetZipBytes() ); +size_type GetBasketSize(TBranch *b, bool verbose) { + size_type result = make_pair(0, 0); + if (b != 0) { + if (b->GetZipBytes() > 0) { + result = make_pair(b->GetTotBytes(), b->GetZipBytes()); } else { - result = make_pair( b->GetTotalSize(), b->GetTotalSize() ); + result = make_pair(b->GetTotalSize(), b->GetTotalSize()); } - if ( verbose ) + if (verbose) cout << " branch: " << b->GetName() << ", size:" << result.first << "/" << result.second << endl; - result += GetBasketSize( b->GetListOfBranches(), verbose ); + result += GetBasketSize(b->GetListOfBranches(), verbose); } return result; } -size_type GetTotalSize( TBranch * br, bool verbose ) { - TBufferFile buf( TBuffer::kWrite, 10000 ); - TBranch::Class()->WriteBuffer( buf, br ); - size_type size = GetBasketSize( br, verbose ); - if ( br->GetZipBytes() > 0 ) +size_type GetTotalSize(TBranch *br, bool verbose) { + TBufferFile buf(TBuffer::kWrite, 10000); + TBranch::Class()->WriteBuffer(buf, br); + size_type size = GetBasketSize(br, verbose); + if (br->GetZipBytes() > 0) size.first += buf.Length(); - if ( verbose ) - cout << ">>> total branch size: " << br->GetName() << ":" << size.first << "/" << size.second << endl; + if (verbose) + cout << ">>> total branch size: " << br->GetName() << ":" << size.first << "/" << size.second << endl; return size; } -size_type GetTotalSize( TObjArray * branches, bool verbose ) { - size_type result = make_pair( 0, 0 ); +size_type GetTotalSize(TObjArray *branches, bool verbose) { + size_type result = make_pair(0, 0); size_t n = branches->GetEntries(); - for( size_t i = 0; i < n; ++ i ) { - result += GetTotalSize( dynamic_cast( branches->At( i ) ), verbose ); + for (size_t i = 0; i < n; ++i) { + result += GetTotalSize(dynamic_cast(branches->At(i)), verbose); } return result; } -size_type GetTotalSize( TTree *t ) { +size_type GetTotalSize(TTree *t) { size_t total = t->GetTotBytes(); TBufferFile b(TBuffer::kWrite, 10000); TTree::Class()->WriteBuffer(b, t); total += b.Length(); - return make_pair( total, t->GetZipBytes() ); -} + return make_pair(total, t->GetZipBytes()); +} -size_type GetTotalBranchSize( TTree *t, bool verbose ) { - return GetTotalSize( t->GetListOfBranches(), verbose ); -} +size_type GetTotalBranchSize(TTree *t, bool verbose) { return GetTotalSize(t->GetListOfBranches(), verbose); } struct sortByCompressedSize { - bool operator()( const BranchRecord & t1, const BranchRecord & t2 ) const { + bool operator()(const BranchRecord &t1, const BranchRecord &t2) const { size_t s1 = t1.second.second, s2 = t2.second.second; - if ( s1 == 0 && s2 == 0 ) { - s1 = t1.second.first; s2 = t2.second.first; + if (s1 == 0 && s2 == 0) { + s1 = t1.second.first; + s2 = t2.second.first; } return s1 > s2; } }; struct sortByName { - bool operator()( const BranchRecord & t1, const BranchRecord & t2 ) const { - return t1.first < t2.first; - } + bool operator()(const BranchRecord &t1, const BranchRecord &t2) const { return t1.first < t2.first; } }; -int main( int argc, char * argv[] ) { +int main(int argc, char *argv[]) { using namespace boost::program_options; using namespace std; - string programName( argv[ 0 ] ); - string descString( programName ); + string programName(argv[0]); + string descString(programName); descString += " [options] "; descString += "data_file \nAllowed options"; - options_description desc( descString ); - - desc.add_options() - ( kHelpCommandOpt, "produce help message" ) - ( kAutoLoadCommandOpt, "automatic library loading (avoid root warnings)" ) - ( kDataFileCommandOpt, value(), "data file" ) - ( kAlphabeticOrderCommandOpt, "sort by alphabetic order (default: sort by size)" ) - ( kPlotCommandOpt, value(), "produce a summary plot" ) - ( kPlotTopCommandOpt, value(), "plot only the top size branches" ) - ( kSavePlotCommandOpt, value(), "save plot into root file " ) - ( kVerboseCommandOpt, "verbose printout" ); + options_description desc(descString); + + desc.add_options()(kHelpCommandOpt, "produce help message")(kAutoLoadCommandOpt, + "automatic library loading (avoid root warnings)")( + kDataFileCommandOpt, value(), "data file")(kAlphabeticOrderCommandOpt, + "sort by alphabetic order (default: sort by size)")( + kPlotCommandOpt, value(), "produce a summary plot")( + kPlotTopCommandOpt, value(), "plot only the top size branches")( + kSavePlotCommandOpt, value(), "save plot into root file ")(kVerboseCommandOpt, "verbose printout"); positional_options_description p; - p.add( kDataFileOpt, -1 ); + p.add(kDataFileOpt, -1); variables_map vm; try { - store( command_line_parser(argc,argv).options(desc).positional(p).run(), vm ); - notify( vm ); - } catch( const error& ) { + store(command_line_parser(argc, argv).options(desc).positional(p).run(), vm); + notify(vm); + } catch (const error &) { return 7000; } - if( vm.count( kHelpOpt ) ) { - cout << desc <SetBatch(); - - if( vm.count( kAutoLoadOpt ) != 0 ) { - gSystem->Load( "libFWCoreFWLite" ); + + if (vm.count(kAutoLoadOpt) != 0) { + gSystem->Load("libFWCoreFWLite"); FWLiteEnabler::enable(); } string fileName = vm[kDataFileOpt].as(); - TFile file( fileName.c_str() ); - if( ! file.IsOpen() ) { + TFile file(fileName.c_str()); + if (!file.IsOpen()) { cerr << programName << ": unable to open data file " << fileName << endl; return 7002; } - TObject * o = file.Get( "Events" ); - if ( o == 0 ) { + TObject *o = file.Get("Events"); + if (o == 0) { cerr << programName << ": no object \"Events\" found in file: " << fileName << endl; return 7003; } - TTree * events = dynamic_cast( o ); - if ( events == 0 ) { + TTree *events = dynamic_cast(o); + if (events == 0) { cerr << programName << ": object \"Events\" is not a TTree in file: " << fileName << endl; return 7004; } - TObjArray * branches = events->GetListOfBranches(); - if ( branches == 0 ) { - cerr << programName << ": tree \"Events\" in file " << fileName - << " contains no branches" << endl; + TObjArray *branches = events->GetListOfBranches(); + if (branches == 0) { + cerr << programName << ": tree \"Events\" in file " << fileName << " contains no branches" << endl; return 7004; } - bool verbose = vm.count( kVerboseOpt ) > 0; + bool verbose = vm.count(kVerboseOpt) > 0; BranchVector v; - const size_t n = branches->GetEntries(); + const size_t n = branches->GetEntries(); cout << fileName << " has " << n << " branches" << endl; - for( size_t i = 0; i < n; ++i ) { - TBranch * b = dynamic_cast( branches->At( i ) ); - assert( b != 0 ); - string name( b->GetName() ); - if ( name == "EventAux" ) continue; - size_type s = GetTotalSize( b, verbose ); - v.push_back( make_pair( b->GetName(), s ) ); + for (size_t i = 0; i < n; ++i) { + TBranch *b = dynamic_cast(branches->At(i)); + assert(b != 0); + string name(b->GetName()); + if (name == "EventAux") + continue; + size_type s = GetTotalSize(b, verbose); + v.push_back(make_pair(b->GetName(), s)); } - if ( vm.count( kAlphabeticOrderOpt ) ) { - sort( v.begin(), v.end(), sortByName() ); + if (vm.count(kAlphabeticOrderOpt)) { + sort(v.begin(), v.end(), sortByName()); } else { - sort( v.begin(), v.end(), sortByCompressedSize() ); + sort(v.begin(), v.end(), sortByCompressedSize()); } - bool plot = ( vm.count( kPlotOpt ) > 0 ); - bool save = ( vm.count( kSavePlotOpt ) > 0 ); + bool plot = (vm.count(kPlotOpt) > 0); + bool save = (vm.count(kSavePlotOpt) > 0); int top = n; - if( vm.count( kPlotTopOpt ) > 0 ) top = vm[ kPlotTopOpt ].as(); - TH1F uncompressed( "uncompressed", "branch sizes", top, -0.5, - 0.5 + top ); - TH1F compressed( "compressed", "branch sizes", top, -0.5, - 0.5 + top ); + if (vm.count(kPlotTopOpt) > 0) + top = vm[kPlotTopOpt].as(); + TH1F uncompressed("uncompressed", "branch sizes", top, -0.5, -0.5 + top); + TH1F compressed("compressed", "branch sizes", top, -0.5, -0.5 + top); int x = 0; - TAxis * cxAxis = compressed.GetXaxis(); - TAxis * uxAxis = uncompressed.GetXaxis(); + TAxis *cxAxis = compressed.GetXaxis(); + TAxis *uxAxis = uncompressed.GetXaxis(); - for( BranchVector::const_iterator b = v.begin(); b != v.end(); ++ b ) { - const string & name = b->first; + for (BranchVector::const_iterator b = v.begin(); b != v.end(); ++b) { + const string &name = b->first; size_type size = b->second; cout << size << " " << name << endl; - if ( x < top ) { - cxAxis->SetBinLabel( x + 1, name.c_str() ); - uxAxis->SetBinLabel( x + 1, name.c_str() ); - compressed.Fill( x, size.second ); - uncompressed.Fill( x, size.first ); + if (x < top) { + cxAxis->SetBinLabel(x + 1, name.c_str()); + uxAxis->SetBinLabel(x + 1, name.c_str()); + compressed.Fill(x, size.second); + uncompressed.Fill(x, size.first); x++; } } // size_type branchSize = GetTotalBranchSize( events ); - // cout << "total branches size: " << branchSize.first << " bytes (uncompressed), " + // cout << "total branches size: " << branchSize.first << " bytes (uncompressed), " // << branchSize.second << " bytes (compressed)"<< endl; - size_type totalSize = GetTotalSize( events ); - cout << "total tree size: " << totalSize.first << " bytes (uncompressed), " - << totalSize.second << " bytes (compressed)"<< endl; + size_type totalSize = GetTotalSize(events); + cout << "total tree size: " << totalSize.first << " bytes (uncompressed), " << totalSize.second + << " bytes (compressed)" << endl; double mn = DBL_MAX; - for( int i = 1; i <= top; ++i ) { - double cm = compressed.GetMinimum( i ), um = uncompressed.GetMinimum( i ); - if ( cm > 0 && cm < mn ) mn = cm; - if ( um > 0 && um < mn ) mn = um; + for (int i = 1; i <= top; ++i) { + double cm = compressed.GetMinimum(i), um = uncompressed.GetMinimum(i); + if (cm > 0 && cm < mn) + mn = cm; + if (um > 0 && um < mn) + mn = um; } mn *= 0.8; - double mx = max( compressed.GetMaximum(), uncompressed.GetMaximum() ); + double mx = max(compressed.GetMaximum(), uncompressed.GetMaximum()); mx *= 1.2; - uncompressed.SetMinimum( mn ); - uncompressed.SetMaximum( mx ); - compressed.SetMinimum( mn ); + uncompressed.SetMinimum(mn); + uncompressed.SetMaximum(mx); + compressed.SetMinimum(mn); // compressed.SetMaximum( mx ); - cxAxis->SetLabelOffset( -0.32 ); - cxAxis->LabelsOption( "v" ); - cxAxis->SetLabelSize( 0.03 ); - uxAxis->SetLabelOffset( -0.32 ); - uxAxis->LabelsOption( "v" ); - uxAxis->SetLabelSize( 0.03 ); - compressed.GetYaxis()->SetTitle( "Bytes" ); - compressed.SetFillColor( kBlue ); - compressed.SetLineWidth( 2 ); - uncompressed.GetYaxis()->SetTitle( "Bytes" ); - uncompressed.SetFillColor( kRed ); - uncompressed.SetLineWidth( 2 ); - if( plot ) { + cxAxis->SetLabelOffset(-0.32); + cxAxis->LabelsOption("v"); + cxAxis->SetLabelSize(0.03); + uxAxis->SetLabelOffset(-0.32); + uxAxis->LabelsOption("v"); + uxAxis->SetLabelSize(0.03); + compressed.GetYaxis()->SetTitle("Bytes"); + compressed.SetFillColor(kBlue); + compressed.SetLineWidth(2); + uncompressed.GetYaxis()->SetTitle("Bytes"); + uncompressed.SetFillColor(kRed); + uncompressed.SetLineWidth(2); + if (plot) { string plotName = vm[kPlotOpt].as(); - gROOT->SetStyle( "Plain" ); - gStyle->SetOptStat( kFALSE ); + gROOT->SetStyle("Plain"); + gStyle->SetOptStat(kFALSE); gStyle->SetOptLogy(); TCanvas c; uncompressed.Draw(); - compressed.Draw( "same" ); - c.SaveAs( plotName.c_str() ); + compressed.Draw("same"); + c.SaveAs(plotName.c_str()); } - if ( save ) { + if (save) { string fileName = vm[kSavePlotOpt].as(); - TFile f( fileName.c_str(), "RECREATE" ); + TFile f(fileName.c_str(), "RECREATE"); compressed.Write(); uncompressed.Write(); f.Close(); diff --git a/Utilities/DCacheAdaptor/interface/DCacheFile.h b/Utilities/DCacheAdaptor/interface/DCacheFile.h index ed34d602e0640..975f5a8b5032b 100644 --- a/Utilities/DCacheAdaptor/interface/DCacheFile.h +++ b/Utilities/DCacheAdaptor/interface/DCacheFile.h @@ -1,51 +1,42 @@ #ifndef DCACHE_ADAPTOR_DCACHE_FILE_H -# define DCACHE_ADAPTOR_DCACHE_FILE_H +#define DCACHE_ADAPTOR_DCACHE_FILE_H -# include "Utilities/StorageFactory/interface/Storage.h" -# include "Utilities/StorageFactory/interface/IOFlags.h" -# include +#include "Utilities/StorageFactory/interface/Storage.h" +#include "Utilities/StorageFactory/interface/IOFlags.h" +#include -class DCacheFile : public Storage -{ +class DCacheFile : public Storage { public: - DCacheFile (void); - DCacheFile (IOFD fd); - DCacheFile (const char *name, int flags = IOFlags::OpenRead, int perms = 0666); - DCacheFile (const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); - ~DCacheFile (void) override; - - virtual void create (const char *name, - bool exclusive = false, - int perms = 0666); - virtual void create (const std::string &name, - bool exclusive = false, - int perms = 0666); - virtual void open (const char *name, - int flags = IOFlags::OpenRead, - int perms = 0666); - virtual void open (const std::string &name, - int flags = IOFlags::OpenRead, - int perms = 0666); + DCacheFile(void); + DCacheFile(IOFD fd); + DCacheFile(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + DCacheFile(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + ~DCacheFile(void) override; + virtual void create(const char *name, bool exclusive = false, int perms = 0666); + virtual void create(const std::string &name, bool exclusive = false, int perms = 0666); + virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + + using Storage::position; using Storage::read; using Storage::write; - using Storage::position; - IOSize read (void *into, IOSize n) override; - IOSize readv (IOBuffer *into, IOSize buffers) override; - IOSize readv (IOPosBuffer *into, IOSize buffers) override; - IOSize write (const void *from, IOSize n) override; + IOSize read(void *into, IOSize n) override; + IOSize readv(IOBuffer *into, IOSize buffers) override; + IOSize readv(IOPosBuffer *into, IOSize buffers) override; + IOSize write(const void *from, IOSize n) override; - IOOffset position (IOOffset offset, Relative whence = SET) override; - void resize (IOOffset size) override; + IOOffset position(IOOffset offset, Relative whence = SET) override; + void resize(IOOffset size) override; - void close (void) override; - virtual void abort (void); + void close(void) override; + virtual void abort(void); private: - IOFD m_fd; - bool m_close; - std::string m_name; + IOFD m_fd; + bool m_close; + std::string m_name; }; -#endif // DCACHE_ADAPTOR_DCACHE_FILE_H +#endif // DCACHE_ADAPTOR_DCACHE_FILE_H diff --git a/Utilities/DCacheAdaptor/plugins/DCacheStorageMaker.cc b/Utilities/DCacheAdaptor/plugins/DCacheStorageMaker.cc index 11d3529f9c985..537a9a6318d64 100644 --- a/Utilities/DCacheAdaptor/plugins/DCacheStorageMaker.cc +++ b/Utilities/DCacheAdaptor/plugins/DCacheStorageMaker.cc @@ -6,14 +6,12 @@ #include #include -class DCacheStorageMaker : public StorageMaker -{ +class DCacheStorageMaker : public StorageMaker { /** Return appropriate path for use with dcap library. If the path is in the URL form ([gsi]dcap://host:port/path), return the full URL as it is. If the path is /pnfs form (dcap:/pnfs/path), return only the path part, unless the protocol is 'gsidcap', in which case return the whole thing. */ - static std::string normalise (const std::string &proto, const std::string &path) - { + static std::string normalise(const std::string &proto, const std::string &path) { size_t p = path.find("/pnfs"); if (p < 3) return (proto == "gsidcap") ? proto + ':' + path.substr(p) : path.substr(p); @@ -26,21 +24,18 @@ class DCacheStorageMaker : public StorageMaker } public: - /** Open a storage object for the given URL (protocol + path), using the @a mode bits. No temporary files are downloaded. */ - std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - AuxSettings const& aux) const override - { + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + AuxSettings const &aux) const override { setTimeout(aux.timeout); const StorageFactory *f = StorageFactory::get(); StorageFactory::ReadHint readHint = f->readHint(); StorageFactory::CacheHint cacheHint = f->cacheHint(); - if (readHint != StorageFactory::READ_HINT_UNBUFFERED - || cacheHint == StorageFactory::CACHE_HINT_STORAGE) + if (readHint != StorageFactory::READ_HINT_UNBUFFERED || cacheHint == StorageFactory::CACHE_HINT_STORAGE) mode &= ~IOFlags::OpenUnbuffered; else mode |= IOFlags::OpenUnbuffered; @@ -49,51 +44,45 @@ class DCacheStorageMaker : public StorageMaker return f->wrapNonLocalFile(std::move(file), proto, std::string(), mode); } - void stagein (const std::string &proto, - const std::string &path, - const AuxSettings& aux) const override - { + void stagein(const std::string &proto, const std::string &path, const AuxSettings &aux) const override { setTimeout(aux.timeout); std::string npath = normalise(proto, path); if (dc_stage(npath.c_str(), 0, nullptr) != 0) { cms::Exception ex("FileStageInError"); - ex << "Cannot stage in file '" << npath - << "', error was: " << dc_strerror(dc_errno) - << " (dc_errno=" << dc_errno << ")"; + ex << "Cannot stage in file '" << npath << "', error was: " << dc_strerror(dc_errno) << " (dc_errno=" << dc_errno + << ")"; ex.addContext("Calling DCacheStorageMaker::stagein()"); throw ex; } } - bool check (const std::string &proto, - const std::string &path, - const AuxSettings& aux, - IOOffset *size = nullptr) const override - { + bool check(const std::string &proto, + const std::string &path, + const AuxSettings &aux, + IOOffset *size = nullptr) const override { setTimeout(aux.timeout); - std::string testpath (normalise (proto, path)); - if (dc_access (testpath.c_str (), R_OK) != 0) + std::string testpath(normalise(proto, path)); + if (dc_access(testpath.c_str(), R_OK) != 0) return false; - if (size) - { + if (size) { struct stat64 buf; - if (dc_stat64 (testpath.c_str (), &buf) != 0) + if (dc_stat64(testpath.c_str(), &buf) != 0) return false; - + *size = buf.st_size; } return true; } - - private: +private: void setTimeout(unsigned int timeout) const { - if (timeout != 0) dc_setOpenTimeout(timeout); + if (timeout != 0) + dc_setOpenTimeout(timeout); } }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, DCacheStorageMaker, "dcache"); -DEFINE_EDM_PLUGIN (StorageMakerFactory, DCacheStorageMaker, "dcap"); -DEFINE_EDM_PLUGIN (StorageMakerFactory, DCacheStorageMaker, "gsidcap"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, DCacheStorageMaker, "dcache"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, DCacheStorageMaker, "dcap"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, DCacheStorageMaker, "gsidcap"); diff --git a/Utilities/DCacheAdaptor/src/DCacheFile.cc b/Utilities/DCacheAdaptor/src/DCacheFile.cc index bfa6f06bf7527..3d86d4e13a917 100644 --- a/Utilities/DCacheAdaptor/src/DCacheFile.cc +++ b/Utilities/DCacheAdaptor/src/DCacheFile.cc @@ -8,72 +8,44 @@ #include #include -DCacheFile::DCacheFile (void) - : m_fd (EDM_IOFD_INVALID), - m_close (false) -{} - -DCacheFile::DCacheFile (IOFD fd) - : m_fd (fd), - m_close (true) -{} - -DCacheFile::DCacheFile (const char *name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) - : m_fd (EDM_IOFD_INVALID), - m_close (false) -{ open (name, flags, perms); } - -DCacheFile::DCacheFile (const std::string &name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) - : m_fd (EDM_IOFD_INVALID), - m_close (false) -{ open (name.c_str (), flags, perms); } - -DCacheFile::~DCacheFile (void) -{ +DCacheFile::DCacheFile(void) : m_fd(EDM_IOFD_INVALID), m_close(false) {} + +DCacheFile::DCacheFile(IOFD fd) : m_fd(fd), m_close(true) {} + +DCacheFile::DCacheFile(const char *name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) + : m_fd(EDM_IOFD_INVALID), m_close(false) { + open(name, flags, perms); +} + +DCacheFile::DCacheFile(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) + : m_fd(EDM_IOFD_INVALID), m_close(false) { + open(name.c_str(), flags, perms); +} + +DCacheFile::~DCacheFile(void) { if (m_close) - edm::LogError("DCacheFileError") - << "Destructor called on dCache file '" << m_name - << "' but the file is still open"; + edm::LogError("DCacheFileError") << "Destructor called on dCache file '" << m_name + << "' but the file is still open"; } ////////////////////////////////////////////////////////////////////// -void -DCacheFile::create (const char *name, - bool exclusive /* = false */, - int perms /* = 066 */) -{ - open (name, - (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate - | (exclusive ? IOFlags::OpenExclusive : 0)), - perms); +void DCacheFile::create(const char *name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name, + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), + perms); +} + +void DCacheFile::create(const std::string &name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name.c_str(), + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), + perms); } -void -DCacheFile::create (const std::string &name, - bool exclusive /* = false */, - int perms /* = 066 */) -{ - open (name.c_str (), - (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate - | (exclusive ? IOFlags::OpenExclusive : 0)), - perms); +void DCacheFile::open(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { + open(name.c_str(), flags, perms); } -void -DCacheFile::open (const std::string &name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) -{ open (name.c_str (), flags, perms); } - -void -DCacheFile::open (const char *name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) -{ +void DCacheFile::open(const char *name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { // Actual open if ((name == nullptr) || (*name == 0)) { edm::Exception ex(edm::errors::FileOpenError); @@ -91,7 +63,7 @@ DCacheFile::open (const char *name, // If I am already open, close old file first if (m_fd != EDM_IOFD_INVALID && m_close) - close (); + close(); // Translate our flags to system flags int openflags = 0; @@ -120,13 +92,10 @@ DCacheFile::open (const char *name, IOFD newfd = EDM_IOFD_INVALID; dc_errno = 0; - if ((newfd = dc_open (name, openflags, perms)) == -1) { + if ((newfd = dc_open(name, openflags, perms)) == -1) { edm::Exception ex(edm::errors::FileOpenError); - ex << "dc_open(name='" << name - << "', flags=0x" << std::hex << openflags - << ", permissions=0" << std::oct << perms << std::dec - << ") => error '" << dc_strerror(dc_errno) - << "' (dc_errno=" << dc_errno << ")"; + ex << "dc_open(name='" << name << "', flags=0x" << std::hex << openflags << ", permissions=0" << std::oct << perms + << std::dec << ") => error '" << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::open()"); throw ex; } @@ -150,24 +119,17 @@ DCacheFile::open (const char *name, edm::LogInfo("DCacheFileInfo") << "Opened " << m_name; } -void -DCacheFile::close (void) -{ - if (m_fd == EDM_IOFD_INVALID) - { - edm::LogError("DCacheFileError") - << "DCacheFile::close(name='" << m_name - << "') called but the file is not open"; +void DCacheFile::close(void) { + if (m_fd == EDM_IOFD_INVALID) { + edm::LogError("DCacheFileError") << "DCacheFile::close(name='" << m_name << "') called but the file is not open"; m_close = false; return; } dc_errno = 0; - if (dc_close (m_fd) == -1) - edm::LogWarning("DCacheFileWarning") - << "dc_close(name='" << m_name - << "') failed with error '" << dc_strerror (dc_errno) - << "' (dc_errno=" << dc_errno << ")"; + if (dc_close(m_fd) == -1) + edm::LogWarning("DCacheFileWarning") << "dc_close(name='" << m_name << "') failed with error '" + << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; m_close = false; m_fd = EDM_IOFD_INVALID; @@ -176,11 +138,9 @@ DCacheFile::close (void) // edm::LogInfo("DCacheFileInfo") << "Closed " << m_name; } -void -DCacheFile::abort (void) -{ +void DCacheFile::abort(void) { if (m_fd != EDM_IOFD_INVALID) - dc_close (m_fd); + dc_close(m_fd); m_close = false; m_fd = EDM_IOFD_INVALID; @@ -202,91 +162,75 @@ static const int BUGLINE = __LINE__ + 1; // (In other words, barring signals (which should use SA_RESTART and // in any case should not affect dCache) the only way a read from a // file can return short is when there is nothing left to read.) -IOSize -DCacheFile::read (void *into, IOSize n) -{ +IOSize DCacheFile::read(void *into, IOSize n) { IOSize done = 0; - while (done < n) - { + while (done < n) { dc_errno = 0; - ssize_t s = dc_read (m_fd, (char *) into + done, n - done); + ssize_t s = dc_read(m_fd, (char *)into + done, n - done); if (s == -1) { edm::Exception ex(edm::errors::FileReadError); - ex << "dc_read(name='" << m_name << "', n=" << (n-done) - << ") failed with error '" << dc_strerror(dc_errno) + ex << "dc_read(name='" << m_name << "', n=" << (n - done) << ") failed with error '" << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::read()"); throw ex; - } - else if (s == 0) + } else if (s == 0) // end of file break; - else if (s < ssize_t (n-done)) - edm::LogInfo("DCacheFileWarning") - << "dc_read(name='" << m_name << "', n=" << (n-done) - << ") returned a short read of " << s << " bytes; " - << "please report a bug in dCache referencing the " - << "comment on line " << BUGLINE << " of " << __FILE__; + else if (s < ssize_t(n - done)) + edm::LogInfo("DCacheFileWarning") << "dc_read(name='" << m_name << "', n=" << (n - done) + << ") returned a short read of " << s << " bytes; " + << "please report a bug in dCache referencing the " + << "comment on line " << BUGLINE << " of " << __FILE__; done += s; } return done; } -IOSize -DCacheFile::write (const void *from, IOSize n) -{ +IOSize DCacheFile::write(const void *from, IOSize n) { IOSize done = 0; - while (done < n) - { + while (done < n) { dc_errno = 0; - ssize_t s = dc_write (m_fd, (const char *) from + done, n - done); + ssize_t s = dc_write(m_fd, (const char *)from + done, n - done); if (s == -1) { edm::Exception ex(edm::errors::FileWriteError); - ex << "dc_write(name='" << m_name << "', n=" << (n-done) - << ") failed with error '" << dc_strerror(dc_errno) + ex << "dc_write(name='" << m_name << "', n=" << (n - done) << ") failed with error '" << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::write()"); throw ex; - } - else if (s < ssize_t (n-done)) - edm::LogInfo("DCacheFileWarning") - << "dc_write(name='" << m_name << "', n=" << (n-done) - << ") returned a short write of " << s << " bytes; " - << "please report a bug in dCache referencing the " - << "comment on line " << BUGLINE << " of " << __FILE__; + } else if (s < ssize_t(n - done)) + edm::LogInfo("DCacheFileWarning") << "dc_write(name='" << m_name << "', n=" << (n - done) + << ") returned a short write of " << s << " bytes; " + << "please report a bug in dCache referencing the " + << "comment on line " << BUGLINE << " of " << __FILE__; done += s; } return done; } -IOSize -DCacheFile::readv (IOBuffer *into, IOSize buffers) -{ - assert (! buffers || into); +IOSize DCacheFile::readv(IOBuffer *into, IOSize buffers) { + assert(!buffers || into); // readv may not support zero buffers. - if (! buffers) + if (!buffers) return 0; // Convert the buffers to system format. - std::vector bufs (buffers); - for (IOSize i = 0; i < buffers; ++i) - { - bufs [i].iov_len = into [i].size (); - bufs [i].iov_base = (caddr_t) into [i].data (); + std::vector bufs(buffers); + for (IOSize i = 0; i < buffers; ++i) { + bufs[i].iov_len = into[i].size(); + bufs[i].iov_base = (caddr_t)into[i].data(); } // Read as long as signals cancel the read before doing anything. dc_errno = 0; - ssize_t n = dc_readv (m_fd, &bufs [0], buffers); + ssize_t n = dc_readv(m_fd, &bufs[0], buffers); // If it was serious error, throw it. if (n == -1) { edm::Exception ex(edm::errors::FileReadError); - ex << "dc_readv(name='" << m_name << "', iov[" << buffers - << "]) failed with error '" << dc_strerror(dc_errno) + ex << "dc_readv(name='" << m_name << "', iov[" << buffers << "]) failed with error '" << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::readv()"); throw ex; @@ -296,35 +240,31 @@ DCacheFile::readv (IOBuffer *into, IOSize buffers) return n; } -IOSize -DCacheFile::readv (IOPosBuffer *into, IOSize buffers) -{ - assert (! buffers || into); +IOSize DCacheFile::readv(IOPosBuffer *into, IOSize buffers) { + assert(!buffers || into); // readv may not support zero buffers. - if (! buffers) + if (!buffers) return 0; // Convert the buffers to system format. - std::vector bufs (buffers); + std::vector bufs(buffers); IOSize total = 0; - for (IOSize i = 0; i < buffers; ++i) - { - bufs [i].offset = into [i].offset (); - bufs [i].len = into [i].size (); - bufs [i].buf = (char *) into [i].data (); - total += into [i].size (); + for (IOSize i = 0; i < buffers; ++i) { + bufs[i].offset = into[i].offset(); + bufs[i].len = into[i].size(); + bufs[i].buf = (char *)into[i].data(); + total += into[i].size(); } // Read as long as signals cancel the read before doing anything. dc_errno = 0; - ssize_t n = dc_readv2 (m_fd, &bufs [0], buffers); + ssize_t n = dc_readv2(m_fd, &bufs[0], buffers); // If it was serious error, throw it. if (n == -1) { edm::Exception ex(edm::errors::FileReadError); - ex << "dc_readv2(name='" << m_name << "', iov2[" << buffers - << "]) failed with error '" << dc_strerror(dc_errno) + ex << "dc_readv2(name='" << m_name << "', iov2[" << buffers << "]) failed with error '" << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::readv()"); throw ex; @@ -336,9 +276,7 @@ DCacheFile::readv (IOPosBuffer *into, IOSize buffers) ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -IOOffset -DCacheFile::position (IOOffset offset, Relative whence /* = SET */) -{ +IOOffset DCacheFile::position(IOOffset offset, Relative whence /* = SET */) { if (m_fd == EDM_IOFD_INVALID) { cms::Exception ex("FilePositionError"); ex << "DCacheFile::position() called on a closed file"; @@ -349,36 +287,30 @@ DCacheFile::position (IOOffset offset, Relative whence /* = SET */) ex << "DCacheFile::position() called with incorrect 'whence' parameter"; throw ex; } - IOOffset result; - int mywhence = (whence == SET ? SEEK_SET - : whence == CURRENT ? SEEK_CUR - : SEEK_END); + IOOffset result; + int mywhence = (whence == SET ? SEEK_SET : whence == CURRENT ? SEEK_CUR : SEEK_END); dc_errno = 0; - if ((result = dc_lseek64 (m_fd, offset, mywhence)) == -1) { + if ((result = dc_lseek64(m_fd, offset, mywhence)) == -1) { cms::Exception ex("FilePositionError"); - ex << "dc_lseek64(name='" << m_name << "', offset=" << offset - << ", whence=" << mywhence << ") failed with error '" - << dc_strerror (dc_errno) << "' (dc_errno=" << dc_errno << ")"; + ex << "dc_lseek64(name='" << m_name << "', offset=" << offset << ", whence=" << mywhence << ") failed with error '" + << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::position()"); throw ex; } // FIXME: dCache returns incorrect value on SEEK_END. // Remove this hack when dcap has been fixed. - if (whence == SEEK_END && (result = dc_lseek64 (m_fd, result, SEEK_SET)) == -1) { + if (whence == SEEK_END && (result = dc_lseek64(m_fd, result, SEEK_SET)) == -1) { cms::Exception ex("FilePositionError"); - ex << "dc_lseek64(name='" << m_name << "', offset=" << offset - << ", whence=" << SEEK_SET << ") failed with error '" - << dc_strerror (dc_errno) << "' (dc_errno=" << dc_errno << ")"; + ex << "dc_lseek64(name='" << m_name << "', offset=" << offset << ", whence=" << SEEK_SET << ") failed with error '" + << dc_strerror(dc_errno) << "' (dc_errno=" << dc_errno << ")"; ex.addContext("Calling DCacheFile::position()"); throw ex; } return result; } -void -DCacheFile::resize (IOOffset /* size */) -{ +void DCacheFile::resize(IOOffset /* size */) { cms::Exception ex("FileResizeError"); ex << "DCacheFile::resize(name='" << m_name << "') not implemented"; throw ex; diff --git a/Utilities/DavixAdaptor/interface/DavixFile.h b/Utilities/DavixAdaptor/interface/DavixFile.h index 756e11b482b5c..5ff70e59e7e0f 100644 --- a/Utilities/DavixAdaptor/interface/DavixFile.h +++ b/Utilities/DavixAdaptor/interface/DavixFile.h @@ -9,23 +9,18 @@ class DavixFile : public Storage { public: DavixFile(void); DavixFile(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); - DavixFile(const std::string &name, int flags = IOFlags::OpenRead, - int perms = 0666); + DavixFile(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); ~DavixFile(void) override; static void configureDavixLogLevel(); - virtual void create(const char *name, bool exclusive = false, - int perms = 0666); - virtual void create(const std::string &name, bool exclusive = false, - int perms = 0666); - virtual void open(const char *name, int flags = IOFlags::OpenRead, - int perms = 0666); - virtual void open(const std::string &name, int flags = IOFlags::OpenRead, - int perms = 0666); + virtual void create(const char *name, bool exclusive = false, int perms = 0666); + virtual void create(const std::string &name, bool exclusive = false, int perms = 0666); + virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + using Storage::position; using Storage::read; using Storage::write; - using Storage::position; IOSize read(void *into, IOSize n) override; IOSize readv(IOBuffer *into, IOSize buffers) override; @@ -46,4 +41,4 @@ class DavixFile : public Storage { std::string m_name; }; -#endif // DAVIX_ADAPTOR_DAVIX_FILE_H +#endif // DAVIX_ADAPTOR_DAVIX_FILE_H diff --git a/Utilities/DavixAdaptor/plugins/DavixStorageMaker.cc b/Utilities/DavixAdaptor/plugins/DavixStorageMaker.cc index 6975f5ade3e97..613375eaa57d3 100644 --- a/Utilities/DavixAdaptor/plugins/DavixStorageMaker.cc +++ b/Utilities/DavixAdaptor/plugins/DavixStorageMaker.cc @@ -7,20 +7,23 @@ #include class DavixStorageMaker : public StorageMaker { - public: /** Open a storage object for the given URL (protocol + path), using the @a mode bits. No temporary files are downloaded. */ - std::unique_ptr open(const std::string &proto, const std::string &path, int mode, - AuxSettings const &aux) const override { + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + AuxSettings const &aux) const override { const StorageFactory *f = StorageFactory::get(); std::string newurl((proto == "web" ? "http" : proto) + ":" + path); auto file = std::make_unique(newurl, mode); return f->wrapNonLocalFile(std::move(file), proto, std::string(), mode); } - bool check(const std::string &proto, const std::string &path, const AuxSettings &aux, - IOOffset *size = nullptr) const override { + bool check(const std::string &proto, + const std::string &path, + const AuxSettings &aux, + IOOffset *size = nullptr) const override { std::string newurl((proto == "web" ? "http" : proto) + ":" + path); Davix::DavixError *err = nullptr; Davix::Context c; @@ -30,8 +33,7 @@ class DavixStorageMaker : public StorageMaker { if (err) { std::unique_ptr davixErrManaged(err); cms::Exception ex("FileCheckError"); - ex << "Check failed with error " << err->getErrMsg().c_str() << " and error code" - << err->getStatus(); + ex << "Check failed with error " << err->getErrMsg().c_str() << " and error code" << err->getStatus(); ex.addContext("Calling DavixFile::check()"); throw ex; } diff --git a/Utilities/DavixAdaptor/src/DavixFile.cc b/Utilities/DavixAdaptor/src/DavixFile.cc index 474d4fc8c54b9..bca6f5fb2e6de 100644 --- a/Utilities/DavixAdaptor/src/DavixFile.cc +++ b/Utilities/DavixAdaptor/src/DavixFile.cc @@ -21,8 +21,7 @@ DavixFile::DavixFile(const char *name, int flags /* = IOFlags::OpenRead */, int open(name, flags, perms); } -DavixFile::DavixFile(const std::string &name, int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) { +DavixFile::DavixFile(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { open(name.c_str(), flags, perms); } @@ -40,8 +39,8 @@ void DavixFile::close(void) { if (err) { std::unique_ptr davixErrManaged(err); cms::Exception ex("FileCloseError"); - ex << "Davix::close(name='" << m_name << ") failed with error " - << err->getErrMsg().c_str() << " and error code " << err->getStatus(); + ex << "Davix::close(name='" << m_name << ") failed with error " << err->getErrMsg().c_str() << " and error code " + << err->getStatus(); ex.addContext("Calling DavixFile::close()"); throw ex; } @@ -56,8 +55,8 @@ void DavixFile::abort(void) { if (err) { std::unique_ptr davixErrManaged(err); cms::Exception ex("FileAbortError"); - ex << "Davix::abort(name='" << m_name << ") failed with error " - << err->getErrMsg().c_str() << " and error code " << err->getStatus(); + ex << "Davix::abort(name='" << m_name << ") failed with error " << err->getErrMsg().c_str() << " and error code " + << err->getStatus(); ex.addContext("Calling DavixFile::abort()"); throw ex; } @@ -68,7 +67,7 @@ void DavixFile::abort(void) { void DavixFile::configureDavixLogLevel() { long logLevel = 0; char *logptr = nullptr; - char const * const davixDebug = getenv("Davix_Debug"); + char const *const davixDebug = getenv("Davix_Debug"); if (davixDebug != nullptr) { logLevel = strtol(davixDebug, &logptr, 0); if (errno) { @@ -88,26 +87,25 @@ void DavixFile::configureDavixLogLevel() { } } switch (logLevel) { - case 0: - std::call_once(davixDebugInit, davix_set_log_level, 0); - break; - case 1: - std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_WARNING); - break; - case 2: - std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_VERBOSE); - break; - case 3: - std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_DEBUG); - break; - default: - std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_ALL); - break; + case 0: + std::call_once(davixDebugInit, davix_set_log_level, 0); + break; + case 1: + std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_WARNING); + break; + case 2: + std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_VERBOSE); + break; + case 3: + std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_DEBUG); + break; + default: + std::call_once(davixDebugInit, davix_set_log_level, DAVIX_LOG_ALL); + break; } } -static int X509Authentication(void *userdata, const SessionInfo &info, X509Credential *cert, - DavixError **davixErr) { +static int X509Authentication(void *userdata, const SessionInfo &info, X509Credential *cert, DavixError **davixErr) { std::string ucert, ukey; char default_proxy[64]; snprintf(default_proxy, sizeof(default_proxy), "/tmp/x509up_u%d", geteuid()); @@ -144,20 +142,18 @@ static int X509Authentication(void *userdata, const SessionInfo &info, X509Crede } void DavixFile::create(const char *name, bool exclusive /* = false */, int perms /* = 066 */) { - open(name, (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | - (exclusive ? IOFlags::OpenExclusive : 0)), + open(name, + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), perms); } -void DavixFile::create(const std::string &name, bool exclusive /* = false */, - int perms /* = 066 */) { - open(name.c_str(), (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | - (exclusive ? IOFlags::OpenExclusive : 0)), +void DavixFile::create(const std::string &name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name.c_str(), + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), perms); } -void DavixFile::open(const std::string &name, int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) { +void DavixFile::open(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { open(name.c_str(), flags, perms); } @@ -232,7 +228,7 @@ IOSize DavixFile::readv(IOBuffer *into, IOSize buffers) { std::vector input_vector(buffers); std::vector output_vector(buffers); - IOSize total = 0; // Total requested bytes + IOSize total = 0; // Total requested bytes for (IOSize i = 0; i < buffers; ++i) { input_vector[i].diov_size = into[i].size(); input_vector[i].diov_buffer = static_cast(into[i].data()); @@ -243,9 +239,9 @@ IOSize DavixFile::readv(IOBuffer *into, IOSize buffers) { if (davixErr) { std::unique_ptr davixErrManaged(davixErr); edm::Exception ex(edm::errors::FileReadError); - ex << "Davix::readv(name='" << m_name << "', buffers=" << (buffers) - << ") failed with error " << davixErr->getErrMsg().c_str() << " and error code " - << davixErr->getStatus() << " and call returned " << s << " bytes"; + ex << "Davix::readv(name='" << m_name << "', buffers=" << (buffers) << ") failed with error " + << davixErr->getErrMsg().c_str() << " and error code " << davixErr->getStatus() << " and call returned " << s + << " bytes"; ex.addContext("Calling DavixFile::readv()"); throw ex; } @@ -290,8 +286,8 @@ IOSize DavixFile::readv(IOPosBuffer *into, IOSize buffers) { std::unique_ptr davixErrManaged(davixErr); edm::Exception ex(edm::errors::FileReadError); ex << "Davix::readv(name='" << m_name << "', n=" << buffers << ") failed with error " - << davixErr->getErrMsg().c_str() << " and error code " << davixErr->getStatus() - << " and call returned " << s << " bytes"; + << davixErr->getErrMsg().c_str() << " and error code " << davixErr->getStatus() << " and call returned " << s + << " bytes"; ex.addContext("Calling DavixFile::readv()"); throw ex; } @@ -303,8 +299,7 @@ IOSize DavixFile::readv(IOPosBuffer *into, IOSize buffers) { // Only check if returned val <= 0 and make proper actions. if (s < 0) { edm::Exception ex(edm::errors::FileReadError); - ex << "Davix::readv(name='" << m_name << "', n=" << buffers - << ") failed and call returned " << s; + ex << "Davix::readv(name='" << m_name << "', n=" << buffers << ") failed and call returned " << s; ex.addContext("Calling DavixFile::readv()"); throw ex; } else if (s == 0) { @@ -323,16 +318,15 @@ IOSize DavixFile::read(void *into, IOSize n) { if (davixErr) { std::unique_ptr davixErrManaged(davixErr); edm::Exception ex(edm::errors::FileReadError); - ex << "Davix::read(name='" << m_name << "', n=" << (n - done) - << ") failed with error " << davixErr->getErrMsg().c_str() << " and error code " - << davixErr->getStatus() << " and call returned " << s << " bytes"; + ex << "Davix::read(name='" << m_name << "', n=" << (n - done) << ") failed with error " + << davixErr->getErrMsg().c_str() << " and error code " << davixErr->getStatus() << " and call returned " << s + << " bytes"; ex.addContext("Calling DavixFile::read()"); throw ex; } if (s < 0) { edm::Exception ex(edm::errors::FileReadError); - ex << "Davix::read(name='" << m_name << "', n=" << (n - done) - << ") failed and call returned " << s; + ex << "Davix::read(name='" << m_name << "', n=" << (n - done) << ") failed and call returned " << s; ex.addContext("Calling DavixFile::read()"); throw ex; } else if (s == 0) { @@ -364,9 +358,8 @@ IOOffset DavixFile::position(IOOffset offset, Relative whence /* = SET */) { if ((result = m_davixPosix->lseek(m_fd, offset, mywhence, &davixErr)) == -1) { cms::Exception ex("FilePositionError"); - ex << "Davix::lseek(name='" << m_name << "', offset=" << offset - << ", whence=" << mywhence << ") failed with error " << davixErr->getErrMsg().c_str() - << " and error code " << davixErr->getStatus() << " and " + ex << "Davix::lseek(name='" << m_name << "', offset=" << offset << ", whence=" << mywhence << ") failed with error " + << davixErr->getErrMsg().c_str() << " and error code " << davixErr->getStatus() << " and " << "call returned " << result; ex.addContext("Calling DavixFile::position()"); throw ex; diff --git a/Utilities/DavixAdaptor/test/davixcheck.cpp b/Utilities/DavixAdaptor/test/davixcheck.cpp index 583a10aa77dda..26fa3f51dd2d1 100644 --- a/Utilities/DavixAdaptor/test/davixcheck.cpp +++ b/Utilities/DavixAdaptor/test/davixcheck.cpp @@ -5,10 +5,11 @@ int main(int, char ** /*argv*/) try { initTest(); IOOffset size = -1; - bool exists = StorageFactory::get()->check("http://opendata.cern.ch/eos/opendata" - "/cms/Run2011A/PhotonHad/AOD/12Oct2013-v1" - "/00000/024938EB-3445-E311-A72B-002590593920.root", - &size); + bool exists = StorageFactory::get()->check( + "http://opendata.cern.ch/eos/opendata" + "/cms/Run2011A/PhotonHad/AOD/12Oct2013-v1" + "/00000/024938EB-3445-E311-A72B-002590593920.root", + &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; std::cout << "stats:\n" << StorageAccount::summaryText() << std::endl; diff --git a/Utilities/DavixAdaptor/test/davixreadfilenotfound.cpp b/Utilities/DavixAdaptor/test/davixreadfilenotfound.cpp index e9f6abf914846..7559d44fbe879 100644 --- a/Utilities/DavixAdaptor/test/davixreadfilenotfound.cpp +++ b/Utilities/DavixAdaptor/test/davixreadfilenotfound.cpp @@ -7,8 +7,8 @@ int main(int, char ** /*argv*/) try { initTest(); char buf[1024]; - std::unique_ptr s = StorageFactory::get()->open( - "http://cern.ch/cmsbuild/cms/mc/this/file/does/not/exist.root"); + std::unique_ptr s = + StorageFactory::get()->open("http://cern.ch/cmsbuild/cms/mc/this/file/does/not/exist.root"); assert(s); s->read(buf, sizeof(buf)); diff --git a/Utilities/DavixAdaptor/test/davixreadwrongurl.cpp b/Utilities/DavixAdaptor/test/davixreadwrongurl.cpp index e6e7e1e922999..cc3858db69e5f 100644 --- a/Utilities/DavixAdaptor/test/davixreadwrongurl.cpp +++ b/Utilities/DavixAdaptor/test/davixreadwrongurl.cpp @@ -7,9 +7,9 @@ int main(int, char ** /*argv*/) try { initTest(); char buf[1024]; - std::unique_ptr s = - StorageFactory::get()->open("http://i.am.davix.plugin.fake.url:1234/store/mc/HC/GenericTTbar/ " - "GEN/127938CD-F8CC-E311-9250-02163E00E8E6.root"); + std::unique_ptr s = StorageFactory::get()->open( + "http://i.am.davix.plugin.fake.url:1234/store/mc/HC/GenericTTbar/ " + "GEN/127938CD-F8CC-E311-9250-02163E00E8E6.root"); assert(s); s->read(buf, sizeof(buf)); diff --git a/Utilities/DavixAdaptor/test/http.cpp b/Utilities/DavixAdaptor/test/http.cpp index db9d05bc31aeb..2845c704951b0 100644 --- a/Utilities/DavixAdaptor/test/http.cpp +++ b/Utilities/DavixAdaptor/test/http.cpp @@ -3,27 +3,25 @@ #include "FWCore/Utilities/interface/Exception.h" #include -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOSize n; - char buf [1024]; - auto s = StorageFactory::get ()->open - ("http://home.web.cern.ch", IOFlags::OpenRead); + IOSize n; + char buf[1024]; + auto s = StorageFactory::get()->open("http://home.web.cern.ch", IOFlags::OpenRead); - assert (s); - while ((n = s->read (buf, sizeof (buf)))) - std::cout.write (buf, n); + assert(s); + while ((n = s->read(buf, sizeof(buf)))) + std::cout.write(buf, n); s->close(); - std::cerr << StorageAccount::summaryText (true) << std::endl; + std::cerr << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/DavixAdaptor/test/http2.cpp b/Utilities/DavixAdaptor/test/http2.cpp index 4e748bef396b5..833b52cd69837 100644 --- a/Utilities/DavixAdaptor/test/http2.cpp +++ b/Utilities/DavixAdaptor/test/http2.cpp @@ -1,21 +1,19 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOOffset size = -1; - bool exists = StorageFactory::get ()->check - ("http://home.web.cern.ch", &size); + IOOffset size = -1; + bool exists = StorageFactory::get()->check("http://home.web.cern.ch", &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; - std::cout << "stats:\n" << StorageAccount::summaryText () << std::endl; + std::cout << "stats:\n" << StorageAccount::summaryText() << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/DavixAdaptor/test/http3.cpp b/Utilities/DavixAdaptor/test/http3.cpp index 7f34622871303..aa93925182ca4 100644 --- a/Utilities/DavixAdaptor/test/http3.cpp +++ b/Utilities/DavixAdaptor/test/http3.cpp @@ -1,20 +1,18 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - bool exists = StorageFactory::get ()->check - ("http://home.web.cern.ch"); + bool exists = StorageFactory::get()->check("http://home.web.cern.ch"); std::cout << "exists = " << exists << "\n"; - std::cout << "stats:\n" << StorageAccount::summaryText () << std::endl; + std::cout << "stats:\n" << StorageAccount::summaryText() << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/DavixAdaptor/test/http4.cpp b/Utilities/DavixAdaptor/test/http4.cpp index 4e748bef396b5..833b52cd69837 100644 --- a/Utilities/DavixAdaptor/test/http4.cpp +++ b/Utilities/DavixAdaptor/test/http4.cpp @@ -1,21 +1,19 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOOffset size = -1; - bool exists = StorageFactory::get ()->check - ("http://home.web.cern.ch", &size); + IOOffset size = -1; + bool exists = StorageFactory::get()->check("http://home.web.cern.ch", &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; - std::cout << "stats:\n" << StorageAccount::summaryText () << std::endl; + std::cout << "stats:\n" << StorageAccount::summaryText() << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/LStoreAdaptor/interface/LStoreFile.h b/Utilities/LStoreAdaptor/interface/LStoreFile.h index 4f669d8e47448..3c6207a3c0fe9 100644 --- a/Utilities/LStoreAdaptor/interface/LStoreFile.h +++ b/Utilities/LStoreAdaptor/interface/LStoreFile.h @@ -1,50 +1,41 @@ #ifndef LSTORE_ADAPTOR_LSTORE_FILE_H -# define LSTORE_ADAPTOR_LSTORE_FILE_H +#define LSTORE_ADAPTOR_LSTORE_FILE_H -# include "Utilities/StorageFactory/interface/Storage.h" -# include "Utilities/StorageFactory/interface/IOFlags.h" -# include +#include "Utilities/StorageFactory/interface/Storage.h" +#include "Utilities/StorageFactory/interface/IOFlags.h" +#include #include -class LStoreFile : public Storage -{ +class LStoreFile : public Storage { public: - LStoreFile (void); - LStoreFile (void * fd); - LStoreFile (const char *name, int flags = IOFlags::OpenRead, int perms = 0666); - LStoreFile (const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); - ~LStoreFile (void) override; + LStoreFile(void); + LStoreFile(void *fd); + LStoreFile(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + LStoreFile(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + ~LStoreFile(void) override; - virtual void create (const char *name, - bool exclusive = false, - int perms = 0666); - virtual void create (const std::string &name, - bool exclusive = false, - int perms = 0666); - virtual void open (const char *name, - int flags = IOFlags::OpenRead, - int perms = 0666); - virtual void open (const std::string &name, - int flags = IOFlags::OpenRead, - int perms = 0666); + virtual void create(const char *name, bool exclusive = false, int perms = 0666); + virtual void create(const std::string &name, bool exclusive = false, int perms = 0666); + virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + using Storage::position; using Storage::read; using Storage::write; - using Storage::position; - IOSize read (void *into, IOSize n) override; - IOSize write (const void *from, IOSize n) override; + IOSize read(void *into, IOSize n) override; + IOSize write(const void *from, IOSize n) override; - IOOffset position (IOOffset offset, Relative whence = SET) override; - void resize (IOOffset size) override; + IOOffset position(IOOffset offset, Relative whence = SET) override; + void resize(IOOffset size) override; + + void close(void) override; + virtual void abort(void); - void close (void) override; - virtual void abort (void); - class MutexWrapper { - public: - MutexWrapper( pthread_mutex_t * lock ); - ~MutexWrapper(); - pthread_mutex_t * m_lock; + public: + MutexWrapper(pthread_mutex_t *lock); + ~MutexWrapper(); + pthread_mutex_t *m_lock; }; static pthread_mutex_t m_dlopen_lock; @@ -57,25 +48,24 @@ class LStoreFile : public Storage //data - void * m_fd; - bool m_close; - std::string m_name; - void * m_library_handle; + void *m_fd; + bool m_close; + std::string m_name; + void *m_library_handle; bool m_is_loaded; // Prototypes for functions // These are data members that are function pointers. int32_t (*redd_init)(); - int64_t (*redd_read)(void *, char*, int64_t); + int64_t (*redd_read)(void *, char *, int64_t); int32_t (*redd_close)(void *); int64_t (*redd_lseek)(void *, int64_t, uint32_t); - void * (*redd_open)(const char *, int32_t, int32_t ); + void *(*redd_open)(const char *, int32_t, int32_t); int64_t (*redd_write)(void *, const char *, int64_t); int32_t (*redd_term)(); int32_t (*redd_errno)(); - const std::string & (*redd_strerror)(); + const std::string &(*redd_strerror)(); }; -#endif // LSTORE_ADAPTOR_LSTORE_FILE_H - +#endif // LSTORE_ADAPTOR_LSTORE_FILE_H diff --git a/Utilities/LStoreAdaptor/plugins/LStoreStorageMaker.cc b/Utilities/LStoreAdaptor/plugins/LStoreStorageMaker.cc index 476f6c0d138fb..24d46572dfa2d 100644 --- a/Utilities/LStoreAdaptor/plugins/LStoreStorageMaker.cc +++ b/Utilities/LStoreAdaptor/plugins/LStoreStorageMaker.cc @@ -7,21 +7,19 @@ #include #include -class LStoreStorageMaker : public StorageMaker -{ - public: +class LStoreStorageMaker : public StorageMaker { +public: /** Open a storage object for the given URL (protocol + path), using the @a mode bits. No temporary files are downloaded. */ - std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - const AuxSettings&) const override - { - std::string fullpath = proto + ":" + path; - return std::make_unique (fullpath, mode); + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + const AuxSettings &) const override { + std::string fullpath = proto + ":" + path; + return std::make_unique(fullpath, mode); } -/* I don't think this is necessary - Melo + /* I don't think this is necessary - Melo virtual void stagein (const std::string &proto, const std::string &path) { std::string fullpath(proto + ":" + path); @@ -35,20 +33,18 @@ class LStoreStorageMaker : public StorageMaker } */ - bool check (const std::string &proto, - const std::string &path, - const AuxSettings&, - IOOffset *size = nullptr) const override - { - std::string fullpath = proto + ":" + path; - try { - LStoreFile fileObj( fullpath ); // = LStoreFile (fullpath); - *size = fileObj.position( 0, Storage::END ); - } catch ( cms::Exception & e) { - return false; - } - return true; + bool check(const std::string &proto, + const std::string &path, + const AuxSettings &, + IOOffset *size = nullptr) const override { + std::string fullpath = proto + ":" + path; + try { + LStoreFile fileObj(fullpath); // = LStoreFile (fullpath); + *size = fileObj.position(0, Storage::END); + } catch (cms::Exception &e) { + return false; + } + return true; } - }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, LStoreStorageMaker, "lstore"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, LStoreStorageMaker, "lstore"); diff --git a/Utilities/LStoreAdaptor/src/LStoreFile.cc b/Utilities/LStoreAdaptor/src/LStoreFile.cc index 21d43373b9be6..470bdbc9669b0 100644 --- a/Utilities/LStoreAdaptor/src/LStoreFile.cc +++ b/Utilities/LStoreAdaptor/src/LStoreFile.cc @@ -10,202 +10,161 @@ #include #include - - // dlsym isn't reentrant, need a locak around it pthread_mutex_t LStoreFile::m_dlopen_lock = PTHREAD_MUTEX_INITIALIZER; -LStoreFile::LStoreFile (void) - : m_fd (nullptr), - m_close (false), - m_name(), - m_library_handle(nullptr), - m_is_loaded(false), - redd_init(nullptr), - redd_read(nullptr), - redd_close(nullptr), - redd_lseek(nullptr), - redd_open(nullptr), - redd_write(nullptr), - redd_term(nullptr), - redd_errno(nullptr), - redd_strerror(nullptr) -{ - loadLibrary(); +LStoreFile::LStoreFile(void) + : m_fd(nullptr), + m_close(false), + m_name(), + m_library_handle(nullptr), + m_is_loaded(false), + redd_init(nullptr), + redd_read(nullptr), + redd_close(nullptr), + redd_lseek(nullptr), + redd_open(nullptr), + redd_write(nullptr), + redd_term(nullptr), + redd_errno(nullptr), + redd_strerror(nullptr) { + loadLibrary(); +} + +LStoreFile::LStoreFile(void *fd) + : m_fd(fd), + m_close(true), + m_name(), + m_library_handle(nullptr), + m_is_loaded(false), + redd_init(nullptr), + redd_read(nullptr), + redd_close(nullptr), + redd_lseek(nullptr), + redd_open(nullptr), + redd_write(nullptr), + redd_term(nullptr), + redd_errno(nullptr), + redd_strerror(nullptr) { + loadLibrary(); +} + +LStoreFile::LStoreFile(const char *name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) + : m_fd(nullptr), m_close(false), m_is_loaded(false) { + loadLibrary(); + open(name, flags, perms); } -LStoreFile::LStoreFile (void * fd) - : m_fd (fd), - m_close (true), - m_name(), - m_library_handle(nullptr), - m_is_loaded(false), - redd_init(nullptr), - redd_read(nullptr), - redd_close(nullptr), - redd_lseek(nullptr), - redd_open(nullptr), - redd_write(nullptr), - redd_term(nullptr), - redd_errno(nullptr), - redd_strerror(nullptr) -{ - loadLibrary(); +LStoreFile::LStoreFile(const std::string &name, int flags /* = IOFlags::OpenRead*/, int perms /* = 066 */) + : m_fd(nullptr), m_close(false), m_is_loaded(false) { + loadLibrary(); + open(name.c_str(), flags, perms); } -LStoreFile::LStoreFile (const char *name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */ ) - : m_fd (nullptr), - m_close (false), - m_is_loaded(false) -{ loadLibrary(); - open (name, flags, perms); } - -LStoreFile::LStoreFile (const std::string &name, - int flags /* = IOFlags::OpenRead*/, - int perms /* = 066 */) - : m_fd (nullptr), - m_close (false), - m_is_loaded(false) -{ loadLibrary(); - open (name.c_str (), flags, perms); } - -LStoreFile::~LStoreFile (void) -{ +LStoreFile::~LStoreFile(void) { if (m_close) - edm::LogError("LStoreFileError") - << "Destructor called on LStore file '" << m_name - << "' but the file is still open"; + edm::LogError("LStoreFileError") << "Destructor called on LStore file '" << m_name + << "' but the file is still open"; closeLibrary(); } - // Helper macro to perform dlsym() // Double cast is supposed to be more compliant // otherwise, GCC complains with the pointer conversion -#define REDD_LOAD_SYMBOL( NAME, TYPE ) dlerror();\ - NAME = reinterpret_cast(reinterpret_cast( \ - dlsym(m_library_handle, #NAME))); \ - if ( (retval = dlerror()) ) {\ - throw cms::Exception("LStoreFile::loadLibrary()") <<\ - "Failed to load dlsym LStore library: " << retval;\ - }\ - if ( NAME == NULL) {\ - throw cms::Exception("LStoreFile::loadLibrary()") <<\ - "Got a null pointer back from dlsym()\n";\ - } +#define REDD_LOAD_SYMBOL(NAME, TYPE) \ + dlerror(); \ + NAME = reinterpret_cast(reinterpret_cast(dlsym(m_library_handle, #NAME))); \ + if ((retval = dlerror())) { \ + throw cms::Exception("LStoreFile::loadLibrary()") << "Failed to load dlsym LStore library: " << retval; \ + } \ + if (NAME == NULL) { \ + throw cms::Exception("LStoreFile::loadLibrary()") << "Got a null pointer back from dlsym()\n"; \ + } void LStoreFile::loadLibrary() { - edm::LogError("LStoreFile::loadLibrary()") << "Loading library\n"; - LStoreFile::MutexWrapper lockObj( & this->m_dlopen_lock ); - // until ACCRE removes the java dependency from their client libs, - // we'll dlopen() them so they don't need to be brought along with cmssw - // if you're running LStore at your site, you will have the libs anyway - // TODO add wrappers to make this work in OSX as well (CMSSW's getting ported?) - // TODO should be easy, just need to know the "proper" way to do #if OSX - // -Melo - - m_library_handle = - dlopen("libreddnet.so", RTLD_LAZY); - if (m_library_handle == nullptr) { - throw cms::Exception("LStoreFile::loadLibrary()") - << "Can't dlopen() LStore libraries: " << dlerror(); - } - - char * retval = nullptr; - // Explicitly state the size of these values, keeps weird 64/32 bit stuff away - REDD_LOAD_SYMBOL( redd_init, int32_t(*)()); - REDD_LOAD_SYMBOL( redd_read, int64_t(*)(void *, char*, int64_t)); - REDD_LOAD_SYMBOL( redd_lseek, int64_t(*)(void*, int64_t, uint32_t)); - REDD_LOAD_SYMBOL( redd_open, void*(*)(const char*,int,int)); - REDD_LOAD_SYMBOL( redd_write, int64_t(*)(void *, const char *, int64_t)); - REDD_LOAD_SYMBOL( redd_term, int32_t(*)()); - REDD_LOAD_SYMBOL( redd_errno, int32_t(*)()); - REDD_LOAD_SYMBOL( redd_strerror, const std::string & (*)()); - - if ( (*redd_init)() ) { - throw cms::Exception("LStoreFile::loadLibrary()") - << "Error in redd_init: " << (*redd_strerror)(); - } - m_is_loaded = true; + edm::LogError("LStoreFile::loadLibrary()") << "Loading library\n"; + LStoreFile::MutexWrapper lockObj(&this->m_dlopen_lock); + // until ACCRE removes the java dependency from their client libs, + // we'll dlopen() them so they don't need to be brought along with cmssw + // if you're running LStore at your site, you will have the libs anyway + // TODO add wrappers to make this work in OSX as well (CMSSW's getting ported?) + // TODO should be easy, just need to know the "proper" way to do #if OSX + // -Melo + + m_library_handle = dlopen("libreddnet.so", RTLD_LAZY); + if (m_library_handle == nullptr) { + throw cms::Exception("LStoreFile::loadLibrary()") << "Can't dlopen() LStore libraries: " << dlerror(); + } + char *retval = nullptr; + // Explicitly state the size of these values, keeps weird 64/32 bit stuff away + REDD_LOAD_SYMBOL(redd_init, int32_t(*)()); + REDD_LOAD_SYMBOL(redd_read, int64_t(*)(void *, char *, int64_t)); + REDD_LOAD_SYMBOL(redd_lseek, int64_t(*)(void *, int64_t, uint32_t)); + REDD_LOAD_SYMBOL(redd_open, void *(*)(const char *, int, int)); + REDD_LOAD_SYMBOL(redd_write, int64_t(*)(void *, const char *, int64_t)); + REDD_LOAD_SYMBOL(redd_term, int32_t(*)()); + REDD_LOAD_SYMBOL(redd_errno, int32_t(*)()); + REDD_LOAD_SYMBOL(redd_strerror, const std::string &(*)()); + + if ((*redd_init)()) { + throw cms::Exception("LStoreFile::loadLibrary()") << "Error in redd_init: " << (*redd_strerror)(); + } + m_is_loaded = true; } void LStoreFile::closeLibrary() { - try { - LStoreFile::MutexWrapper lockObj( & this->m_dlopen_lock ); - - // What is the correct idiom for propagating error messages - // in functions that are exclusively called in destructors? - // Seriously. I have no idea - // melo - if ( m_is_loaded ) { - if ( (*redd_term)() ) { - throw cms::Exception("LStoreFile::closeLibrary()") - << "Error in redd_term: " << (*redd_strerror)(); - } - } - if ( m_library_handle != nullptr ) { - if ( dlclose( m_library_handle ) ) { - throw cms::Exception("LStoreFile::closeLibrary()") - << "Error on dlclose(): " << dlerror(); - } - } - } catch (cms::Exception & e) { - edm::LogError("LStoreFileError") - << "LStoreFile had an error in its destructor: " << e; - } - m_is_loaded = false; + try { + LStoreFile::MutexWrapper lockObj(&this->m_dlopen_lock); + + // What is the correct idiom for propagating error messages + // in functions that are exclusively called in destructors? + // Seriously. I have no idea + // melo + if (m_is_loaded) { + if ((*redd_term)()) { + throw cms::Exception("LStoreFile::closeLibrary()") << "Error in redd_term: " << (*redd_strerror)(); + } + } + if (m_library_handle != nullptr) { + if (dlclose(m_library_handle)) { + throw cms::Exception("LStoreFile::closeLibrary()") << "Error on dlclose(): " << dlerror(); + } + } + } catch (cms::Exception &e) { + edm::LogError("LStoreFileError") << "LStoreFile had an error in its destructor: " << e; + } + m_is_loaded = false; } - ////////////////////////////////////////////////////////////////////// -void -LStoreFile::create (const char *name, - bool exclusive /* = false */, - int perms /* = 066 */) -{ - open (name, - (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate - | (exclusive ? IOFlags::OpenExclusive : 0)), - perms); +void LStoreFile::create(const char *name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name, + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), + perms); } -void -LStoreFile::create (const std::string &name, - bool exclusive /* = false */, - int perms /* = 066 */) -{ - open (name.c_str (), - (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate - | (exclusive ? IOFlags::OpenExclusive : 0)), - perms); +void LStoreFile::create(const std::string &name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name.c_str(), + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), + perms); } -void -LStoreFile::open (const std::string &name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) -{ open (name.c_str (), flags, perms); } - -void -LStoreFile::open (const char *name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) -{ +void LStoreFile::open(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { + open(name.c_str(), flags, perms); +} + +void LStoreFile::open(const char *name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { // Actual open if ((name == nullptr) || (*name == 0)) - throw cms::Exception("LStoreFile::open()") - << "Cannot open a file without a name"; + throw cms::Exception("LStoreFile::open()") << "Cannot open a file without a name"; if ((flags & (IOFlags::OpenRead | IOFlags::OpenWrite)) == 0) - throw cms::Exception("LStoreFile::open()") - << "Must open file '" << name << "' at least for read or write"; + throw cms::Exception("LStoreFile::open()") << "Must open file '" << name << "' at least for read or write"; // If I am already open, close old file first if (m_fd != nullptr && m_close) - close (); + close(); // Translate our flags to system flags int openflags = 0; @@ -232,14 +191,11 @@ LStoreFile::open (const char *name, if (flags & IOFlags::OpenTruncate) openflags |= O_TRUNC; - void * newfd = nullptr; - if ((newfd = (*redd_open) (name, openflags, perms)) == nullptr) + void *newfd = nullptr; + if ((newfd = (*redd_open)(name, openflags, perms)) == nullptr) throw cms::Exception("LStoreFile::open()") - << "redd_open(name='" << name - << "', flags=0x" << std::hex << openflags - << ", permissions=0" << std::oct << perms << std::dec - << ") => error '" << (*redd_strerror)() - << "' (redd_errno=" << (*redd_errno)() << ")"; + << "redd_open(name='" << name << "', flags=0x" << std::hex << openflags << ", permissions=0" << std::oct + << perms << std::dec << ") => error '" << (*redd_strerror)() << "' (redd_errno=" << (*redd_errno)() << ")"; m_name = name; m_fd = newfd; @@ -249,23 +205,16 @@ LStoreFile::open (const char *name, edm::LogInfo("LStoreFileInfo") << "Opened " << m_name; } -void -LStoreFile::close (void) -{ - if (m_fd == nullptr) - { - edm::LogError("LStoreFileError") - << "LStoreFile::close(name='" << m_name - << "') called but the file is not open"; +void LStoreFile::close(void) { + if (m_fd == nullptr) { + edm::LogError("LStoreFileError") << "LStoreFile::close(name='" << m_name << "') called but the file is not open"; m_close = false; return; } edm::LogInfo("LStoreFile::close()") << "closing " << m_name << std::endl; - if ((*redd_close) (m_fd) == -1) - edm::LogWarning("LStoreFileWarning") - << "redd_close(name='" << m_name - << "') failed with error '" << (*redd_strerror) () - << "' (redd_errno=" << (*redd_errno)() << ")"; + if ((*redd_close)(m_fd) == -1) + edm::LogWarning("LStoreFileWarning") << "redd_close(name='" << m_name << "') failed with error '" + << (*redd_strerror)() << "' (redd_errno=" << (*redd_errno)() << ")"; m_close = false; m_fd = nullptr; @@ -274,46 +223,35 @@ LStoreFile::close (void) // edm::LogInfo("LStoreFileInfo") << "Closed " << m_name; } -void -LStoreFile::abort (void) -{ +void LStoreFile::abort(void) { if (m_fd != nullptr) - (*redd_close) (m_fd); + (*redd_close)(m_fd); m_close = false; m_fd = nullptr; } - -IOSize -LStoreFile::read (void *into, IOSize n) -{ +IOSize LStoreFile::read(void *into, IOSize n) { IOSize done = 0; - while (done < n) - { - ssize_t s = (*redd_read) (m_fd, (char *) into + done, n - done); + while (done < n) { + ssize_t s = (*redd_read)(m_fd, (char *)into + done, n - done); if (s == -1) throw cms::Exception("LStoreFile::read()") - << "redd_read(name='" << m_name << "', n=" << (n-done) - << ") failed with error '" << (*redd_strerror)() - << "' (redd_errno=" << (*redd_errno)() << ")"; - done += s; + << "redd_read(name='" << m_name << "', n=" << (n - done) << ") failed with error '" << (*redd_strerror)() + << "' (redd_errno=" << (*redd_errno)() << ")"; + done += s; } return done; } -IOSize -LStoreFile::write (const void *from, IOSize n) -{ +IOSize LStoreFile::write(const void *from, IOSize n) { IOSize done = 0; - while (done < n) - { - ssize_t s = (*redd_write) (m_fd, (const char *) from + done, n - done); + while (done < n) { + ssize_t s = (*redd_write)(m_fd, (const char *)from + done, n - done); if (s == -1) throw cms::Exception("LStoreFile::write()") - << "redd_write(name='" << m_name << "', n=" << (n-done) - << ") failed with error '" << (*redd_strerror)() - << "' (redd_errno=" << (*redd_errno)() << ")"; + << "redd_write(name='" << m_name << "', n=" << (n - done) << ") failed with error '" << (*redd_strerror)() + << "' (redd_errno=" << (*redd_errno)() << ")"; done += s; } @@ -322,58 +260,43 @@ LStoreFile::write (const void *from, IOSize n) ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -IOOffset -LStoreFile::position (IOOffset offset, Relative whence /* = SET */) -{ +IOOffset LStoreFile::position(IOOffset offset, Relative whence /* = SET */) { if (m_fd == nullptr) - throw cms::Exception("LStoreFile::position()") - << "LStoreFile::position() called on a closed file"; + throw cms::Exception("LStoreFile::position()") << "LStoreFile::position() called on a closed file"; if (whence != CURRENT && whence != SET && whence != END) - throw cms::Exception("LStoreFile::position()") - << "LStoreFile::position() called with incorrect 'whence' parameter"; + throw cms::Exception("LStoreFile::position()") << "LStoreFile::position() called with incorrect 'whence' parameter"; - IOOffset result; - uint32_t mywhence = (whence == SET ? SEEK_SET - : whence == CURRENT ? SEEK_CUR - : SEEK_END); - if ((result = (*redd_lseek) (m_fd, (off_t) offset, (uint32_t)mywhence)) == -1) + IOOffset result; + uint32_t mywhence = (whence == SET ? SEEK_SET : whence == CURRENT ? SEEK_CUR : SEEK_END); + if ((result = (*redd_lseek)(m_fd, (off_t)offset, (uint32_t)mywhence)) == -1) throw cms::Exception("LStoreFile::position()") - << "redd_lseek64(name='" << m_name << "', offset=" << offset - << ", whence=" << mywhence << ") failed with error '" - << (*redd_strerror) () << "' (redd_errno=" << (*redd_errno)() << ")"; + << "redd_lseek64(name='" << m_name << "', offset=" << offset << ", whence=" << mywhence + << ") failed with error '" << (*redd_strerror)() << "' (redd_errno=" << (*redd_errno)() << ")"; return result; } -void -LStoreFile::resize (IOOffset /* size */) -{ - throw cms::Exception("LStoreFile::resize()") - << "LStoreFile::resize(name='" << m_name << "') not implemented"; +void LStoreFile::resize(IOOffset /* size */) { + throw cms::Exception("LStoreFile::resize()") << "LStoreFile::resize(name='" << m_name << "') not implemented"; } - //////////////////////////////////////////////////////////////////// -LStoreFile::MutexWrapper::MutexWrapper( pthread_mutex_t * target ) -{ - m_lock = target; - pthread_mutex_lock( m_lock ); // never fails +LStoreFile::MutexWrapper::MutexWrapper(pthread_mutex_t *target) { + m_lock = target; + pthread_mutex_lock(m_lock); // never fails } -LStoreFile::MutexWrapper::~MutexWrapper() -{ - int retval; - if ( (retval = pthread_mutex_unlock( m_lock )) ) { - // congrats. pthread_mutex_lock failed and we're in a destructor - // I don't know what to do here - // Then again, if the mutex is jammed, things are already boned - // Cry for a second, then continue with life, I guess - // melo - - char buf[1024]; - edm::LogError("LStoreFileError") - << "LStoreFile couldn't unlock a mutex. Not good." - << strerror_r( retval, buf, 1024 ); - } +LStoreFile::MutexWrapper::~MutexWrapper() { + int retval; + if ((retval = pthread_mutex_unlock(m_lock))) { + // congrats. pthread_mutex_lock failed and we're in a destructor + // I don't know what to do here + // Then again, if the mutex is jammed, things are already boned + // Cry for a second, then continue with life, I guess + // melo + + char buf[1024]; + edm::LogError("LStoreFileError") << "LStoreFile couldn't unlock a mutex. Not good." + << strerror_r(retval, buf, 1024); + } } - diff --git a/Utilities/StaticAnalyzers/src/ArgSizeChecker.cpp b/Utilities/StaticAnalyzers/src/ArgSizeChecker.cpp index 58c70bff62618..cf8a6e61c2857 100644 --- a/Utilities/StaticAnalyzers/src/ArgSizeChecker.cpp +++ b/Utilities/StaticAnalyzers/src/ArgSizeChecker.cpp @@ -18,156 +18,156 @@ using namespace llvm; namespace clangcms { - -void ArgSizeChecker::checkPreStmt(const CXXConstructExpr *E, CheckerContext &ctx) const -{ - - CmsException m_exception; - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - - const clang::ento::PathDiagnosticLocation ELoc = - clang::ento::PathDiagnosticLocation::createBegin(E, ctx.getSourceManager(),ctx.getLocationContext()); - - if ( ! m_exception.reportGeneral( ELoc, ctx.getBugReporter() ) ) return; - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - -// llvm::errs()<getStmtClassName()<<";\t"; -// E->dumpPretty(ctx.getASTContext()); -// llvm::errs()<<"\n"; -// E->dump(); -// llvm::errs()<<"\n"; - - for (clang::Stmt::const_child_iterator I = E->child_begin(), F = E->child_end(); I!=F; ++I) { - const Expr * child = llvm::dyn_cast_or_null(*I); - if (! child) continue; - if ( llvm::isa(child->IgnoreImpCasts())) { -// (*I)->dump(); -// llvm::errs()<<"\n"; - const NamedDecl * ND = llvm::cast(child->IgnoreImpCasts())->getFoundDecl(); - if ( llvm::isa(ND)) - { -// ND->dump(); -// llvm::errs()<<"\n"; - const ParmVarDecl * PVD = llvm::cast(ND); - QualType QT = PVD->getOriginalType(); -// QT.dump(); -// llvm::errs()<<"\n"; - if (QT->isIncompleteType()||QT->isDependentType()) continue; - clang::QualType PQT = QT.getCanonicalType(); - PQT.removeLocalConst(); - if (PQT->isReferenceType() || PQT->isPointerType() - || PQT->isMemberFunctionPointerType() || PQT->isArrayType() - || PQT->isBuiltinType() || PQT->isUnionType() || PQT->isVectorType() ) continue; - uint64_t size_param = ctx.getASTContext().getTypeSize(PQT); - uint64_t max_bits=128; - if ( size_param <= max_bits ) continue; - std::string qname = QT.getAsString(); - std::string pname = PQT.getAsString(); - std::string bpname = "class boost::"; - std::string cbpname = "const class boost::"; - std::string sfname = "class std::function<"; - std::string ehname = "class edm::Handle<"; - std::string cehname = "const class edm::Handle<"; - std::string xname = "class xoap::"; - std::string ername = "class edm::Ptr<"; - std::string epname = "class edm::Ref<"; - std::string cername = "const class edm::Ptr<"; - std::string cepname = "const class edm::Ref<"; - std::string erviname = "class edm::RefVectorIterator<"; - const CXXMethodDecl * MD = llvm::dyn_cast_or_null(ctx.getCurrentAnalysisDeclContext()->getDecl()) ; -// if ( pname.substr(0,bpname.length()) == bpname || pname.substr(0,cbpname.length()) == cbpname -// || pname.substr(0,ehname.length()) == ehname || pname.substr(0,cehname.length()) == cehname -// || pname.substr(0,epname.length()) == epname || pname.substr(0,cepname.length()) == cepname -// || pname.substr(0,ername.length()) == ername || pname.substr(0,cername.length()) == cername -// || pname.substr(0,sfname.length()) == sfname || pname.substr(0,xname.length()) == xname -// || pname.substr(0,erviname.length()) == erviname ) continue; - os<<"Function parameter copied by value with size '"< max size '"<getNameAsString();os<< fname <<"' class '"<< MD->getParent()->getNameAsString(); } - os << "'\n"; - - std::string oname = "operator"; -// if ( fname.substr(0,oname.length()) == oname ) continue; - - const clang::ento::PathDiagnosticLocation DLoc = - clang::ento::PathDiagnosticLocation::createBegin(PVD, ctx.getSourceManager()); - - BugType * BT = new BugType(this,"Function parameter copied by value with size > max","ArgSize"); - std::unique_ptr report = llvm::make_unique(*BT, os.str() , DLoc); - report->addRange(PVD->getSourceRange()); - ctx.emitReport(std::move(report)); - } - } - } -} - - -void ArgSizeChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager& mgr, - BugReporter &BR) const { - const SourceManager &SM = BR.getSourceManager(); - CmsException m_exception; - PathDiagnosticLocation DLoc =PathDiagnosticLocation::createBegin( MD, SM ); - - -// return; - if ( ! m_exception.reportGeneral( DLoc, BR ) ) return; - - for (CXXMethodDecl::param_const_iterator I = MD->param_begin(), E = MD->param_end(); I!=E; I++) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - QualType QT = (*I)->getOriginalType(); - if (QT->isIncompleteType()||QT->isDependentType()) continue; - clang::QualType PQT = QT.getCanonicalType(); - PQT.removeLocalConst(); - if (PQT->isReferenceType() || PQT->isPointerType() || PQT->isMemberFunctionPointerType() - || PQT->isArrayType()|| PQT->isBuiltinType() || PQT->isUnionType() || PQT->isVectorType() ) continue; - uint64_t size_param = mgr.getASTContext().getTypeSize(PQT); - uint64_t max_bits=128; - if ( size_param <= max_bits ) continue; - std::string qname = QT.getAsString(); - std::string pname = PQT.getAsString(); - std::string bpname = "class boost::"; - std::string cbpname = "const class boost::"; - std::string sfname = "class std::function<"; - std::string ehname = "class edm::Handle<"; - std::string cehname = "const class edm::Handle<"; - std::string xname = "class xoap::"; - std::string ername = "class edm::Ptr<"; - std::string epname = "class edm::Ref<"; - std::string cername = "const class edm::Ptr<"; - std::string cepname = "const class edm::Ref<"; - std::string erviname = "class edm::RefVectorIterator<"; - -// if ( pname.substr(0,bpname.length()) == bpname || pname.substr(0,cbpname.length()) == cbpname -// || pname.substr(0,ehname.length()) == ehname || pname.substr(0,cehname.length()) == cehname -// || pname.substr(0,epname.length()) == epname || pname.substr(0,cepname.length()) == cepname -// || pname.substr(0,ername.length()) == ername || pname.substr(0,cername.length()) == cername -// || pname.substr(0,sfname.length()) == sfname || pname.substr(0,xname.length()) == xname -// || pname.substr(0,erviname.length()) == erviname ) continue; - std::string fname = MD->getNameAsString(); - os<<"Function parameter passed by value with size of parameter '"< max size '"< max size '"<getParent()->getNameAsString() - <<"'\n"; - std::string oname = "operator"; -// if ( fname.substr(0,oname.length()) == oname ) continue; - - BugType * BT = new BugType(this,"Function parameter with size > max", "ArgSize"); - std::unique_ptr report = llvm::make_unique(*BT, os.str() , DLoc); - BR.emitReport(std::move(report)); - } -} - - -} + void ArgSizeChecker::checkPreStmt(const CXXConstructExpr *E, CheckerContext &ctx) const { + CmsException m_exception; + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + + const clang::ento::PathDiagnosticLocation ELoc = + clang::ento::PathDiagnosticLocation::createBegin(E, ctx.getSourceManager(), ctx.getLocationContext()); + + if (!m_exception.reportGeneral(ELoc, ctx.getBugReporter())) + return; + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + + // llvm::errs()<getStmtClassName()<<";\t"; + // E->dumpPretty(ctx.getASTContext()); + // llvm::errs()<<"\n"; + // E->dump(); + // llvm::errs()<<"\n"; + + for (clang::Stmt::const_child_iterator I = E->child_begin(), F = E->child_end(); I != F; ++I) { + const Expr *child = llvm::dyn_cast_or_null(*I); + if (!child) + continue; + if (llvm::isa(child->IgnoreImpCasts())) { + // (*I)->dump(); + // llvm::errs()<<"\n"; + const NamedDecl *ND = llvm::cast(child->IgnoreImpCasts())->getFoundDecl(); + if (llvm::isa(ND)) { + // ND->dump(); + // llvm::errs()<<"\n"; + const ParmVarDecl *PVD = llvm::cast(ND); + QualType QT = PVD->getOriginalType(); + // QT.dump(); + // llvm::errs()<<"\n"; + if (QT->isIncompleteType() || QT->isDependentType()) + continue; + clang::QualType PQT = QT.getCanonicalType(); + PQT.removeLocalConst(); + if (PQT->isReferenceType() || PQT->isPointerType() || PQT->isMemberFunctionPointerType() || + PQT->isArrayType() || PQT->isBuiltinType() || PQT->isUnionType() || PQT->isVectorType()) + continue; + uint64_t size_param = ctx.getASTContext().getTypeSize(PQT); + uint64_t max_bits = 128; + if (size_param <= max_bits) + continue; + std::string qname = QT.getAsString(); + std::string pname = PQT.getAsString(); + std::string bpname = "class boost::"; + std::string cbpname = "const class boost::"; + std::string sfname = "class std::function<"; + std::string ehname = "class edm::Handle<"; + std::string cehname = "const class edm::Handle<"; + std::string xname = "class xoap::"; + std::string ername = "class edm::Ptr<"; + std::string epname = "class edm::Ref<"; + std::string cername = "const class edm::Ptr<"; + std::string cepname = "const class edm::Ref<"; + std::string erviname = "class edm::RefVectorIterator<"; + const CXXMethodDecl *MD = + llvm::dyn_cast_or_null(ctx.getCurrentAnalysisDeclContext()->getDecl()); + // if ( pname.substr(0,bpname.length()) == bpname || pname.substr(0,cbpname.length()) == cbpname + // || pname.substr(0,ehname.length()) == ehname || pname.substr(0,cehname.length()) == cehname + // || pname.substr(0,epname.length()) == epname || pname.substr(0,cepname.length()) == cepname + // || pname.substr(0,ername.length()) == ername || pname.substr(0,cername.length()) == cername + // || pname.substr(0,sfname.length()) == sfname || pname.substr(0,xname.length()) == xname + // || pname.substr(0,erviname.length()) == erviname ) continue; + os << "Function parameter copied by value with size '" << size_param << "' bits > max size '" << max_bits + << "' bits parameter type '" << pname << "' function '"; + + if (MD) { + std::string fname = MD->getNameAsString(); + os << fname << "' class '" << MD->getParent()->getNameAsString(); + } + os << "'\n"; + + std::string oname = "operator"; + // if ( fname.substr(0,oname.length()) == oname ) continue; + + const clang::ento::PathDiagnosticLocation DLoc = + clang::ento::PathDiagnosticLocation::createBegin(PVD, ctx.getSourceManager()); + + BugType *BT = new BugType(this, "Function parameter copied by value with size > max", "ArgSize"); + std::unique_ptr report = llvm::make_unique(*BT, os.str(), DLoc); + report->addRange(PVD->getSourceRange()); + ctx.emitReport(std::move(report)); + } + } + } + } + + void ArgSizeChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager &mgr, BugReporter &BR) const { + const SourceManager &SM = BR.getSourceManager(); + CmsException m_exception; + PathDiagnosticLocation DLoc = PathDiagnosticLocation::createBegin(MD, SM); + + // return; + if (!m_exception.reportGeneral(DLoc, BR)) + return; + + for (CXXMethodDecl::param_const_iterator I = MD->param_begin(), E = MD->param_end(); I != E; I++) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + QualType QT = (*I)->getOriginalType(); + if (QT->isIncompleteType() || QT->isDependentType()) + continue; + clang::QualType PQT = QT.getCanonicalType(); + PQT.removeLocalConst(); + if (PQT->isReferenceType() || PQT->isPointerType() || PQT->isMemberFunctionPointerType() || PQT->isArrayType() || + PQT->isBuiltinType() || PQT->isUnionType() || PQT->isVectorType()) + continue; + uint64_t size_param = mgr.getASTContext().getTypeSize(PQT); + uint64_t max_bits = 128; + if (size_param <= max_bits) + continue; + std::string qname = QT.getAsString(); + std::string pname = PQT.getAsString(); + std::string bpname = "class boost::"; + std::string cbpname = "const class boost::"; + std::string sfname = "class std::function<"; + std::string ehname = "class edm::Handle<"; + std::string cehname = "const class edm::Handle<"; + std::string xname = "class xoap::"; + std::string ername = "class edm::Ptr<"; + std::string epname = "class edm::Ref<"; + std::string cername = "const class edm::Ptr<"; + std::string cepname = "const class edm::Ref<"; + std::string erviname = "class edm::RefVectorIterator<"; + + // if ( pname.substr(0,bpname.length()) == bpname || pname.substr(0,cbpname.length()) == cbpname + // || pname.substr(0,ehname.length()) == ehname || pname.substr(0,cehname.length()) == cehname + // || pname.substr(0,epname.length()) == epname || pname.substr(0,cepname.length()) == cepname + // || pname.substr(0,ername.length()) == ername || pname.substr(0,cername.length()) == cername + // || pname.substr(0,sfname.length()) == sfname || pname.substr(0,xname.length()) == xname + // || pname.substr(0,erviname.length()) == erviname ) continue; + std::string fname = MD->getNameAsString(); + os << "Function parameter passed by value with size of parameter '" << size_param << "' bits > max size '" + << max_bits + // <<"'\n"; + // llvm::errs()<< "Function parameter passed by value with size of parameter '"< max size '"<getParent()->getNameAsString() << "'\n"; + std::string oname = "operator"; + // if ( fname.substr(0,oname.length()) == oname ) continue; + + BugType *BT = new BugType(this, "Function parameter with size > max", "ArgSize"); + std::unique_ptr report = llvm::make_unique(*BT, os.str(), DLoc); + BR.emitReport(std::move(report)); + } + } + +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/ArgSizeChecker.h b/Utilities/StaticAnalyzers/src/ArgSizeChecker.h index 74932453e423a..ec47b51df70a8 100644 --- a/Utilities/StaticAnalyzers/src/ArgSizeChecker.h +++ b/Utilities/StaticAnalyzers/src/ArgSizeChecker.h @@ -8,13 +8,14 @@ #include "CmsException.h" namespace clangcms { -class ArgSizeChecker : public clang::ento::Checker, - clang::ento::check::ASTDecl > { -public: - void checkPreStmt(const clang::CXXConstructExpr *ref, clang::ento::CheckerContext &C) const; - void checkASTDecl(const clang::CXXMethodDecl *CMD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const; -}; -} + class ArgSizeChecker : public clang::ento::Checker, + clang::ento::check::ASTDecl > { + public: + void checkPreStmt(const clang::CXXConstructExpr *ref, clang::ento::CheckerContext &C) const; + void checkASTDecl(const clang::CXXMethodDecl *CMD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + }; +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/CatchAll.cpp b/Utilities/StaticAnalyzers/src/CatchAll.cpp index 5f141a05c57b0..fc00fe31f393c 100644 --- a/Utilities/StaticAnalyzers/src/CatchAll.cpp +++ b/Utilities/StaticAnalyzers/src/CatchAll.cpp @@ -9,34 +9,41 @@ #include "CmsSupport.h" using namespace clangcms; -void CatchAll::checkASTCodeBody(const clang::Decl* D, clang::ento::AnalysisManager& AM, clang::ento::BugReporter& BR) const -{ - const char *sfile=BR.getSourceManager().getPresumedLoc(D->getLocation()).getFilename(); - if ((!sfile) || (!support::isCmsLocalFile(sfile))) return; - const clang::Stmt* s=D->getBody(); - if (!s) return; - s=process(s); - if (!s) return; +void CatchAll::checkASTCodeBody(const clang::Decl* D, + clang::ento::AnalysisManager& AM, + clang::ento::BugReporter& BR) const { + const char* sfile = BR.getSourceManager().getPresumedLoc(D->getLocation()).getFilename(); + if ((!sfile) || (!support::isCmsLocalFile(sfile))) + return; + const clang::Stmt* s = D->getBody(); + if (!s) + return; + s = process(s); + if (!s) + return; clang::ento::LocationOrAnalysisDeclContext x(AM.getAnalysisDeclContext(D)); - clang::ento::PathDiagnosticLocation DLoc = clang::ento::PathDiagnosticLocation::createBegin(s, BR.getSourceManager(),x); - BR.EmitBasicReport(D, this, "'catch(...)' in sources","CMS code rules","using 'catch(...)' is forbidden", DLoc,s->getSourceRange ()); - + clang::ento::PathDiagnosticLocation DLoc = + clang::ento::PathDiagnosticLocation::createBegin(s, BR.getSourceManager(), x); + BR.EmitBasicReport(D, + this, + "'catch(...)' in sources", + "CMS code rules", + "using 'catch(...)' is forbidden", + DLoc, + s->getSourceRange()); } -const clang::Stmt* CatchAll::process(const clang::Stmt* S) const -{ - if (clang::CXXCatchStmt::classof(S) && - checkCatchAll(static_cast(S))) +const clang::Stmt* CatchAll::process(const clang::Stmt* S) const { + if (clang::CXXCatchStmt::classof(S) && checkCatchAll(static_cast(S))) return S; - clang::Stmt::const_child_iterator b=S->child_begin (); - clang::Stmt::const_child_iterator e=S->child_end (); + clang::Stmt::const_child_iterator b = S->child_begin(); + clang::Stmt::const_child_iterator e = S->child_end(); const clang::Stmt* catchAll = nullptr; - while(b!=e) - { - if (*b) - { + while (b != e) { + if (*b) { catchAll = process(*b); - if (catchAll!=nullptr) break; + if (catchAll != nullptr) + break; } b++; } diff --git a/Utilities/StaticAnalyzers/src/CatchAll.h b/Utilities/StaticAnalyzers/src/CatchAll.h index cac40f8b5b17c..521519cc50ed3 100644 --- a/Utilities/StaticAnalyzers/src/CatchAll.h +++ b/Utilities/StaticAnalyzers/src/CatchAll.h @@ -11,13 +11,13 @@ #include namespace clangcms { - class CatchAll : public clang::ento::Checker< clang::ento::check::ASTCodeBody > - { + class CatchAll : public clang::ento::Checker { public: void checkASTCodeBody(const clang::Decl* D, clang::ento::AnalysisManager&, clang::ento::BugReporter& BR) const; + private: const clang::Stmt* process(const clang::Stmt* S) const; - inline bool checkCatchAll(const clang::CXXCatchStmt* S) const {return S->getCaughtType().isNull();} + inline bool checkCatchAll(const clang::CXXCatchStmt* S) const { return S->getCaughtType().isNull(); } }; -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/ClangCmsCheckerPluginRegister.cpp b/Utilities/StaticAnalyzers/src/ClangCmsCheckerPluginRegister.cpp index 28d96547a0825..a841731d1bc08 100644 --- a/Utilities/StaticAnalyzers/src/ClangCmsCheckerPluginRegister.cpp +++ b/Utilities/StaticAnalyzers/src/ClangCmsCheckerPluginRegister.cpp @@ -1,11 +1,9 @@ //== --==// // -// by Thomas Hauth [ Thomas.Hauth@cern.ch ] +// by Thomas Hauth [ Thomas.Hauth@cern.ch ] // //===----------------------------------------------------------------------===// - - #include "ConstCastChecker.h" #include "TrunCastChecker.h" #include "ConstCastAwayChecker.h" @@ -30,37 +28,49 @@ // register all custom checkers with clang // add new entries here if you want to create a new checker -extern "C" -void clang_registerCheckers ( clang::ento::CheckerRegistry ®istry) -{ - - registry.addChecker< clangcms::ConstCastAwayChecker>( "threadsafety.ConstCastAway", "Checks for casts which remove const qualifier and might result in thread-unsafe code" ); - registry.addChecker< clangcms::ConstCastChecker>( "threadsafety.ConstCast", "Checks for casts which remove const qualifier and might result in thread-unsafe code" ); - registry.addChecker< clangcms::TrunCastChecker>( "optional.TruncatingCast", "Checks for implicit casts where ToType is smaller than FromType which might result in truncation" ); - registry.addChecker< clangcms::StaticLocalChecker>( "threadsafety.StaticLocal", "Checks for non-const method local statics which might not be thread-safe" ); - registry.addChecker< clangcms::MutableMemberChecker>( "threadsafety.MutableMember", "Checks for members with the mutable keyword which might not be thread-safe" ); - registry.addChecker< clangcms::GlobalStaticChecker>( "threadsafety.GlobalStatic", "Checks for global non-const statics which might not be thread-safe" ); - registry.addChecker< clangcms::ClassChecker>( "optional.ClassChecker", "Checks data classes for thread safety issues" ); - registry.addChecker< clangcms::ClassDumperCT>( "optional.ClassDumperCT", "dumps template edm::Wrapper,edm::RunCache,edm::LuminosityBlockCache, and edm::GlobalCache types which define data classes " ); - registry.addChecker< clangcms::ClassDumperFT>( "optional.ClassDumperFT", "dumps macro TYPELOOKUP_DATA_REG types which define data classes" ); - registry.addChecker< clangcms::ClassDumperInherit>( "optional.ClassDumperInherit", "Dumps classes inheriting from data classes" ); - registry.addChecker< clangcms::ClassDumper>( "optional.ClassDumper", "Dumps class memmbers and base classes " ); - registry.addChecker< clangcms::FiniteMathChecker>( "cms.NonFiniteMath", "Reports usage of isnan and isinf." ); - registry.addChecker< clangcms::UsingNamespace>( "cms.CodeRules.UsingNamespace", "Checks for 'using namespace' or 'using std::' in header files" ); - registry.addChecker< clangcms::CatchAll>( "cms.CodeRules.CatchAll", "Checks for 'catch(...)' in source files" ); - registry.addChecker< clangcms::edmChecker>( "cms.edmChecker", "Flags classes inheriting from edm::EDProducer,edm::EDFilter,edm::Analyzer or edm::OutputModule" ); - registry.addChecker< clangcms::getByChecker>( "optional.getByChecker", "Checks for calls to edm::getByLabel or edm::getManyByType and reports edm::Handle type passed" ); - registry.addChecker< clangcms::ArgSizeChecker>( "optional.ArgSize", "Reports args passed by value with size>4k." ); - registry.addChecker< clangcms::FunctionChecker>( "cms.FunctionChecker", "Reports functions which access non-const statics" ); - registry.addChecker< clangcms::FunctionDumper>( "cms.FunctionDumper", "Reports function calls and overrides" ); - registry.addChecker< clangcms::EDMPluginDumper>( "optional.EDMPluginDumper", "Dumps macro DEFINE_EDM_PLUGIN types" ); - registry.addChecker< clangcms::ThrUnsafeFCallChecker>( "cms.ThrUnsafeFCallChecker", "Reports calls of known thread unsafe functions" ); - registry.addChecker< clangcms::getParamDumper>( "optional.getParamDumper", "Dumps out calls to edm::ParamaterSet:: getParameter and getUntrackedParameter" ); +extern "C" void clang_registerCheckers(clang::ento::CheckerRegistry ®istry) { + registry.addChecker( + "threadsafety.ConstCastAway", + "Checks for casts which remove const qualifier and might result in thread-unsafe code"); + registry.addChecker( + "threadsafety.ConstCast", "Checks for casts which remove const qualifier and might result in thread-unsafe code"); + registry.addChecker( + "optional.TruncatingCast", + "Checks for implicit casts where ToType is smaller than FromType which might result in truncation"); + registry.addChecker( + "threadsafety.StaticLocal", "Checks for non-const method local statics which might not be thread-safe"); + registry.addChecker( + "threadsafety.MutableMember", "Checks for members with the mutable keyword which might not be thread-safe"); + registry.addChecker( + "threadsafety.GlobalStatic", "Checks for global non-const statics which might not be thread-safe"); + registry.addChecker("optional.ClassChecker", "Checks data classes for thread safety issues"); + registry.addChecker("optional.ClassDumperCT", + "dumps template edm::Wrapper,edm::RunCache,edm::LuminosityBlockCache, " + "and edm::GlobalCache types which define data classes "); + registry.addChecker("optional.ClassDumperFT", + "dumps macro TYPELOOKUP_DATA_REG types which define data classes"); + registry.addChecker("optional.ClassDumperInherit", + "Dumps classes inheriting from data classes"); + registry.addChecker("optional.ClassDumper", "Dumps class memmbers and base classes "); + registry.addChecker("cms.NonFiniteMath", "Reports usage of isnan and isinf."); + registry.addChecker("cms.CodeRules.UsingNamespace", + "Checks for 'using namespace' or 'using std::' in header files"); + registry.addChecker("cms.CodeRules.CatchAll", "Checks for 'catch(...)' in source files"); + registry.addChecker( + "cms.edmChecker", + "Flags classes inheriting from edm::EDProducer,edm::EDFilter,edm::Analyzer or edm::OutputModule"); + registry.addChecker( + "optional.getByChecker", + "Checks for calls to edm::getByLabel or edm::getManyByType and reports edm::Handle type passed"); + registry.addChecker("optional.ArgSize", "Reports args passed by value with size>4k."); + registry.addChecker("cms.FunctionChecker", + "Reports functions which access non-const statics"); + registry.addChecker("cms.FunctionDumper", "Reports function calls and overrides"); + registry.addChecker("optional.EDMPluginDumper", "Dumps macro DEFINE_EDM_PLUGIN types"); + registry.addChecker("cms.ThrUnsafeFCallChecker", + "Reports calls of known thread unsafe functions"); + registry.addChecker( + "optional.getParamDumper", "Dumps out calls to edm::ParamaterSet:: getParameter and getUntrackedParameter"); } -extern "C" -const char clang_analyzerAPIVersionString[] = CLANG_ANALYZER_API_VERSION_STRING; - - - - +extern "C" const char clang_analyzerAPIVersionString[] = CLANG_ANALYZER_API_VERSION_STRING; diff --git a/Utilities/StaticAnalyzers/src/ClassChecker.cpp b/Utilities/StaticAnalyzers/src/ClassChecker.cpp index e64f428d608b3..c7913644c6336 100644 --- a/Utilities/StaticAnalyzers/src/ClassChecker.cpp +++ b/Utilities/StaticAnalyzers/src/ClassChecker.cpp @@ -34,7 +34,7 @@ #include #include #include -#include +#include using namespace clang; using namespace clang::ento; @@ -42,684 +42,726 @@ using namespace llvm; namespace clangcms { + void writeLog(std::string ostring) { + std::string tname = "class-checker.txt.unsorted"; + support::writeLog(ostring, tname); + return; + } -void writeLog(std::string ostring) { - std::string tname ="class-checker.txt.unsorted"; - support::writeLog(ostring,tname); - return; -} - -class WalkAST : public clang::StmtVisitor { - const CheckerBase *Checker; - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext *AC; - const CXXMethodDecl *AD; - typedef const clang::CXXMemberCallExpr * WorkListUnit; - typedef clang::SmallVector DFSWorkList; - - - /// A vector representing the worklist which has a chain of CallExprs. - DFSWorkList WList; - - // PreVisited : A CallExpr to this FunctionDecl is in the worklist, but the - // body has not been visited yet. - // PostVisited : A CallExpr to this FunctionDecl is in the worklist, and the - // body has been visited. - enum Kind { NotVisited, - Visiting, /**< A CallExpr to this FunctionDecl is in the + class WalkAST : public clang::StmtVisitor { + const CheckerBase *Checker; + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; + const CXXMethodDecl *AD; + typedef const clang::CXXMemberCallExpr *WorkListUnit; + typedef clang::SmallVector DFSWorkList; + + /// A vector representing the worklist which has a chain of CallExprs. + DFSWorkList WList; + + // PreVisited : A CallExpr to this FunctionDecl is in the worklist, but the + // body has not been visited yet. + // PostVisited : A CallExpr to this FunctionDecl is in the worklist, and the + // body has been visited. + enum Kind { + NotVisited, + Visiting, /**< A CallExpr to this FunctionDecl is in the worklist, but the body has not yet been visited. */ - Visited /**< A CallExpr to this FunctionDecl is in the + Visited /**< A CallExpr to this FunctionDecl is in the worklist, and the body has been visited. */ - }; - - /// A DenseMap that records visited states of FunctionDecls. - llvm::DenseMap VisitedFunctions; - - /// The CallExpr whose body is currently being visited. This is used for - /// generating bug reports. This is null while visiting the body of a - /// constructor or destructor. - const clang::CXXMemberCallExpr *visitingCallExpr; - const char *sfile; -public: - WalkAST(const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac, const CXXMethodDecl * fd, const char* file) - : Checker(checker), - BR(br), - AC(ac), - AD(fd), - visitingCallExpr(nullptr),sfile(file) {} - - bool hasWork() const { return !WList.empty(); } - - /// This method adds a CallExpr to the worklist - void Enqueue(WorkListUnit WLUnit) { - Kind &K = VisitedFunctions[WLUnit]; - if ((K = Visiting)) return; - K = Visiting; - WList.push_back(WLUnit); - } - - /// This method returns an item from the worklist without removing it. - WorkListUnit Dequeue() { - assert(!WList.empty()); - return WList.back(); - } - - void Execute() { - if (WList.empty()) return; - WorkListUnit WLUnit = Dequeue(); - const clang::CXXMethodDecl *FD = WLUnit->getMethodDecl(); - if (!FD) return; - llvm::SaveAndRestore SaveCall(visitingCallExpr, WLUnit); - if (FD && FD->hasBody()) Visit(FD->getBody()); - VisitedFunctions[WLUnit] = Visited; - WList.pop_back(); - } - - const clang::Stmt * ParentStmt(const Stmt *S) { - const Stmt * P = AC->getParentMap().getParentIgnoreParens(S); - if (!P) return nullptr; - return P; - } + }; + + /// A DenseMap that records visited states of FunctionDecls. + llvm::DenseMap VisitedFunctions; + + /// The CallExpr whose body is currently being visited. This is used for + /// generating bug reports. This is null while visiting the body of a + /// constructor or destructor. + const clang::CXXMemberCallExpr *visitingCallExpr; + const char *sfile; + + public: + WalkAST(const CheckerBase *checker, + clang::ento::BugReporter &br, + clang::AnalysisDeclContext *ac, + const CXXMethodDecl *fd, + const char *file) + : Checker(checker), BR(br), AC(ac), AD(fd), visitingCallExpr(nullptr), sfile(file) {} + + bool hasWork() const { return !WList.empty(); } + + /// This method adds a CallExpr to the worklist + void Enqueue(WorkListUnit WLUnit) { + Kind &K = VisitedFunctions[WLUnit]; + if ((K = Visiting)) + return; + K = Visiting; + WList.push_back(WLUnit); + } - void WListDump(llvm::raw_ostream & os) { - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - if (!WList.empty()) { - for (llvm::SmallVectorImpl::iterator - I = WList.begin(), E = WList.end(); I != E; I++) { - (*I)->printPretty(os, nullptr , Policy); - os <<" "; - } - } - } + /// This method returns an item from the worklist without removing it. + WorkListUnit Dequeue() { + assert(!WList.empty()); + return WList.back(); + } - // Stmt visitor methods. - void VisitChildren(clang::Stmt *S); - void VisitStmt( clang::Stmt *S) { VisitChildren(S); } - void VisitMemberExpr(clang::MemberExpr *E); - void VisitCXXMemberCallExpr( clang::CXXMemberCallExpr *CE); - void VisitDeclRefExpr(clang::DeclRefExpr * DRE); - void VisitCXXConstCastExpr(clang::CXXConstCastExpr *CCE); - void ReportDeclRef( const clang::DeclRefExpr * DRE); - void CheckCXXOperatorCallExpr(const clang::CXXOperatorCallExpr *CE,const clang::MemberExpr *E); - void CheckBinaryOperator(const clang::BinaryOperator * BO,const clang::MemberExpr *E); - void CheckUnaryOperator(const clang::UnaryOperator * UO,const clang::MemberExpr *E); - void CheckExplicitCastExpr(const clang::ExplicitCastExpr * CE,const clang::MemberExpr *E); - void CheckReturnStmt(const clang::ReturnStmt * RS, const clang::MemberExpr *E); - void ReportCast(const clang::ExplicitCastExpr *CE); - void ReportCall(const clang::CXXMemberCallExpr *CE); - void ReportMember(const clang::MemberExpr *ME); - void ReportCallReturn(const clang::ReturnStmt * RS); - void ReportCallArg(const clang::CXXMemberCallExpr *CE, const int i); -}; - -//===----------------------------------------------------------------------===// -// AST walking. -//===----------------------------------------------------------------------===// - - - - -void WalkAST::VisitChildren( clang::Stmt *S) { - for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); + void Execute() { + if (WList.empty()) + return; + WorkListUnit WLUnit = Dequeue(); + const clang::CXXMethodDecl *FD = WLUnit->getMethodDecl(); + if (!FD) + return; + llvm::SaveAndRestore SaveCall(visitingCallExpr, WLUnit); + if (FD && FD->hasBody()) + Visit(FD->getBody()); + VisitedFunctions[WLUnit] = Visited; + WList.pop_back(); } -} + const clang::Stmt *ParentStmt(const Stmt *S) { + const Stmt *P = AC->getParentMap().getParentIgnoreParens(S); + if (!P) + return nullptr; + return P; + } -void WalkAST::CheckBinaryOperator(const clang::BinaryOperator * BO,const clang::MemberExpr *IME) { - if (BO->isAssignmentOp()) { + void WListDump(llvm::raw_ostream &os) { + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + if (!WList.empty()) { + for (llvm::SmallVectorImpl::iterator I = WList.begin(), E = WList.end(); + I != E; + I++) { + (*I)->printPretty(os, nullptr, Policy); + os << " "; + } + } + } - if (clang::MemberExpr * ME = dyn_cast_or_null(BO->getLHS())){ - if (ME->isImplicitAccess()) ReportMember(ME); - } - } else { - if (clang::UnaryOperator * UO = llvm::dyn_cast_or_null(BO->getLHS()->IgnoreParenImpCasts()) ) { - if (UO->getOpcode() == clang::UnaryOperatorKind::UO_Deref) { - if (clang::MemberExpr * ME = dyn_cast_or_null(UO->getSubExpr()->IgnoreParenImpCasts())){ - if (ME->isImplicitAccess()) ReportMember(ME); - } - if (clang::DeclRefExpr * DRE =dyn_cast_or_null(UO->getSubExpr()->IgnoreParenImpCasts())){ - if (const clang::VarDecl * D = llvm::dyn_cast_or_null(DRE->getDecl())) { - clang::QualType t = D->getType(); - const clang::Expr * E = llvm::dyn_cast_or_null(D->getInit()); - if (E && t->isPointerType() ) { - const clang::MemberExpr * ME = dyn_cast_or_null(E->IgnoreParenImpCasts()); - if (ME && ME->isImplicitAccess()) ReportMember(ME); - } - - } - } - } - } - } -} - -void WalkAST::CheckUnaryOperator(const clang::UnaryOperator * UO,const clang::MemberExpr *E) { - if (UO->isIncrementDecrementOp()) { - if (clang::MemberExpr * ME = dyn_cast_or_null(UO->getSubExpr()->IgnoreParenImpCasts())) ReportMember(ME); - } -} - - -void WalkAST::CheckCXXOperatorCallExpr(const clang::CXXOperatorCallExpr *OCE,const clang::MemberExpr *E) { - switch ( OCE->getOperator() ) { - - case OO_Equal: - case OO_PlusEqual: - case OO_MinusEqual: - case OO_StarEqual: - case OO_SlashEqual: - case OO_PercentEqual: - case OO_AmpEqual: - case OO_PipeEqual: - case OO_LessLessEqual: - case OO_GreaterGreaterEqual: - if (const clang::MemberExpr * ME = dyn_cast_or_null((*OCE->arg_begin())->IgnoreParenImpCasts())){ - if (ME->isImplicitAccess()) - ReportMember(ME); - } + // Stmt visitor methods. + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + void VisitMemberExpr(clang::MemberExpr *E); + void VisitCXXMemberCallExpr(clang::CXXMemberCallExpr *CE); + void VisitDeclRefExpr(clang::DeclRefExpr *DRE); + void VisitCXXConstCastExpr(clang::CXXConstCastExpr *CCE); + void ReportDeclRef(const clang::DeclRefExpr *DRE); + void CheckCXXOperatorCallExpr(const clang::CXXOperatorCallExpr *CE, const clang::MemberExpr *E); + void CheckBinaryOperator(const clang::BinaryOperator *BO, const clang::MemberExpr *E); + void CheckUnaryOperator(const clang::UnaryOperator *UO, const clang::MemberExpr *E); + void CheckExplicitCastExpr(const clang::ExplicitCastExpr *CE, const clang::MemberExpr *E); + void CheckReturnStmt(const clang::ReturnStmt *RS, const clang::MemberExpr *E); + void ReportCast(const clang::ExplicitCastExpr *CE); + void ReportCall(const clang::CXXMemberCallExpr *CE); + void ReportMember(const clang::MemberExpr *ME); + void ReportCallReturn(const clang::ReturnStmt *RS); + void ReportCallArg(const clang::CXXMemberCallExpr *CE, const int i); + }; - case OO_PlusPlus: - case OO_MinusMinus: - if (const clang::MemberExpr * ME = dyn_cast_or_null(OCE->getCallee()->IgnoreParenCasts())) { - if (ME->isImplicitAccess()) - ReportMember(ME); - } + //===----------------------------------------------------------------------===// + // AST walking. + //===----------------------------------------------------------------------===// - default: return; + void WalkAST::VisitChildren(clang::Stmt *S) { + for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } } -} - - -void WalkAST::CheckExplicitCastExpr(const clang::ExplicitCastExpr * CE,const clang::MemberExpr *ME){ - if (! ( clang::CStyleCastExpr::classof(CE) || clang::CXXConstCastExpr::classof(CE) )) return; - const clang::Expr *E = CE->getSubExpr(); - clang::ASTContext &Ctx = AC->getASTContext(); - clang::QualType OrigTy = Ctx.getCanonicalType(E->getType()); - clang::QualType ToTy = Ctx.getCanonicalType(CE->getType()); - - if ( support::isConst( OrigTy ) && ! support::isConst(ToTy) ) - ReportCast(CE); - -} - - -void WalkAST::CheckReturnStmt(const clang::ReturnStmt * RS, const clang::MemberExpr * E){ - if (const clang::Expr * RE = RS->getRetValue()) { - clang::ASTContext &Ctx = AC->getASTContext(); - const clang::CXXMethodDecl * MD; - if (visitingCallExpr) - MD = visitingCallExpr->getMethodDecl(); - else - MD = llvm::dyn_cast(AD); - if ( llvm::isa(RE) ) return; - clang::QualType RQT = MD->getCallResultType(); - clang::QualType RTy = Ctx.getCanonicalType(RQT); - if ( (RTy->isPointerType() || RTy->isReferenceType() ) ) { - if( !support::isConst(RTy) ) { - ReportCallReturn(RS); - } + void WalkAST::CheckBinaryOperator(const clang::BinaryOperator *BO, const clang::MemberExpr *IME) { + if (BO->isAssignmentOp()) { + if (clang::MemberExpr *ME = dyn_cast_or_null(BO->getLHS())) { + if (ME->isImplicitAccess()) + ReportMember(ME); + } + } else { + if (clang::UnaryOperator *UO = + llvm::dyn_cast_or_null(BO->getLHS()->IgnoreParenImpCasts())) { + if (UO->getOpcode() == clang::UnaryOperatorKind::UO_Deref) { + if (clang::MemberExpr *ME = dyn_cast_or_null(UO->getSubExpr()->IgnoreParenImpCasts())) { + if (ME->isImplicitAccess()) + ReportMember(ME); } - std::string svname = "const class std::vector<"; - std::string rtname = RTy.getAsString(); - if ( (RTy->isReferenceType() || RTy ->isRecordType() ) && support::isConst(RTy) && rtname.substr(0,svname.length()) == svname ) { - const CXXRecordDecl *RD; - if ( RTy->isRecordType() ) RD = RTy->getAsCXXRecordDecl(); - else RD = RTy->getPointeeCXXRecordDecl(); - const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); - for (unsigned J = 0, F = SD->getTemplateArgs().size(); J!=F; ++J) { - if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { - const QualType QAT = SD->getTemplateArgs().get(J).getAsType(); - if ( QAT->isPointerType() && !support::isConst(QAT)) { - std::string buf; - llvm::raw_string_ostream os(buf); - std::string mname = support::getQualifiedName(*MD); - support::fixAnonNS(mname,sfile); - std::string pname = support::getQualifiedName(*(MD->getParent())); - support::fixAnonNS(pname,sfile); - os << mname << " is a const member function that returns a const std::vector<*> or const std::vector<*>& "<(UO->getSubExpr()->IgnoreParenImpCasts())) { + if (const clang::VarDecl *D = llvm::dyn_cast_or_null(DRE->getDecl())) { + clang::QualType t = D->getType(); + const clang::Expr *E = llvm::dyn_cast_or_null(D->getInit()); + if (E && t->isPointerType()) { + const clang::MemberExpr *ME = dyn_cast_or_null(E->IgnoreParenImpCasts()); + if (ME && ME->isImplicitAccess()) + ReportMember(ME); + } + } } - } -} - - -void WalkAST::VisitCXXConstCastExpr(clang::CXXConstCastExpr *CCE) { - - clang::ento::PathDiagnosticLocation CELoc = clang::ento::PathDiagnosticLocation::createBegin(CCE, BR.getSourceManager(),AC); - std::string buf; - llvm::raw_string_ostream os(buf); - os <<"const_cast used\n"; - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str()+"."; - writeLog(tolog); - BugType * BT = new BugType(Checker,"const_cast used in const function ","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique (*BT,tolog,CELoc); - BR.emitReport(std::move(R)); - return; -} - -void WalkAST::VisitDeclRefExpr( clang::DeclRefExpr * DRE) { - if (clang::VarDecl * D = llvm::dyn_cast_or_null(DRE->getDecl()) ) { - clang::SourceLocation SL = DRE->getLocStart(); - if (BR.getSourceManager().isInSystemHeader(SL) || BR.getSourceManager().isInExternCSystemHeader(SL)) return; - if ( support::isSafeClassName( D->getCanonicalDecl()->getQualifiedNameAsString() ) ) return; - ReportDeclRef( DRE ); + } + } + } } -} - -void WalkAST::ReportDeclRef( const clang::DeclRefExpr * DRE) { - - if (const clang::VarDecl * D = llvm::dyn_cast_or_null(DRE->getDecl())) { - clang::QualType t = D->getType(); - const clang::Stmt * PS = ParentStmt(DRE); - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - - - clang::ento::PathDiagnosticLocation CELoc = clang::ento::PathDiagnosticLocation::createBegin(DRE, BR.getSourceManager(),AC); - if ( support::isSafeClassName( t.getCanonicalType().getAsString() ) ) return; - if ( D->hasAttr() || D->hasAttr()) return; - if ( D->isStaticLocal() && D->getTSCSpec() != clang::ThreadStorageClassSpecifier::TSCS_thread_local && ! support::isConst( t ) ) - { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "Non-const variable '" << support::getQualifiedName(*D) << "' is static local and accessed in statement '"; - PS->printPretty(os,nullptr,Policy); - os << "'.\n"; - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - writeLog(tolog); - BugType * BT = new BugType(Checker,"ClassChecker : non-const static local variable accessed","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); - return; - } - - if ( D->isStaticDataMember() && D->getTSCSpec() != clang::ThreadStorageClassSpecifier::TSCS_thread_local && ! support::isConst( t ) ) - { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "Non-const variable '" << support::getQualifiedName(*D) << "' is static member data and accessed in statement '"; - PS->printPretty(os,nullptr,Policy); - os << "'.\n"; - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - writeLog(tolog); - BugType * BT = new BugType(Checker,"Non-const static member variable accessed","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); - return; - } + void WalkAST::CheckUnaryOperator(const clang::UnaryOperator *UO, const clang::MemberExpr *E) { + if (UO->isIncrementDecrementOp()) { + if (clang::MemberExpr *ME = dyn_cast_or_null(UO->getSubExpr()->IgnoreParenImpCasts())) + ReportMember(ME); + } + } - if ( (D->getStorageClass() == clang::SC_Static) && - !D->isStaticDataMember() && - !D->isStaticLocal() && - !support::isConst( t ) ) - { + void WalkAST::CheckCXXOperatorCallExpr(const clang::CXXOperatorCallExpr *OCE, const clang::MemberExpr *E) { + switch (OCE->getOperator()) { + case OO_Equal: + case OO_PlusEqual: + case OO_MinusEqual: + case OO_StarEqual: + case OO_SlashEqual: + case OO_PercentEqual: + case OO_AmpEqual: + case OO_PipeEqual: + case OO_LessLessEqual: + case OO_GreaterGreaterEqual: + if (const clang::MemberExpr *ME = + dyn_cast_or_null((*OCE->arg_begin())->IgnoreParenImpCasts())) { + if (ME->isImplicitAccess()) + ReportMember(ME); + } - std::string buf; - llvm::raw_string_ostream os(buf); - os << "Non-const variable '" << support::getQualifiedName(*D) << "' is global static and accessed in statement '"; - PS->printPretty(os,nullptr,Policy); - os << "'.\n"; - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - writeLog(tolog); - BugType * BT = new BugType(Checker,"Non-const global static variable accessed","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); - return; - - } + case OO_PlusPlus: + case OO_MinusMinus: + if (const clang::MemberExpr *ME = dyn_cast_or_null(OCE->getCallee()->IgnoreParenCasts())) { + if (ME->isImplicitAccess()) + ReportMember(ME); + } + default: + return; + } } -} + void WalkAST::CheckExplicitCastExpr(const clang::ExplicitCastExpr *CE, const clang::MemberExpr *ME) { + if (!(clang::CStyleCastExpr::classof(CE) || clang::CXXConstCastExpr::classof(CE))) + return; + const clang::Expr *E = CE->getSubExpr(); + clang::ASTContext &Ctx = AC->getASTContext(); + clang::QualType OrigTy = Ctx.getCanonicalType(E->getType()); + clang::QualType ToTy = Ctx.getCanonicalType(CE->getType()); -void WalkAST::VisitMemberExpr( clang::MemberExpr *ME) { + if (support::isConst(OrigTy) && !support::isConst(ToTy)) + ReportCast(CE); + } - clang::SourceLocation SL = ME->getExprLoc(); - if (BR.getSourceManager().isInSystemHeader(SL) || BR.getSourceManager().isInExternCSystemHeader(SL)) return; - const ValueDecl * D = ME->getMemberDecl(); - if ( D->hasAttr() || D->hasAttr()) return; - if (!(ME->isImplicitAccess())) return; - Stmt * P = AC->getParentMap().getParent(ME); - while (AC->getParentMap().hasParent(P)) { - if (const clang::UnaryOperator * UO = llvm::dyn_cast_or_null(P)) - { WalkAST::CheckUnaryOperator(UO,ME);} - if (const clang::BinaryOperator * BO = llvm::dyn_cast_or_null(P)) - { WalkAST::CheckBinaryOperator(BO,ME);} - if (const clang::CXXOperatorCallExpr *OCE = llvm::dyn_cast_or_null(P)) - { WalkAST::CheckCXXOperatorCallExpr(OCE,ME);} - if (const clang::ExplicitCastExpr * CE = llvm::dyn_cast_or_null(P)) - { WalkAST::CheckExplicitCastExpr(CE,ME);} - if (const clang::ReturnStmt * RS = llvm::dyn_cast_or_null(P)) - { WalkAST::CheckReturnStmt(RS,ME); } - if (const clang::CXXConstCastExpr * CCE = llvm::dyn_cast_or_null(P)) - { WalkAST::ReportCast(CCE);} - const clang::CXXNewExpr * NE = llvm::dyn_cast_or_null(P);if (NE) break; - P = AC->getParentMap().getParent(P); - } -} + void WalkAST::CheckReturnStmt(const clang::ReturnStmt *RS, const clang::MemberExpr *E) { + if (const clang::Expr *RE = RS->getRetValue()) { + clang::ASTContext &Ctx = AC->getASTContext(); + const clang::CXXMethodDecl *MD; + if (visitingCallExpr) + MD = visitingCallExpr->getMethodDecl(); + else + MD = llvm::dyn_cast(AD); + if (llvm::isa(RE)) + return; + clang::QualType RQT = MD->getCallResultType(); + clang::QualType RTy = Ctx.getCanonicalType(RQT); + if ((RTy->isPointerType() || RTy->isReferenceType())) { + if (!support::isConst(RTy)) { + ReportCallReturn(RS); + } + } + std::string svname = "const class std::vector<"; + std::string rtname = RTy.getAsString(); + if ((RTy->isReferenceType() || RTy->isRecordType()) && support::isConst(RTy) && + rtname.substr(0, svname.length()) == svname) { + const CXXRecordDecl *RD; + if (RTy->isRecordType()) + RD = RTy->getAsCXXRecordDecl(); + else + RD = RTy->getPointeeCXXRecordDecl(); + const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); + for (unsigned J = 0, F = SD->getTemplateArgs().size(); J != F; ++J) { + if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { + const QualType QAT = SD->getTemplateArgs().get(J).getAsType(); + if (QAT->isPointerType() && !support::isConst(QAT)) { + std::string buf; + llvm::raw_string_ostream os(buf); + std::string mname = support::getQualifiedName(*MD); + support::fixAnonNS(mname, sfile); + std::string pname = support::getQualifiedName(*(MD->getParent())); + support::fixAnonNS(pname, sfile); + os << mname << " is a const member function that returns a const std::vector<*> or const std::vector<*>& " + << rtname << "\n"; + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + ReportCallReturn(RS); + } + } + } + } + } + } + void WalkAST::VisitCXXConstCastExpr(clang::CXXConstCastExpr *CCE) { + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(CCE, BR.getSourceManager(), AC); + std::string buf; + llvm::raw_string_ostream os(buf); + os << "const_cast used\n"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str() + "."; + writeLog(tolog); + BugType *BT = new BugType(Checker, "const_cast used in const function ", "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, tolog, CELoc); + BR.emitReport(std::move(R)); + return; + } + void WalkAST::VisitDeclRefExpr(clang::DeclRefExpr *DRE) { + if (clang::VarDecl *D = llvm::dyn_cast_or_null(DRE->getDecl())) { + clang::SourceLocation SL = DRE->getLocStart(); + if (BR.getSourceManager().isInSystemHeader(SL) || BR.getSourceManager().isInExternCSystemHeader(SL)) + return; + if (support::isSafeClassName(D->getCanonicalDecl()->getQualifiedNameAsString())) + return; + ReportDeclRef(DRE); + } + } + void WalkAST::ReportDeclRef(const clang::DeclRefExpr *DRE) { + if (const clang::VarDecl *D = llvm::dyn_cast_or_null(DRE->getDecl())) { + clang::QualType t = D->getType(); + const clang::Stmt *PS = ParentStmt(DRE); + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(DRE, BR.getSourceManager(), AC); + if (support::isSafeClassName(t.getCanonicalType().getAsString())) + return; + if (D->hasAttr() || D->hasAttr()) + return; + if (D->isStaticLocal() && D->getTSCSpec() != clang::ThreadStorageClassSpecifier::TSCS_thread_local && + !support::isConst(t)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "Non-const variable '" << support::getQualifiedName(*D) + << "' is static local and accessed in statement '"; + PS->printPretty(os, nullptr, Policy); + os << "'.\n"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + BugType *BT = new BugType( + Checker, "ClassChecker : non-const static local variable accessed", "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); + return; + } + + if (D->isStaticDataMember() && D->getTSCSpec() != clang::ThreadStorageClassSpecifier::TSCS_thread_local && + !support::isConst(t)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "Non-const variable '" << support::getQualifiedName(*D) + << "' is static member data and accessed in statement '"; + PS->printPretty(os, nullptr, Policy); + os << "'.\n"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + BugType *BT = new BugType(Checker, "Non-const static member variable accessed", "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); + return; + } + + if ((D->getStorageClass() == clang::SC_Static) && !D->isStaticDataMember() && !D->isStaticLocal() && + !support::isConst(t)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "Non-const variable '" << support::getQualifiedName(*D) + << "' is global static and accessed in statement '"; + PS->printPretty(os, nullptr, Policy); + os << "'.\n"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + BugType *BT = new BugType(Checker, "Non-const global static variable accessed", "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); + return; + } + } + } -void WalkAST::VisitCXXMemberCallExpr( clang::CXXMemberCallExpr *CE) { + void WalkAST::VisitMemberExpr(clang::MemberExpr *ME) { + clang::SourceLocation SL = ME->getExprLoc(); + if (BR.getSourceManager().isInSystemHeader(SL) || BR.getSourceManager().isInExternCSystemHeader(SL)) + return; + const ValueDecl *D = ME->getMemberDecl(); + if (D->hasAttr() || D->hasAttr()) + return; + if (!(ME->isImplicitAccess())) + return; + Stmt *P = AC->getParentMap().getParent(ME); + while (AC->getParentMap().hasParent(P)) { + if (const clang::UnaryOperator *UO = llvm::dyn_cast_or_null(P)) { + WalkAST::CheckUnaryOperator(UO, ME); + } + if (const clang::BinaryOperator *BO = llvm::dyn_cast_or_null(P)) { + WalkAST::CheckBinaryOperator(BO, ME); + } + if (const clang::CXXOperatorCallExpr *OCE = llvm::dyn_cast_or_null(P)) { + WalkAST::CheckCXXOperatorCallExpr(OCE, ME); + } + if (const clang::ExplicitCastExpr *CE = llvm::dyn_cast_or_null(P)) { + WalkAST::CheckExplicitCastExpr(CE, ME); + } + if (const clang::ReturnStmt *RS = llvm::dyn_cast_or_null(P)) { + WalkAST::CheckReturnStmt(RS, ME); + } + if (const clang::CXXConstCastExpr *CCE = llvm::dyn_cast_or_null(P)) { + WalkAST::ReportCast(CCE); + } + const clang::CXXNewExpr *NE = llvm::dyn_cast_or_null(P); + if (NE) + break; + P = AC->getParentMap().getParent(P); + } + } - if (BR.getSourceManager().isInSystemHeader(CE->getExprLoc()) || BR.getSourceManager().isInExternCSystemHeader(CE->getExprLoc())) return; + void WalkAST::VisitCXXMemberCallExpr(clang::CXXMemberCallExpr *CE) { + if (BR.getSourceManager().isInSystemHeader(CE->getExprLoc()) || + BR.getSourceManager().isInExternCSystemHeader(CE->getExprLoc())) + return; - clang::CXXMethodDecl * MD = CE->getMethodDecl(); + clang::CXXMethodDecl *MD = CE->getMethodDecl(); - if (! MD) return; + if (!MD) + return; - Enqueue(CE); - Execute(); - Visit(CE->getImplicitObjectArgument()->IgnoreParenCasts()); + Enqueue(CE); + Execute(); + Visit(CE->getImplicitObjectArgument()->IgnoreParenCasts()); - const Expr * IOA = CE->getImplicitObjectArgument()->IgnoreParenCasts(); - const MemberExpr * ME = dyn_cast_or_null(IOA); - if ( !MD->isConst() && ME && ME->isImplicitAccess() ) ReportCall(CE); + const Expr *IOA = CE->getImplicitObjectArgument()->IgnoreParenCasts(); + const MemberExpr *ME = dyn_cast_or_null(IOA); + if (!MD->isConst() && ME && ME->isImplicitAccess()) + ReportCall(CE); - for(int i=0, j=CE->getNumArgs(); igetArg(i)) { - if ( const clang::Expr *E = llvm::dyn_cast_or_null(CE->getArg(i))) { - const clang::MemberExpr *AME=llvm::dyn_cast_or_null(E); + for (int i = 0, j = CE->getNumArgs(); i < j; i++) { + if (CE->getArg(i)) { + if (const clang::Expr *E = llvm::dyn_cast_or_null(CE->getArg(i))) { + const clang::MemberExpr *AME = llvm::dyn_cast_or_null(E); if (AME && AME->isImplicitAccess()) { - clang::ParmVarDecl *PVD=llvm::dyn_cast_or_null(MD->getParamDecl(i)); - clang::QualType QT = PVD->getOriginalType(); - const clang::Type * T = QT.getTypePtr(); - if (!support::isConst(QT) && T->isReferenceType() && ME && ME->isImplicitAccess()) ReportCallArg(CE,i); - } + clang::ParmVarDecl *PVD = llvm::dyn_cast_or_null(MD->getParamDecl(i)); + clang::QualType QT = PVD->getOriginalType(); + const clang::Type *T = QT.getTypePtr(); + if (!support::isConst(QT) && T->isReferenceType() && ME && ME->isImplicitAccess()) + ReportCallArg(CE, i); } - } + } + } + } + } + + void WalkAST::ReportMember(const clang::MemberExpr *ME) { + const ValueDecl *D = ME->getMemberDecl(); + if (D->hasAttr() || D->hasAttr()) + return; + if (visitingCallExpr) { + clang::Expr *IOA = visitingCallExpr->getImplicitObjectArgument(); + if (!(IOA->isImplicitCXXThis() || llvm::dyn_cast_or_null(IOA->IgnoreParenCasts()))) + return; + } + std::string buf; + llvm::raw_string_ostream os(buf); + clang::ento::PathDiagnosticLocation CELoc; + clang::SourceRange R; + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + + CELoc = clang::ento::PathDiagnosticLocation::createBegin(ME, BR.getSourceManager(), AC); + R = ME->getSourceRange(); + + os << "Member data '"; + ME->printPretty(os, nullptr, Policy); + os << "' is directly or indirectly modified in const function\n"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + BR.EmitBasicReport( + AD, Checker, "Member data modified in const function", "Data Class Const Correctness", os.str(), CELoc); } -} - -void WalkAST::ReportMember(const clang::MemberExpr *ME) { - const ValueDecl * D = ME->getMemberDecl(); - if ( D->hasAttr() || D->hasAttr()) return; - if ( visitingCallExpr ) { - clang::Expr * IOA = visitingCallExpr->getImplicitObjectArgument(); - if (!( IOA->isImplicitCXXThis() || llvm::dyn_cast_or_null(IOA->IgnoreParenCasts()))) return; - } - std::string buf; - llvm::raw_string_ostream os(buf); - clang::ento::PathDiagnosticLocation CELoc; - clang::SourceRange R; - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - - CELoc = clang::ento::PathDiagnosticLocation::createBegin(ME, BR.getSourceManager(),AC); - R = ME->getSourceRange(); - - os << "Member data '"; - ME->printPretty(os,nullptr,Policy); - os << "' is directly or indirectly modified in const function\n"; - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: " + os.str(); - writeLog(tolog); - BR.EmitBasicReport(AD,Checker,"Member data modified in const function","Data Class Const Correctness",os.str(),CELoc); -} - -void WalkAST::ReportCall(const clang::CXXMemberCallExpr *CE) { - - const clang::CXXRecordDecl * RD = CE->getRecordDecl(); - const clang::CXXMethodDecl * MD = CE->getMethodDecl(); - if ( !RD || support::isSafeClassName( RD->getQualifiedNameAsString() ) ) return; - std::string buf; - llvm::raw_string_ostream os(buf); - clang::ento::PathDiagnosticLocation CELoc = clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(),AC); - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - os << "call expr '"; - CE->printPretty(os,nullptr,Policy); - os << "' with implicit object argument '"; - CE->getImplicitObjectArgument()->IgnoreParenCasts()->printPretty(os,nullptr,Policy); - os << "'"; - os<<"' is a non-const member function '"<getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+ pname +"' const function '" + mname + "' Warning: "+os.str(); - if ( support::isSafeClassName(support::getQualifiedName(*MD)) ) return; - writeLog(tolog); - BugType * BT = new BugType(Checker,"Non-const member function could modify member data object","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); - - -} - - -void WalkAST::ReportCast(const clang::ExplicitCastExpr *CE) { - std::string buf; - llvm::raw_string_ostream os(buf); - - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - - - os << "Const qualifier of member data object"; - os <<" was removed via cast expression '"; - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - clang::ento::PathDiagnosticLocation CELoc = - clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(),AC); - - writeLog(tolog); - BugType * BT = new BugType(Checker,"Const cast away from member data in const function","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); - - -} - -void WalkAST::ReportCallArg(const clang::CXXMemberCallExpr *CE,const int i) { - - std::string buf; - llvm::raw_string_ostream os(buf); - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - - clang::CXXMethodDecl * CMD = llvm::dyn_cast(CE)->getMethodDecl(); - const clang::MemberExpr *E = llvm::dyn_cast(CE->getArg(i)); - clang::ValueDecl * VD = llvm::dyn_cast(E->getMemberDecl()); - os << "Member data '" << VD->getQualifiedNameAsString(); - os << "' is passed to a non-const reference parameter"; - os <<" of CXX method '" << CMD->getQualifiedNameAsString() << "' in const function"; - os << "\n"; - - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - - clang::ento::PathDiagnosticLocation ELoc = - clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(),AC); - - writeLog(tolog); - BR.EmitBasicReport(CE->getCalleeDecl(),Checker,"Member data passed to non-const reference","Data Class Const Correctness",os.str(),ELoc); - -} - -void WalkAST::ReportCallReturn(const clang::ReturnStmt * RS) { - std::string buf; - llvm::raw_string_ostream os(buf); - - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - - os << "Returns a pointer or reference to a non-const member data object "; - os << " or a const std::vector<*> or const std::vector<*>& "; - os << "in const function in statement '"; - RS->printPretty(os,nullptr,Policy); - os << "\n"; - const clang::CXXMethodDecl * MD = llvm::cast(AD); - clang::ento::PathDiagnosticLocation CELoc = - clang::ento::PathDiagnosticLocation::createBegin(RS, BR.getSourceManager(),AC); - std::string pname = support::getQualifiedName(*(AD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*AD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - writeLog(tolog); - clang::ASTContext &Ctx = AC->getASTContext(); - clang::QualType RQT = MD->getCallResultType(); - clang::QualType RTy = Ctx.getCanonicalType(RQT); - if ( (RTy->isPointerType() || RTy->isReferenceType() ) ) { - if( !support::isConst(RTy) ) { - BugType * BT = new BugType(Checker,"Const function returns pointer or reference to non-const member data object","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); - } + + void WalkAST::ReportCall(const clang::CXXMemberCallExpr *CE) { + const clang::CXXRecordDecl *RD = CE->getRecordDecl(); + const clang::CXXMethodDecl *MD = CE->getMethodDecl(); + if (!RD || support::isSafeClassName(RD->getQualifiedNameAsString())) + return; + std::string buf; + llvm::raw_string_ostream os(buf); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + os << "call expr '"; + CE->printPretty(os, nullptr, Policy); + os << "' with implicit object argument '"; + CE->getImplicitObjectArgument()->IgnoreParenCasts()->printPretty(os, nullptr, Policy); + os << "'"; + os << "' is a non-const member function '" << support::getQualifiedName(*MD); + os << "' that could modify member data object of type '" << support::getQualifiedName(*RD) << "'\n"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + if (support::isSafeClassName(support::getQualifiedName(*MD))) + return; + writeLog(tolog); + BugType *BT = new BugType( + Checker, "Non-const member function could modify member data object", "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); } - std::string svname = "const class std::vector<"; - std::string rtname = RTy.getAsString(); - if ( (RTy->isReferenceType() || RTy ->isRecordType() ) && support::isConst(RTy) && rtname.substr(0,svname.length()) == svname ) { - BugType * BT = new BugType(Checker,"Const function returns member data object of type const std::vector<*> or const std::vector<*>&","Data Class Const Correctness"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - BR.emitReport(std::move(R)); + + void WalkAST::ReportCast(const clang::ExplicitCastExpr *CE) { + std::string buf; + llvm::raw_string_ostream os(buf); + + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + + os << "Const qualifier of member data object"; + os << " was removed via cast expression '"; + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + + writeLog(tolog); + BugType *BT = + new BugType(Checker, "Const cast away from member data in const function", "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); } - -} - - -void ClassChecker::checkASTDecl(const clang::CXXRecordDecl *RD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const { - - const clang::SourceManager &SM = BR.getSourceManager(); - const char *sfile=SM.getPresumedLoc(RD->getLocation()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - std::string buf; - llvm::raw_string_ostream os(buf); - std::string name = RD->getQualifiedNameAsString(); - if ( ! support::isDataClass(name) ) return; - for ( auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I) - { - const FieldDecl * D = (*I) ; - if ( D->hasAttr() || D->hasAttr()) return; - if ( D->isMutable() ) - { - clang::QualType t = D->getType(); - clang::ento::PathDiagnosticLocation DLoc = - clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); - if ( support::isSafeClassName( t.getCanonicalType().getAsString() ) ) return; - if ( ! support::isDataClass( support::getQualifiedName(*RD) ) ) return; - std::string buf; - llvm::raw_string_ostream os(buf); - os << "Mutable member '" <(CE)->getMethodDecl(); + const clang::MemberExpr *E = llvm::dyn_cast(CE->getArg(i)); + clang::ValueDecl *VD = llvm::dyn_cast(E->getMemberDecl()); + os << "Member data '" << VD->getQualifiedNameAsString(); + os << "' is passed to a non-const reference parameter"; + os << " of CXX method '" << CMD->getQualifiedNameAsString() << "' in const function"; + os << "\n"; + + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + + clang::ento::PathDiagnosticLocation ELoc = + clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + + writeLog(tolog); + BR.EmitBasicReport(CE->getCalleeDecl(), + Checker, + "Member data passed to non-const reference", + "Data Class Const Correctness", + os.str(), + ELoc); + } + + void WalkAST::ReportCallReturn(const clang::ReturnStmt *RS) { + std::string buf; + llvm::raw_string_ostream os(buf); + + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + + os << "Returns a pointer or reference to a non-const member data object "; + os << " or a const std::vector<*> or const std::vector<*>& "; + os << "in const function in statement '"; + RS->printPretty(os, nullptr, Policy); + os << "\n"; + const clang::CXXMethodDecl *MD = llvm::cast(AD); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(RS, BR.getSourceManager(), AC); + std::string pname = support::getQualifiedName(*(AD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*AD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + clang::ASTContext &Ctx = AC->getASTContext(); + clang::QualType RQT = MD->getCallResultType(); + clang::QualType RTy = Ctx.getCanonicalType(RQT); + if ((RTy->isPointerType() || RTy->isReferenceType())) { + if (!support::isConst(RTy)) { + BugType *BT = new BugType(Checker, + "Const function returns pointer or reference to non-const member data object", + "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); + } + } + std::string svname = "const class std::vector<"; + std::string rtname = RTy.getAsString(); + if ((RTy->isReferenceType() || RTy->isRecordType()) && support::isConst(RTy) && + rtname.substr(0, svname.length()) == svname) { + BugType *BT = + new BugType(Checker, + "Const function returns member data object of type const std::vector<*> or const std::vector<*>&", + "Data Class Const Correctness"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + BR.emitReport(std::move(R)); + } + } + void ClassChecker::checkASTDecl(const clang::CXXRecordDecl *RD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + const clang::SourceManager &SM = BR.getSourceManager(); + const char *sfile = SM.getPresumedLoc(RD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; + std::string buf; + llvm::raw_string_ostream os(buf); + std::string name = RD->getQualifiedNameAsString(); + if (!support::isDataClass(name)) + return; + for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I) { + const FieldDecl *D = (*I); + if (D->hasAttr() || D->hasAttr()) + return; + if (D->isMutable()) { + clang::QualType t = D->getType(); + clang::ento::PathDiagnosticLocation DLoc = + clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); + if (support::isSafeClassName(t.getCanonicalType().getAsString())) + return; + if (!support::isDataClass(support::getQualifiedName(*RD))) + return; + std::string buf; + llvm::raw_string_ostream os(buf); + os << "Mutable member '" << t.getCanonicalType().getAsString() << " " << *D << "' in data class '" + << support::getQualifiedName(*RD) << "', might be thread-unsafe when accessing via a const handle."; + BR.EmitBasicReport(D, this, "Mutable member in data class", "Data Class Const Correctness", os.str(), DLoc); + std::string pname = support::getQualifiedName(*(RD)); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*D); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' mutable member '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + } + } + + // Check the class methods (member methods). + for (clang::CXXRecordDecl::method_iterator I = RD->method_begin(), E = RD->method_end(); I != E; ++I) { + if (!llvm::isa((*I))) + continue; + if (!(*I)->isConst()) + continue; + clang::CXXMethodDecl *MD = llvm::cast((*I)->getMostRecentDecl()); + if (MD->hasAttr() || MD->hasAttr()) + continue; + if (MD->hasBody()) { + clang::Stmt *Body = MD->getBody(); + WalkAST walker(this, BR, mgr.getAnalysisDeclContext(MD), MD, sfile); + walker.Visit(Body); + clang::QualType RQT = MD->getCallResultType(); + clang::ASTContext &Ctx = BR.getContext(); + clang::QualType RTy = Ctx.getCanonicalType(RQT); + clang::ento::PathDiagnosticLocation ELoc = clang::ento::PathDiagnosticLocation::createBegin(MD, SM); + if ((RTy->isPointerType() || RTy->isReferenceType()) && (!support::isConst(RTy)) && + (support::getQualifiedName(*MD).find("clone") == std::string::npos)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << MD->getQualifiedNameAsString() + << " is a const member function that returns a pointer or reference to a non-const object \n"; + std::string pname = support::getQualifiedName(*(MD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*MD); + support::fixAnonNS(mname, sfile); + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + BR.EmitBasicReport(MD, + this, + "Const function returns pointer or reference to non-const object.", + "Data Class Const Correctness", + os.str(), + ELoc); + } + std::string svname = "const class std::vector<"; + std::string rtname = RTy.getAsString(); + if ((RTy->isReferenceType() || RTy->isRecordType()) && support::isConst(RTy) && + rtname.substr(0, svname.length()) == svname) { + const CXXRecordDecl *RD; + if (RTy->isRecordType()) + RD = RTy->getAsCXXRecordDecl(); + else + RD = RTy->getPointeeCXXRecordDecl(); + const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); + for (unsigned J = 0, F = SD->getTemplateArgs().size(); J != F; ++J) { + if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { + const QualType QAT = SD->getTemplateArgs().get(J).getAsType(); + if (QAT->isPointerType() && !support::isConst(QAT)) { + std::string buf; + llvm::raw_string_ostream os(buf); + std::string pname = support::getQualifiedName(*(MD->getParent())); + support::fixAnonNS(pname, sfile); + std::string mname = support::getQualifiedName(*MD); + support::fixAnonNS(mname, sfile); + os << mname + << " is a const member function that returns an object of type const std::vector<*> or const " + "std::vector<*>& " + << rtname << "\n"; + std::string tolog = "data class '" + pname + "' const function '" + mname + "' Warning: " + os.str(); + writeLog(tolog); + BR.EmitBasicReport(MD, + this, + "Const function returns const std::vector<*> or const std::vector<*>&", + "Data Class Const Correctness", + os.str(), + ELoc); + } + } } + } + } + } /* end of methods loop */ + + } //end of class -// Check the class methods (member methods). - for (clang::CXXRecordDecl::method_iterator - I = RD->method_begin(), E = RD->method_end(); I != E; ++I) { - if ( !llvm::isa((*I)) ) continue; - if (!(*I)->isConst()) continue; - clang::CXXMethodDecl * MD = llvm::cast((*I)->getMostRecentDecl()); - if ( MD->hasAttr() || MD->hasAttr()) continue; - if ( MD->hasBody() ) { - clang::Stmt *Body = MD->getBody(); - WalkAST walker(this,BR, mgr.getAnalysisDeclContext(MD),MD,sfile); - walker.Visit(Body); - clang::QualType RQT = MD->getCallResultType(); - clang::ASTContext &Ctx = BR.getContext(); - clang::QualType RTy = Ctx.getCanonicalType(RQT); - clang::ento::PathDiagnosticLocation ELoc =clang::ento::PathDiagnosticLocation::createBegin( MD , SM ); - if ( (RTy->isPointerType() || RTy->isReferenceType() ) &&(!support::isConst(RTy) ) && ( support::getQualifiedName(*MD).find("clone")==std::string::npos ) ) - { - std::string buf; - llvm::raw_string_ostream os(buf); - os << MD->getQualifiedNameAsString() << " is a const member function that returns a pointer or reference to a non-const object \n"; - std::string pname = support::getQualifiedName(*(MD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*MD); - support::fixAnonNS(mname,sfile); - std::string tolog = "data class '"+pname+"' const function '" + mname + "' Warning: "+os.str(); - writeLog(tolog); - BR.EmitBasicReport(MD,this, "Const function returns pointer or reference to non-const object.","Data Class Const Correctness",os.str(),ELoc); - } - std::string svname = "const class std::vector<"; - std::string rtname = RTy.getAsString(); - if ( (RTy->isReferenceType() || RTy ->isRecordType() ) && support::isConst(RTy) && rtname.substr(0,svname.length()) == svname ) { - const CXXRecordDecl *RD; - if ( RTy->isRecordType() ) RD = RTy->getAsCXXRecordDecl(); - else RD = RTy->getPointeeCXXRecordDecl(); - const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); - for (unsigned J = 0, F = SD->getTemplateArgs().size(); J!=F; ++J) { - if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { - const QualType QAT = SD->getTemplateArgs().get(J).getAsType(); - if ( QAT->isPointerType() && !support::isConst(QAT) ) { - std::string buf; - llvm::raw_string_ostream os(buf); - std::string pname = support::getQualifiedName(*(MD->getParent())); - support::fixAnonNS(pname,sfile); - std::string mname = support::getQualifiedName(*MD); - support::fixAnonNS(mname,sfile); - os << mname << " is a const member function that returns an object of type const std::vector<*> or const std::vector<*>& "< or const std::vector<*>&","Data Class Const Correctness",os.str(),ELoc); - } - } - } - } - } - } /* end of methods loop */ - - -} //end of class - -} //end namespace +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/ClassChecker.h b/Utilities/StaticAnalyzers/src/ClassChecker.h index 142a199bb2232..10d0729dbdaab 100644 --- a/Utilities/StaticAnalyzers/src/ClassChecker.h +++ b/Utilities/StaticAnalyzers/src/ClassChecker.h @@ -9,14 +9,12 @@ namespace clangcms { -class ClassChecker : public clang::ento::Checker > { + class ClassChecker : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::CXXRecordDecl *CRD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + }; - -public: - void checkASTDecl(const clang::CXXRecordDecl *CRD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - -}; - -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/ClassDumper.cpp b/Utilities/StaticAnalyzers/src/ClassDumper.cpp index 6466183f6450e..d23e413775808 100644 --- a/Utilities/StaticAnalyzers/src/ClassDumper.cpp +++ b/Utilities/StaticAnalyzers/src/ClassDumper.cpp @@ -11,221 +11,227 @@ using namespace llvm; namespace clangcms { + void ClassDumper::checkASTDecl(const clang::CXXRecordDecl *RD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR, + std::string tname) const { + const char *sfile = BR.getSourceManager().getPresumedLoc(RD->getLocation()).getFilename(); + if (!RD->hasDefinition()) + return; + std::string rname = RD->getQualifiedNameAsString(); + support::fixAnonNS(rname, sfile); + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + std::string crname("class '"); + const ClassTemplateSpecializationDecl *SD = dyn_cast_or_null(RD); + if (SD) { + std::string buf; + llvm::raw_string_ostream os(buf); + SD->getNameForDiagnostic(os, Policy, true); + crname = crname + os.str() + "'"; + support::writeLog(crname, tname); + for (unsigned J = 0, F = SD->getTemplateArgs().size(); J != F; ++J) { + if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { + std::string taname; + auto tt = SD->getTemplateArgs().get(J).getAsType().getTypePtr(); + if (tt->isRecordType()) { + auto TAD = tt->getAsCXXRecordDecl(); + if (TAD) + taname = TAD->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string sdname = SD->getQualifiedNameAsString(); + support::fixAnonNS(sdname, sfile); + std::string cfname = "templated data class '" + sdname + "' template type class '" + taname + "'"; + support::writeLog(crname + " " + cfname, tname); + } + if (tt->isPointerType() || tt->isReferenceType()) { + auto TAD = tt->getPointeeCXXRecordDecl(); + if (TAD) + taname = TAD->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string sdname = SD->getQualifiedNameAsString(); + support::fixAnonNS(sdname, sfile); + std::string cfname = "templated data class '" + sdname + "' template type class '" + taname + "'"; + std::string cbname = "templated data class 'bare_ptr' template type class '" + taname + "'"; + support::writeLog(crname + " " + cfname, tname); + support::writeLog(crname + " " + cbname, tname); + } + } + } -void ClassDumper::checkASTDecl(const clang::CXXRecordDecl *RD,clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR, std::string tname ) const { - const char *sfile=BR.getSourceManager().getPresumedLoc(RD->getLocation()).getFilename(); - if (!RD->hasDefinition()) return; - std::string rname = RD->getQualifiedNameAsString(); - support::fixAnonNS(rname,sfile); - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - std::string crname("class '"); - const ClassTemplateSpecializationDecl *SD = dyn_cast_or_null(RD); - if (SD) { + } else { + // Dump the class name + crname = crname + rname + "'"; + support::writeLog(crname, tname); + } + + // Dump the class member classes + for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I) { + clang::QualType qual; + clang::QualType type = I->getType(); + if (type.getTypePtr()->isAnyPointerType()) + qual = type.getTypePtr()->getPointeeType(); + else + qual = type.getNonReferenceType(); + if (!qual.getTypePtr()->isRecordType()) + continue; + if (const CXXRecordDecl *TRD = qual.getTypePtr()->getAsCXXRecordDecl()) { + std::string fname = TRD->getQualifiedNameAsString(); + support::fixAnonNS(fname, sfile); + const ClassTemplateSpecializationDecl *SD = dyn_cast_or_null(TRD); + if (SD) { std::string buf; llvm::raw_string_ostream os(buf); - SD->getNameForDiagnostic(os,Policy,true); - crname = crname+os.str()+"'"; - support::writeLog(crname, tname); - for (unsigned J = 0, F = SD->getTemplateArgs().size(); J!=F; ++J) { - if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { - std::string taname; - auto tt = SD->getTemplateArgs().get(J).getAsType().getTypePtr(); - if (tt->isRecordType()) { - auto TAD = tt->getAsCXXRecordDecl(); - if (TAD) taname = TAD->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string sdname = SD->getQualifiedNameAsString(); - support::fixAnonNS(sdname,sfile); - std::string cfname = "templated data class '"+sdname+"' template type class '"+taname+"'"; - support::writeLog(crname+" "+cfname,tname); - } - if ( tt->isPointerType() || tt->isReferenceType() ) { - auto TAD = tt->getPointeeCXXRecordDecl(); - if (TAD) taname = TAD->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string sdname = SD->getQualifiedNameAsString(); - support::fixAnonNS(sdname,sfile); - std::string cfname = "templated data class '"+sdname+"' template type class '"+taname+"'"; - std::string cbname = "templated data class 'bare_ptr' template type class '"+taname+"'"; - support::writeLog(crname+" "+cfname,tname); - support::writeLog(crname+" "+cbname,tname); - } - } + SD->getNameForDiagnostic(os, Policy, true); + std::string cfname = "member data class '" + os.str() + "'"; + support::writeLog(crname + " " + cfname, tname); + // Recurse the template args + for (unsigned J = 0, F = SD->getTemplateArgs().size(); J != F; ++J) { + if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { + std::string taname; + const clang::Type *tt = SD->getTemplateArgs().get(J).getAsType().getTypePtr(); + if (tt->isRecordType()) { + const clang::CXXRecordDecl *TAD = tt->getAsCXXRecordDecl(); + if (TAD) + taname = TAD->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string sdname = SD->getQualifiedNameAsString(); + support::fixAnonNS(sdname, sfile); + std::string cfname = + "templated member data class '" + sdname + "' template type class '" + taname + "'"; + support::writeLog(crname + " " + cfname, tname); + } + if (tt->isPointerType() || tt->isReferenceType()) { + const clang::CXXRecordDecl *TAD = tt->getPointeeCXXRecordDecl(); + if (TAD) + taname = TAD->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string sdname = SD->getQualifiedNameAsString(); + support::fixAnonNS(sdname, sfile); + std::string cfname = + "templated member data class '" + sdname + "' template type class '" + taname + "'"; + std::string cbname = "templated member data class 'bare_ptr' template type class '" + taname + "'"; + support::writeLog(crname + " " + cfname, tname); + support::writeLog(crname + " " + cbname, tname); + } + } } - - } else { -// Dump the class name - crname = crname+rname+"'"; - support::writeLog(crname,tname); - - } - -// Dump the class member classes - for ( auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I) { - clang::QualType qual; - clang::QualType type = I->getType(); - if (type.getTypePtr()->isAnyPointerType()) - qual = type.getTypePtr()->getPointeeType(); - else - qual = type.getNonReferenceType(); - if (!qual.getTypePtr()->isRecordType()) continue; - if (const CXXRecordDecl * TRD = qual.getTypePtr()->getAsCXXRecordDecl()) { - std::string fname = TRD->getQualifiedNameAsString(); - support::fixAnonNS(fname, sfile); - const ClassTemplateSpecializationDecl *SD = dyn_cast_or_null(TRD); - if (SD) { - std::string buf; - llvm::raw_string_ostream os(buf); - SD->getNameForDiagnostic(os,Policy,true); - std::string cfname ="member data class '"+os.str()+"'"; - support::writeLog(crname+" "+cfname,tname); - // Recurse the template args - for (unsigned J = 0, F = SD->getTemplateArgs().size(); J!=F; ++J) { - if (SD->getTemplateArgs().get(J).getKind() == clang::TemplateArgument::Type) { - std::string taname; - const clang::Type * tt = SD->getTemplateArgs().get(J).getAsType().getTypePtr(); - if ( tt->isRecordType() ) { - const clang::CXXRecordDecl * TAD = tt->getAsCXXRecordDecl(); - if (TAD) taname = TAD->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string sdname = SD->getQualifiedNameAsString(); - support::fixAnonNS(sdname,sfile); - std::string cfname = "templated member data class '"+sdname+"' template type class '"+taname+"'"; - support::writeLog(crname+" "+cfname,tname); - } - if ( tt->isPointerType() || tt->isReferenceType() ) { - const clang::CXXRecordDecl * TAD = tt->getPointeeCXXRecordDecl(); - if (TAD) taname = TAD->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string sdname = SD->getQualifiedNameAsString(); - support::fixAnonNS(sdname,sfile); - std::string cfname = "templated member data class '"+sdname+"' template type class '"+taname+"'"; - std::string cbname = "templated member data class 'bare_ptr' template type class '"+taname+"'"; - support::writeLog(crname+" "+cfname,tname); - support::writeLog(crname+" "+cbname,tname); - } - } - } - } else { - if (type.getTypePtr()->isRecordType()) { - std::string cfname ="member data class '"+fname+"' "; - support::writeLog(crname+" "+cfname,tname); - } - if (type.getTypePtr()->isAnyPointerType()) { - std::string cfname = "templated member data class 'bare_ptr' template type class '"+fname+"'"; - support::writeLog(crname+" "+cfname,tname); - } - } - } - } - - - - - -// Dump the base classes - - for ( auto J=RD->bases_begin(), F=RD->bases_end();J != F; ++J) { - auto BRD = J->getType()->getAsCXXRecordDecl(); - if (!BRD) continue; - std::string bname = BRD->getQualifiedNameAsString(); - support::fixAnonNS(bname,sfile); - std::string cbname = "base class '"+bname+"'"; - support::writeLog(crname+" "+cbname,tname); + } else { + if (type.getTypePtr()->isRecordType()) { + std::string cfname = "member data class '" + fname + "' "; + support::writeLog(crname + " " + cfname, tname); } - - -} //end class - -void ClassDumperCT::checkASTDecl(const clang::ClassTemplateDecl *TD,clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const { - - const char *sfile=BR.getSourceManager().getPresumedLoc(TD->getLocation()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - std::string crname("class '"); - std::string pname = "classes.txt.dumperct.unsorted"; - std::string tname = TD->getTemplatedDecl()->getQualifiedNameAsString(); - - if ( tname == "edm::Wrapper" || tname == "edm::RunCache" || tname == "edm::LuminosityBlockCache" || tname == "edm::GlobalCache" ) { - for ( auto I = TD->spec_begin(), - E = TD->spec_end(); I != E; ++I) { - for ( unsigned J = 0, F = I->getTemplateArgs().size(); J!=F; ++J) { - auto D = I->getTemplateArgs().get(J).getAsType()->getAsCXXRecordDecl(); - if (D) { - ClassDumper dumper; dumper.checkASTDecl( D, mgr, BR,pname ); - std::string taname = D->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string tdname = TD->getQualifiedNameAsString(); - support::fixAnonNS(tdname,sfile); - std::string cfname = "templated class '"+tdname+"' template type class '"+taname+"'"; - support::writeLog(cfname,pname); - } - auto E = I->getTemplateArgs().get(J).getAsType()->getPointeeCXXRecordDecl(); - if (E) { - ClassDumper dumper; dumper.checkASTDecl( E, mgr, BR,pname ); - std::string taname = E->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string tdname = TD->getQualifiedNameAsString(); - support::fixAnonNS(tdname,sfile); - std::string cfname = "templated class '"+tdname+"' template type class '"+taname+"'"; - support::writeLog(cfname,pname); - std::string cbname = "templated class 'bare_ptr' template type class '"+taname+"'"; - support::writeLog(crname+" "+cbname,pname); - } - } + if (type.getTypePtr()->isAnyPointerType()) { + std::string cfname = "templated member data class 'bare_ptr' template type class '" + fname + "'"; + support::writeLog(crname + " " + cfname, tname); } - } -} //end class - -void ClassDumperFT::checkASTDecl(const clang::FunctionTemplateDecl *TD,clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const { - - const char *sfile=BR.getSourceManager().getPresumedLoc(TD->getLocation()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - std::string crname("class '"); - std::string pname = "classes.txt.dumperft.unsorted"; - if (TD->getTemplatedDecl()->getQualifiedNameAsString().find("typelookup::className") != std::string::npos ) { - for ( auto I = TD->spec_begin(), - E = TD->spec_end(); I != E; ++I) { - auto * SD = (*I); - for (unsigned J = 0, F = SD->getTemplateSpecializationArgs()->size(); J!=F;++J) { - auto D = SD->getTemplateSpecializationArgs()->get(J).getAsType()->getAsCXXRecordDecl(); - if (D) { - ClassDumper dumper; dumper.checkASTDecl( D, mgr, BR,pname ); - std::string taname = D->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string sdname = SD->getQualifiedNameAsString(); - support::fixAnonNS(sdname,sfile); - std::string cfname = "templated function '"+sdname+"' template type class '"+taname+"'"; - support::writeLog(cfname,pname); - } - auto E = SD->getTemplateSpecializationArgs()->get(J).getAsType()->getPointeeCXXRecordDecl(); - if (E) { - ClassDumper dumper; dumper.checkASTDecl( E, mgr, BR,pname ); - std::string taname = E->getQualifiedNameAsString(); - support::fixAnonNS(taname,sfile); - std::string sdname = SD->getQualifiedNameAsString(); - support::fixAnonNS(sdname,sfile); - std::string cfname = "templated function '"+sdname+"' template type class '"+taname+"'"; - support::writeLog(cfname,pname); - std::string cbname = "templated function 'bare_ptr' template type class '"+taname+"'"; - support::writeLog(crname+" "+cbname,pname); - } - - } + } + } + } + + // Dump the base classes + + for (auto J = RD->bases_begin(), F = RD->bases_end(); J != F; ++J) { + auto BRD = J->getType()->getAsCXXRecordDecl(); + if (!BRD) + continue; + std::string bname = BRD->getQualifiedNameAsString(); + support::fixAnonNS(bname, sfile); + std::string cbname = "base class '" + bname + "'"; + support::writeLog(crname + " " + cbname, tname); + } + + } //end class + + void ClassDumperCT::checkASTDecl(const clang::ClassTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + const char *sfile = BR.getSourceManager().getPresumedLoc(TD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; + std::string crname("class '"); + std::string pname = "classes.txt.dumperct.unsorted"; + std::string tname = TD->getTemplatedDecl()->getQualifiedNameAsString(); + + if (tname == "edm::Wrapper" || tname == "edm::RunCache" || tname == "edm::LuminosityBlockCache" || + tname == "edm::GlobalCache") { + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + for (unsigned J = 0, F = I->getTemplateArgs().size(); J != F; ++J) { + auto D = I->getTemplateArgs().get(J).getAsType()->getAsCXXRecordDecl(); + if (D) { + ClassDumper dumper; + dumper.checkASTDecl(D, mgr, BR, pname); + std::string taname = D->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string tdname = TD->getQualifiedNameAsString(); + support::fixAnonNS(tdname, sfile); + std::string cfname = "templated class '" + tdname + "' template type class '" + taname + "'"; + support::writeLog(cfname, pname); } + auto E = I->getTemplateArgs().get(J).getAsType()->getPointeeCXXRecordDecl(); + if (E) { + ClassDumper dumper; + dumper.checkASTDecl(E, mgr, BR, pname); + std::string taname = E->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string tdname = TD->getQualifiedNameAsString(); + support::fixAnonNS(tdname, sfile); + std::string cfname = "templated class '" + tdname + "' template type class '" + taname + "'"; + support::writeLog(cfname, pname); + std::string cbname = "templated class 'bare_ptr' template type class '" + taname + "'"; + support::writeLog(crname + " " + cbname, pname); + } + } } -} //end class - -void ClassDumperInherit::checkASTDecl(const clang::CXXRecordDecl *RD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const { - return; -} //end of class - - -}//end namespace + } + } //end class + + void ClassDumperFT::checkASTDecl(const clang::FunctionTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + const char *sfile = BR.getSourceManager().getPresumedLoc(TD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; + std::string crname("class '"); + std::string pname = "classes.txt.dumperft.unsorted"; + if (TD->getTemplatedDecl()->getQualifiedNameAsString().find("typelookup::className") != std::string::npos) { + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + auto *SD = (*I); + for (unsigned J = 0, F = SD->getTemplateSpecializationArgs()->size(); J != F; ++J) { + auto D = SD->getTemplateSpecializationArgs()->get(J).getAsType()->getAsCXXRecordDecl(); + if (D) { + ClassDumper dumper; + dumper.checkASTDecl(D, mgr, BR, pname); + std::string taname = D->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string sdname = SD->getQualifiedNameAsString(); + support::fixAnonNS(sdname, sfile); + std::string cfname = "templated function '" + sdname + "' template type class '" + taname + "'"; + support::writeLog(cfname, pname); + } + auto E = SD->getTemplateSpecializationArgs()->get(J).getAsType()->getPointeeCXXRecordDecl(); + if (E) { + ClassDumper dumper; + dumper.checkASTDecl(E, mgr, BR, pname); + std::string taname = E->getQualifiedNameAsString(); + support::fixAnonNS(taname, sfile); + std::string sdname = SD->getQualifiedNameAsString(); + support::fixAnonNS(sdname, sfile); + std::string cfname = "templated function '" + sdname + "' template type class '" + taname + "'"; + support::writeLog(cfname, pname); + std::string cbname = "templated function 'bare_ptr' template type class '" + taname + "'"; + support::writeLog(crname + " " + cbname, pname); + } + } + } + } + } //end class + void ClassDumperInherit::checkASTDecl(const clang::CXXRecordDecl *RD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + return; + } //end of class +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/ClassDumper.h b/Utilities/StaticAnalyzers/src/ClassDumper.h index 5146e4f774fc1..6a91ea14d6fac 100644 --- a/Utilities/StaticAnalyzers/src/ClassDumper.h +++ b/Utilities/StaticAnalyzers/src/ClassDumper.h @@ -18,46 +18,41 @@ namespace clangcms { -class ClassDumper : public clang::ento::Checker > { - -public: - void checkASTDecl(const clang::CXXRecordDecl *CRD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR, std::string tname ) const ; - - void checkASTDecl(const clang::CXXRecordDecl *RD,clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const { - std::string pname = "classes.txt.dumperall.unsorted"; - checkASTDecl(RD,mgr,BR,pname); -} - -}; - -class ClassDumperCT : public clang::ento::Checker > { - -public: - - void checkASTDecl(const clang::ClassTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const ; - -}; - -class ClassDumperFT : public clang::ento::Checker > { - -public: - - void checkASTDecl(const clang::FunctionTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const ; - -}; - -class ClassDumperInherit : public clang::ento::Checker > { - -public: - void checkASTDecl(const clang::CXXRecordDecl *CRD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const ; - -}; - - -} + class ClassDumper : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::CXXRecordDecl *CRD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR, + std::string tname) const; + + void checkASTDecl(const clang::CXXRecordDecl *RD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + std::string pname = "classes.txt.dumperall.unsorted"; + checkASTDecl(RD, mgr, BR, pname); + } + }; + + class ClassDumperCT : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::ClassTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + }; + + class ClassDumperFT : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::FunctionTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + }; + + class ClassDumperInherit : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::CXXRecordDecl *CRD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + }; + +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/CmsException.cpp b/Utilities/StaticAnalyzers/src/CmsException.cpp index eb28b6e67bee6..1ee686a953f61 100644 --- a/Utilities/StaticAnalyzers/src/CmsException.cpp +++ b/Utilities/StaticAnalyzers/src/CmsException.cpp @@ -4,68 +4,52 @@ // //===----------------------------------------------------------------------===// - #include "CmsException.h" #include "CmsSupport.h" namespace clangcms { -bool CmsException::reportGeneral( clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const -{ - const char *sfile=BR.getSourceManager().getPresumedLoc(path.asLocation()).getFilename(); - if ((!sfile) || (!support::isCmsLocalFile(sfile))) return false; - return true; -} - -bool CmsException::reportConstCast( const clang::ento::BugReport &R, - clang::ento::CheckerContext &C) const -{ - clang::ento::BugReporter &BR = C.getBugReporter(); - const clang::SourceManager &SM = BR.getSourceManager(); - clang::ento::PathDiagnosticLocation const& path = R.getLocation(SM); - return reportGeneral ( path, BR ); -} - - -bool CmsException::reportConstCastAway( const clang::ento::BugReport &R, - clang::ento::CheckerContext &C) const -{ - clang::ento::BugReporter & BR = C.getBugReporter(); - const clang::SourceManager &SM = BR.getSourceManager(); - clang::ento::PathDiagnosticLocation const& path = R.getLocation(SM); - return reportGeneral ( path, BR ); - -} - -bool CmsException::reportGlobalStatic( clang::QualType const& t, - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const -{ - return reportGeneral ( path, BR ); -} - -bool CmsException::reportMutableMember( clang::QualType const& t, - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const -{ - return reportGeneral ( path, BR ); -} - -bool CmsException::reportClass( - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const -{ - return reportGeneral ( path, BR ); -} - - -bool CmsException::reportGlobalStaticForType( clang::QualType const& t, - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const -{ - return reportGeneral ( path, BR ); -} -} - - + bool CmsException::reportGeneral(clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const { + const char* sfile = BR.getSourceManager().getPresumedLoc(path.asLocation()).getFilename(); + if ((!sfile) || (!support::isCmsLocalFile(sfile))) + return false; + return true; + } + + bool CmsException::reportConstCast(const clang::ento::BugReport& R, clang::ento::CheckerContext& C) const { + clang::ento::BugReporter& BR = C.getBugReporter(); + const clang::SourceManager& SM = BR.getSourceManager(); + clang::ento::PathDiagnosticLocation const& path = R.getLocation(SM); + return reportGeneral(path, BR); + } + + bool CmsException::reportConstCastAway(const clang::ento::BugReport& R, clang::ento::CheckerContext& C) const { + clang::ento::BugReporter& BR = C.getBugReporter(); + const clang::SourceManager& SM = BR.getSourceManager(); + clang::ento::PathDiagnosticLocation const& path = R.getLocation(SM); + return reportGeneral(path, BR); + } + + bool CmsException::reportGlobalStatic(clang::QualType const& t, + clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const { + return reportGeneral(path, BR); + } + + bool CmsException::reportMutableMember(clang::QualType const& t, + clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const { + return reportGeneral(path, BR); + } + + bool CmsException::reportClass(clang::ento::PathDiagnosticLocation const& path, clang::ento::BugReporter& BR) const { + return reportGeneral(path, BR); + } + + bool CmsException::reportGlobalStaticForType(clang::QualType const& t, + clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const { + return reportGeneral(path, BR); + } +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/CmsException.h b/Utilities/StaticAnalyzers/src/CmsException.h index 4ca89316c2596..48cef212af091 100644 --- a/Utilities/StaticAnalyzers/src/CmsException.h +++ b/Utilities/StaticAnalyzers/src/CmsException.h @@ -14,39 +14,30 @@ #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" - namespace clangcms { -class CmsException { -public: - bool reportGlobalStaticForType( clang::QualType const& t, - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const; - - bool reportGlobalStatic( clang::QualType const& t, - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const; - - bool reportMutableMember( clang::QualType const& t, - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const; - bool reportClass( - clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const; - + class CmsException { + public: + bool reportGlobalStaticForType(clang::QualType const& t, + clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const; - bool reportConstCast ( const clang::ento::BugReport &R, - clang::ento::CheckerContext &C) const; + bool reportGlobalStatic(clang::QualType const& t, + clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const; - bool reportConstCastAway ( const clang::ento::BugReport &R, - clang::ento::CheckerContext &C) const; + bool reportMutableMember(clang::QualType const& t, + clang::ento::PathDiagnosticLocation const& path, + clang::ento::BugReporter& BR) const; + bool reportClass(clang::ento::PathDiagnosticLocation const& path, clang::ento::BugReporter& BR) const; + bool reportConstCast(const clang::ento::BugReport& R, clang::ento::CheckerContext& C) const; - bool reportGeneral( clang::ento::PathDiagnosticLocation const& path, - clang::ento::BugReporter & BR ) const; + bool reportConstCastAway(const clang::ento::BugReport& R, clang::ento::CheckerContext& C) const; -}; + bool reportGeneral(clang::ento::PathDiagnosticLocation const& path, clang::ento::BugReporter& BR) const; + }; -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/CmsSupport.cpp b/Utilities/StaticAnalyzers/src/CmsSupport.cpp index b1f00ff652026..af0fe5fa7c19f 100644 --- a/Utilities/StaticAnalyzers/src/CmsSupport.cpp +++ b/Utilities/StaticAnalyzers/src/CmsSupport.cpp @@ -30,54 +30,49 @@ extern "C" { using namespace clang; using namespace llvm; -bool clangcms::support::isCmsLocalFile(const char* file) -{ - static const char* LocalDir= std::getenv("LOCALRT"); - CMS_THREAD_SAFE static int DirLen=-1; - if (DirLen==-1) - { - DirLen=0; - if (LocalDir!=nullptr) DirLen=strlen(LocalDir); +bool clangcms::support::isCmsLocalFile(const char *file) { + static const char *LocalDir = std::getenv("LOCALRT"); + CMS_THREAD_SAFE static int DirLen = -1; + if (DirLen == -1) { + DirLen = 0; + if (LocalDir != nullptr) + DirLen = strlen(LocalDir); } - if ((DirLen==0) || (strncmp(file,LocalDir,DirLen)!=0) || (strncmp(&file[DirLen],"/src/",5)!=0)) return false; + if ((DirLen == 0) || (strncmp(file, LocalDir, DirLen) != 0) || (strncmp(&file[DirLen], "/src/", 5) != 0)) + return false; return true; } - // This is a wrapper around NamedDecl::getQualifiedNameAsString. // It produces more qualified output to distinguish several cases // which would otherwise be ambiguous. std::string clangcms::support::getQualifiedName(const clang::NamedDecl &d) { std::string ret; const DeclContext *ctx = d.getDeclContext(); - if (ctx->isFunctionOrMethod() && isa(ctx)) - { + if (ctx->isFunctionOrMethod() && isa(ctx)) { // This is a local variable. // d.getQualifiedNameAsString() will return the unqualifed name for this // but we want an actual qualified name so we can distinguish variables // with the same name but that are in different functions. ret = getQualifiedName(*cast(ctx)) + "::" + d.getNameAsString(); - } - else - { + } else { ret = d.getQualifiedNameAsString(); } - if (const FunctionDecl *fd = dyn_cast_or_null(&d)) - { - if (fd->isFunctionTemplateSpecialization()) - { + if (const FunctionDecl *fd = dyn_cast_or_null(&d)) { + if (fd->isFunctionTemplateSpecialization()) { ret += "<"; const TemplateArgumentList *TemplateArgs = fd->getTemplateSpecializationArgs(); - if (TemplateArgs) - { - unsigned num_args = TemplateArgs->size(); - for (unsigned i = 0; i < num_args; ++i) { - if (i) ret +=","; - TemplateArgument TemplateArg = TemplateArgs->get(i); - if (TemplateArg.getKind() == TemplateArgument::ArgKind::Type) ret += TemplateArg.getAsType().getAsString(); - } + if (TemplateArgs) { + unsigned num_args = TemplateArgs->size(); + for (unsigned i = 0; i < num_args; ++i) { + if (i) + ret += ","; + TemplateArgument TemplateArg = TemplateArgs->get(i); + if (TemplateArg.getKind() == TemplateArgument::ArgKind::Type) + ret += TemplateArg.getAsType().getAsString(); } + } ret += ">"; } // This is a function. getQualifiedNameAsString will return a string @@ -88,8 +83,7 @@ std::string clangcms::support::getQualifiedName(const clang::NamedDecl &d) { // void ANamespace::AFunction(float); ret += "("; const clang::FunctionType *ft = fd->getType()->castAs(); - if (const FunctionProtoType *fpt = dyn_cast_or_null(ft)) - { + if (const FunctionProtoType *fpt = dyn_cast_or_null(ft)) { unsigned num_params = fd->getNumParams(); for (unsigned i = 0; i < num_params; ++i) { if (i) @@ -111,125 +105,131 @@ std::string clangcms::support::getQualifiedName(const clang::NamedDecl &d) { return ret; } - bool clangcms::support::isSafeClassName(const std::string &cname) { - - static const std::vector names = { - "atomic<", - "std::atomic<", - "struct std::atomic<", - "std::__atomic_", - "std::mutex", - "std::recursive_mutex", - "boost::thread_specific_ptr", - "class std::atomic", - "class std::__atomic_", - "class std::mutex", - "class std::recursive_mutex", - "class boost::thread_specific_ptr", - "tbb::", - "class tbb::", - "concurrent_unordered_map", - "class concurrent_unordered_map", - "edm::AtomicPtrCache", - "class edm::AtomicPtrCache", - "edm::InputTag", - "class edm::InputTag", - "std::once_flag", - "struct std::once_flag", - "boost::::extents", - "cout", "cerr", - "std::cout","std::cerr", - "edm::RunningAverage","class edm::RunningAverage", - "TVirtualMutex", "class TVirtualMutex", - "boost::(anonymous namespace)::extents", "(anonymous namespace)::_1", "(anonymous namespace)::_2" - }; - - for (auto& name: names) - if ( cname.substr(0,name.length()) == name ) return true; + static const std::vector names = {"atomic<", + "std::atomic<", + "struct std::atomic<", + "std::__atomic_", + "std::mutex", + "std::recursive_mutex", + "boost::thread_specific_ptr", + "class std::atomic", + "class std::__atomic_", + "class std::mutex", + "class std::recursive_mutex", + "class boost::thread_specific_ptr", + "tbb::", + "class tbb::", + "concurrent_unordered_map", + "class concurrent_unordered_map", + "edm::AtomicPtrCache", + "class edm::AtomicPtrCache", + "edm::InputTag", + "class edm::InputTag", + "std::once_flag", + "struct std::once_flag", + "boost::::extents", + "cout", + "cerr", + "std::cout", + "std::cerr", + "edm::RunningAverage", + "class edm::RunningAverage", + "TVirtualMutex", + "class TVirtualMutex", + "boost::(anonymous namespace)::extents", + "(anonymous namespace)::_1", + "(anonymous namespace)::_2"}; + + for (auto &name : names) + if (cname.substr(0, name.length()) == name) + return true; return false; } -bool clangcms::support::isDataClass(const std::string & name) { - CMS_THREAD_SAFE static std::string iname(""); - if ( iname.empty()) { - clang::FileSystemOptions FSO; - clang::FileManager FM(FSO); - const char * lPath = std::getenv("LOCALRT"); - const char * rPath = std::getenv("CMSSW_RELEASE_BASE"); - if ( lPath == nullptr || rPath == nullptr ) { - llvm::errs()<<"\n\nThe scram runtime envorinment is not set.\nRun 'cmsenv' or 'eval `scram runtime -csh`'.\n\n\n"; - exit(1); - } - const std::string lname = std::string(lPath); - const std::string rname = std::string(rPath); - const std::string tname("/src/Utilities/StaticAnalyzers/scripts/bloom.bin"); - const std::string fname1 = lname + tname; - const std::string fname2 = rname + tname; - if (!(FM.getFile(fname1) || FM.getFile(fname2))) { - llvm::errs()<<"\n\nChecker cannot find bloom filter file" < names = { - "TGraph::Fit(const char *,", - "TGraph2D::Fit(const char *,", - "TH1::Fit(const char *,", - "TMultiGraph::Fit(const char *,", - "TTable::Fit(const char *,", - "TTree::Fit(const char *,", - "TTreePlayer::Fit(const char *,", - "CLHEP::HepMatrix::determinant(" - }; - for (auto& name: names) - if ( fname.substr(0,name.length()) == name ) return true; - return false; +bool clangcms::support::isKnownThrUnsafeFunc(const std::string &fname) { + static const std::vector names = {"TGraph::Fit(const char *,", + "TGraph2D::Fit(const char *,", + "TH1::Fit(const char *,", + "TMultiGraph::Fit(const char *,", + "TTable::Fit(const char *,", + "TTree::Fit(const char *,", + "TTreePlayer::Fit(const char *,", + "CLHEP::HepMatrix::determinant("}; + for (auto &name : names) + if (fname.substr(0, name.length()) == name) + return true; + return false; } -void clangcms::support::writeLog(const std::string &ostring,const std::string &tfstring) { - const char * pPath = std::getenv("LOCALRT"); - if ( pPath == nullptr ) { - llvm::errs()<<"\n\nThe scram runtime envorinment is not set.\nRun 'cmsenv' or 'eval `scram runtime -csh`'.\n\n\n"; - exit(1); - } +void clangcms::support::writeLog(const std::string &ostring, const std::string &tfstring) { + const char *pPath = std::getenv("LOCALRT"); + if (pPath == nullptr) { + llvm::errs() << "\n\nThe scram runtime envorinment is not set.\nRun 'cmsenv' or 'eval `scram runtime -csh`'.\n\n\n"; + exit(1); + } - std::string pname = std::string(pPath) +"/tmp/"; - const std::string tname = pname + tfstring; + std::string pname = std::string(pPath) + "/tmp/"; + const std::string tname = pname + tfstring; - std::fstream file; - file.open(tname.c_str(),std::ios::out|std::ios::app); - file<isReferenceType() ) - { - // remove only the surounding reference type - return qt.getNonReferenceType().isConstQualified(); - } - if ( qt->isPointerType() ) - { - clang::PointerType const* pt = qt->getAs(); - return pt->getPointeeType().isConstQualified(); - } - - // regular type - return qt.isConstQualified(); -} - -bool isCmsLocalFile(const char* file); -std::string getQualifiedName(const clang::NamedDecl &d); -bool isSafeClassName(const std::string &d); -bool isDataClass(const std::string &d); -bool isInterestingLocation(const std::string &d); -bool isKnownThrUnsafeFunc(const std::string &name ); -void writeLog(const std::string &ostring,const std::string &tfstring); -void fixAnonNS(std::string & name, const char * fname ); -} -} + namespace support { + + // The three cases + // + // const int var; + // int const& var; + // int const* var; + // + // have to be handled slightly different. This function implements the functionality to check + // for const qualifier for all of them. + // + inline bool isConst(clang::QualType const &qt) { + if (qt->isReferenceType()) { + // remove only the surounding reference type + return qt.getNonReferenceType().isConstQualified(); + } + if (qt->isPointerType()) { + clang::PointerType const *pt = qt->getAs(); + return pt->getPointeeType().isConstQualified(); + } + + // regular type + return qt.isConstQualified(); + } + + bool isCmsLocalFile(const char *file); + std::string getQualifiedName(const clang::NamedDecl &d); + bool isSafeClassName(const std::string &d); + bool isDataClass(const std::string &d); + bool isInterestingLocation(const std::string &d); + bool isKnownThrUnsafeFunc(const std::string &name); + void writeLog(const std::string &ostring, const std::string &tfstring); + void fixAnonNS(std::string &name, const char *fname); + } // namespace support +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.cpp b/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.cpp index ed38a4cabe9c4..c0c15e8716cfc 100644 --- a/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.cpp +++ b/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.cpp @@ -8,56 +8,51 @@ #include #include #include "ConstCastAwayChecker.h" -#include "CmsSupport.h" +#include "CmsSupport.h" using namespace clang; using namespace clang::ento; using namespace llvm; -namespace clangcms -{ +namespace clangcms { - -void ConstCastAwayChecker::checkPreStmt(const clang::ExplicitCastExpr *CE, - clang::ento::CheckerContext &C) const -{ - if (! ( clang::CStyleCastExpr::classof(CE) || clang::CXXConstCastExpr::classof(CE) )) + void ConstCastAwayChecker::checkPreStmt(const clang::ExplicitCastExpr *CE, clang::ento::CheckerContext &C) const { + if (!(clang::CStyleCastExpr::classof(CE) || clang::CXXConstCastExpr::classof(CE))) return; - const Expr * SE = CE->getSubExpr(); - const CXXRecordDecl * CRD = nullptr; - std::string cname; - if (SE->getType()->isPointerType()) + const Expr *SE = CE->getSubExpr(); + const CXXRecordDecl *CRD = nullptr; + std::string cname; + if (SE->getType()->isPointerType()) CRD = SE->getType()->getPointeeCXXRecordDecl(); - else + else CRD = SE->getType()->getAsCXXRecordDecl(); - if (CRD) + if (CRD) cname = CRD->getQualifiedNameAsString(); - - clang::ASTContext &Ctx = C.getASTContext(); - clang::QualType OrigTy = Ctx.getCanonicalType(SE->getType()); - clang::QualType ToTy = Ctx.getCanonicalType(CE->getType()); - if ( support::isConst( OrigTy ) && ! support::isConst(ToTy) ) { - if ( clang::ento::ExplodedNode *errorNode = C.generateErrorNode()) { - if (!BT) - BT.reset(new clang::ento::BugType(this,"const cast away","ConstThreadSafety")); - std::string buf; - llvm::raw_string_ostream os(buf); - os << "const qualifier was removed via a cast, this may result in thread-unsafe code."; - std::unique_ptr R = llvm::make_unique(*BT, - os.str(), errorNode); - R->addRange(CE->getSourceRange()); - if ( ! m_exception.reportConstCastAway( *R, C ) ) - return; - C.emitReport(std::move(R)); - if (cname.empty()) - return; - std::string tname ="constcastaway-checker.txt.unsorted"; - std::string tolog ="flagged class '"+cname+"' const qualifier cast away"; - support::writeLog(tolog,tname); + clang::ASTContext &Ctx = C.getASTContext(); + clang::QualType OrigTy = Ctx.getCanonicalType(SE->getType()); + clang::QualType ToTy = Ctx.getCanonicalType(CE->getType()); + + if (support::isConst(OrigTy) && !support::isConst(ToTy)) { + if (clang::ento::ExplodedNode *errorNode = C.generateErrorNode()) { + if (!BT) + BT.reset(new clang::ento::BugType(this, "const cast away", "ConstThreadSafety")); + std::string buf; + llvm::raw_string_ostream os(buf); + os << "const qualifier was removed via a cast, this may result in thread-unsafe code."; + std::unique_ptr R = llvm::make_unique(*BT, os.str(), errorNode); + R->addRange(CE->getSourceRange()); + if (!m_exception.reportConstCastAway(*R, C)) + return; + C.emitReport(std::move(R)); + if (cname.empty()) + return; + std::string tname = "constcastaway-checker.txt.unsorted"; + std::string tolog = "flagged class '" + cname + "' const qualifier cast away"; + support::writeLog(tolog, tname); } - } -} + } + } -} +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.h b/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.h index a55dd57871797..edadfbe03e591 100644 --- a/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.h +++ b/Utilities/StaticAnalyzers/src/ConstCastAwayChecker.h @@ -12,20 +12,19 @@ #include #include #include "FWCore/Utilities/interface/thread_safety_macros.h" -#include "CmsException.h" +#include "CmsException.h" namespace clangcms { -class ConstCastAwayChecker: public clang::ento::Checker< clang::ento::check::PreStmt< clang::ExplicitCastExpr> > { -public: - CMS_THREAD_SAFE mutable std::unique_ptr BT; - void checkPreStmt(const clang::ExplicitCastExpr *CE, clang::ento::CheckerContext &C) const; + class ConstCastAwayChecker : public clang::ento::Checker > { + public: + CMS_THREAD_SAFE mutable std::unique_ptr BT; + void checkPreStmt(const clang::ExplicitCastExpr *CE, clang::ento::CheckerContext &C) const; -private: - CmsException m_exception; + private: + CmsException m_exception; + }; -}; - -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/ConstCastChecker.cpp b/Utilities/StaticAnalyzers/src/ConstCastChecker.cpp index 50f29e92ccd5c..91edc8a01dbef 100644 --- a/Utilities/StaticAnalyzers/src/ConstCastChecker.cpp +++ b/Utilities/StaticAnalyzers/src/ConstCastChecker.cpp @@ -4,11 +4,10 @@ // //===----------------------------------------------------------------------===// - #include #include #include "ConstCastChecker.h" -#include "CmsSupport.h" +#include "CmsSupport.h" using namespace clang; using namespace clang::ento; @@ -16,39 +15,33 @@ using namespace llvm; namespace clangcms { -void ConstCastChecker::checkPreStmt(const clang::CXXConstCastExpr *CE, - clang::ento::CheckerContext &C) const -{ - const Expr * SE = CE->getSubExprAsWritten(); - const CXXRecordDecl * CRD = nullptr; - std::string cname; - if (SE->getType()->isPointerType()) - CRD = SE->getType()->getPointeeCXXRecordDecl(); - else - CRD = SE->getType()->getAsCXXRecordDecl(); - if (CRD) - cname = CRD->getQualifiedNameAsString(); - if (clang::ento::ExplodedNode *errorNode = C.generateErrorNode()) { + void ConstCastChecker::checkPreStmt(const clang::CXXConstCastExpr *CE, clang::ento::CheckerContext &C) const { + const Expr *SE = CE->getSubExprAsWritten(); + const CXXRecordDecl *CRD = nullptr; + std::string cname; + if (SE->getType()->isPointerType()) + CRD = SE->getType()->getPointeeCXXRecordDecl(); + else + CRD = SE->getType()->getAsCXXRecordDecl(); + if (CRD) + cname = CRD->getQualifiedNameAsString(); + if (clang::ento::ExplodedNode *errorNode = C.generateErrorNode()) { if (!BT) - BT.reset(new clang::ento::BugType(this,"const_cast used on pointer to class", "ConstThreadSafety")); + BT.reset(new clang::ento::BugType(this, "const_cast used on pointer to class", "ConstThreadSafety")); std::string buf; llvm::raw_string_ostream os(buf); os << "const_cast was used, this may result in thread-unsafe code."; - std::unique_ptr R = llvm::make_unique(*BT, - os.str(), errorNode); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), errorNode); R->addRange(CE->getSourceRange()); - if ( ! m_exception.reportConstCast( *R, C ) ) - return; + if (!m_exception.reportConstCast(*R, C)) + return; C.emitReport(std::move(R)); if (cname.empty()) - return; + return; std::string tname = "constcast-checker.txt.unsorted"; - std::string tolog = "flagged class '"+cname+"' const_cast used "; - support::writeLog(tolog,tname); - } - -} - -} - + std::string tolog = "flagged class '" + cname + "' const_cast used "; + support::writeLog(tolog, tname); + } + } +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/ConstCastChecker.h b/Utilities/StaticAnalyzers/src/ConstCastChecker.h index 6f33593ad5554..c3c83d9b5293a 100644 --- a/Utilities/StaticAnalyzers/src/ConstCastChecker.h +++ b/Utilities/StaticAnalyzers/src/ConstCastChecker.h @@ -13,17 +13,16 @@ #include "CmsException.h" #include "FWCore/Utilities/interface/thread_safety_macros.h" - namespace clangcms { -class ConstCastChecker: public clang::ento::Checker< clang::ento::check::PreStmt< clang::CXXConstCastExpr> > { -public: - CMS_THREAD_SAFE mutable std::unique_ptr BT; - void checkPreStmt(const clang::CXXConstCastExpr *CE, clang::ento::CheckerContext &C) const; + class ConstCastChecker : public clang::ento::Checker > { + public: + CMS_THREAD_SAFE mutable std::unique_ptr BT; + void checkPreStmt(const clang::CXXConstCastExpr *CE, clang::ento::CheckerContext &C) const; -private: - CmsException m_exception; -}; -} + private: + CmsException m_exception; + }; +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/EDMPluginDumper.cc b/Utilities/StaticAnalyzers/src/EDMPluginDumper.cc index f7eb4b03849b1..71f740db54032 100644 --- a/Utilities/StaticAnalyzers/src/EDMPluginDumper.cc +++ b/Utilities/StaticAnalyzers/src/EDMPluginDumper.cc @@ -3,7 +3,7 @@ #include #include #include -#include +#include using namespace clang; using namespace clang::ento; @@ -11,27 +11,24 @@ using namespace llvm; namespace clangcms { -void EDMPluginDumper::checkASTDecl(const clang::ClassTemplateDecl *TD,clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const { - - std::string tname = TD->getTemplatedDecl()->getQualifiedNameAsString(); - if ( tname == "edm::WorkerMaker" ) { - for ( auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { - for (unsigned J = 0, F = I->getTemplateArgs().size(); J!=F; ++J) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - I->getTemplateArgs().get(J).print(mgr.getASTContext().getPrintingPolicy(),os); - std::string rname = os.str(); - std::string fname("plugins.txt.unsorted"); - std::string ostring = rname +"\n"; - support::writeLog(ostring,fname); - } - } - } - -} //end class - - -}//end namespace - - + void EDMPluginDumper::checkASTDecl(const clang::ClassTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + std::string tname = TD->getTemplatedDecl()->getQualifiedNameAsString(); + if (tname == "edm::WorkerMaker") { + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + for (unsigned J = 0, F = I->getTemplateArgs().size(); J != F; ++J) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + I->getTemplateArgs().get(J).print(mgr.getASTContext().getPrintingPolicy(), os); + std::string rname = os.str(); + std::string fname("plugins.txt.unsorted"); + std::string ostring = rname + "\n"; + support::writeLog(ostring, fname); + } + } + } + + } //end class + +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/EDMPluginDumper.h b/Utilities/StaticAnalyzers/src/EDMPluginDumper.h index f56e90c7182a7..7360cec50d943 100644 --- a/Utilities/StaticAnalyzers/src/EDMPluginDumper.h +++ b/Utilities/StaticAnalyzers/src/EDMPluginDumper.h @@ -18,18 +18,15 @@ namespace clangcms { -class EDMPluginDumper : public clang::ento::Checker > { + class EDMPluginDumper : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::ClassTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; -public: + private: + CmsException m_exception; + }; - void checkASTDecl(const clang::ClassTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR ) const ; - -private: - CmsException m_exception; - -}; - - -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/FiniteMathChecker.cc b/Utilities/StaticAnalyzers/src/FiniteMathChecker.cc index 077c279f23b5a..84816e1634e7e 100644 --- a/Utilities/StaticAnalyzers/src/FiniteMathChecker.cc +++ b/Utilities/StaticAnalyzers/src/FiniteMathChecker.cc @@ -5,41 +5,41 @@ #include #include - #include "CmsSupport.h" #include #include namespace clangcms { -void FiniteMathChecker::checkPreStmt(const clang::CallExpr *CE, clang::ento::CheckerContext &ctx) const -{ - const clang::ento::ProgramStateRef state = ctx.getState(); - const clang::LocationContext *LC = ctx.getLocationContext(); - const clang::Expr *Callee = CE->getCallee(); - const clang::FunctionDecl *FD = state->getSVal(Callee, LC).getAsFunctionDecl(); - - if (!FD) - return; - - // Get the name of the callee. - clang::IdentifierInfo *II = FD->getIdentifier(); - if (!II) // if no identifier, not a simple C function - return; - - if (!II->isStr("isnan") && !II->isStr("isinf")) - return; - - clang::ento::ExplodedNode *N = ctx.generateErrorNode(); - if (!N) - return; - - if (!BT) - BT.reset(new clang::ento::BugType(this,"std::isnan / std::isinf does not work when fast-math is used. Please use edm::isNotFinite from 'FWCore/Utilities/interface/isFinite.h'", "fastmath plugin")); - - std::unique_ptr report = llvm::make_unique(*BT, BT->getName(), N); - report->addRange(Callee->getSourceRange()); - ctx.emitReport(std::move(report)); -} -} - + void FiniteMathChecker::checkPreStmt(const clang::CallExpr *CE, clang::ento::CheckerContext &ctx) const { + const clang::ento::ProgramStateRef state = ctx.getState(); + const clang::LocationContext *LC = ctx.getLocationContext(); + const clang::Expr *Callee = CE->getCallee(); + const clang::FunctionDecl *FD = state->getSVal(Callee, LC).getAsFunctionDecl(); + + if (!FD) + return; + + // Get the name of the callee. + clang::IdentifierInfo *II = FD->getIdentifier(); + if (!II) // if no identifier, not a simple C function + return; + + if (!II->isStr("isnan") && !II->isStr("isinf")) + return; + + clang::ento::ExplodedNode *N = ctx.generateErrorNode(); + if (!N) + return; + + if (!BT) + BT.reset(new clang::ento::BugType(this, + "std::isnan / std::isinf does not work when fast-math is used. Please use " + "edm::isNotFinite from 'FWCore/Utilities/interface/isFinite.h'", + "fastmath plugin")); + + std::unique_ptr report = llvm::make_unique(*BT, BT->getName(), N); + report->addRange(Callee->getSourceRange()); + ctx.emitReport(std::move(report)); + } +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/FiniteMathChecker.h b/Utilities/StaticAnalyzers/src/FiniteMathChecker.h index 323864fa6b2ab..f605a6209f11f 100644 --- a/Utilities/StaticAnalyzers/src/FiniteMathChecker.h +++ b/Utilities/StaticAnalyzers/src/FiniteMathChecker.h @@ -8,11 +8,12 @@ #include "CmsException.h" namespace clangcms { -class FiniteMathChecker : public clang::ento::Checker > { - CMS_THREAD_SAFE mutable std::unique_ptr BT; -public: - void checkPreStmt(const clang::CallExpr *ref, clang::ento::CheckerContext &C) const; -}; -} + class FiniteMathChecker : public clang::ento::Checker > { + CMS_THREAD_SAFE mutable std::unique_ptr BT; + + public: + void checkPreStmt(const clang::CallExpr *ref, clang::ento::CheckerContext &C) const; + }; +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/FunctionChecker.cpp b/Utilities/StaticAnalyzers/src/FunctionChecker.cpp index 73c28427dab4c..f49cbe5b9b976 100644 --- a/Utilities/StaticAnalyzers/src/FunctionChecker.cpp +++ b/Utilities/StaticAnalyzers/src/FunctionChecker.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include "FunctionChecker.h" @@ -27,179 +27,174 @@ using namespace llvm; namespace clangcms { -class FWalker : public clang::StmtVisitor { - const CheckerBase *Checker; - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext *AC; - -public: - FWalker(const CheckerBase * checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac ) - : Checker(checker), - BR(br), - AC(ac) {} - - const clang::Stmt * ParentStmt(const Stmt *S) { - const Stmt * P = AC->getParentMap().getParentIgnoreParens(S); - if (!P) return nullptr; - return P; + class FWalker : public clang::StmtVisitor { + const CheckerBase *Checker; + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; + + public: + FWalker(const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac) + : Checker(checker), BR(br), AC(ac) {} + + const clang::Stmt *ParentStmt(const Stmt *S) { + const Stmt *P = AC->getParentMap().getParentIgnoreParens(S); + if (!P) + return nullptr; + return P; + } + + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + void VisitDeclRefExpr(clang::DeclRefExpr *DRE); + void ReportDeclRef(const clang::DeclRefExpr *DRE); + }; + + void FWalker::VisitChildren(clang::Stmt *S) { + for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } } + void FWalker::VisitDeclRefExpr(clang::DeclRefExpr *DRE) { + if (const clang::VarDecl *D = llvm::dyn_cast_or_null(DRE->getDecl())) { + if (D && support::isSafeClassName(D->getCanonicalDecl()->getQualifiedNameAsString())) + return; + ReportDeclRef(DRE); + } + } - void VisitChildren(clang::Stmt *S ); - void VisitStmt( clang::Stmt *S) { VisitChildren(S); } - void VisitDeclRefExpr( clang::DeclRefExpr * DRE); - void ReportDeclRef( const clang::DeclRefExpr * DRE); - -}; + void FWalker::ReportDeclRef(const clang::DeclRefExpr *DRE) { + const clang::VarDecl *D = llvm::dyn_cast_or_null(DRE->getDecl()); + if (D && (D->hasAttr() || D->hasAttr())) + return; + if (support::isSafeClassName(D->getCanonicalDecl()->getQualifiedNameAsString())) + return; + + const char *sfile = BR.getSourceManager().getPresumedLoc(D->getLocation()).getFilename(); + std::string fname(sfile); + if (!support::isInterestingLocation(fname)) + return; + clang::QualType t = D->getType(); + if (support::isSafeClassName(t.getCanonicalType().getAsString())) + return; + const Decl *PD = AC->getDecl(); + std::string dname = ""; + std::string sdname = ""; + if (const NamedDecl *ND = llvm::dyn_cast_or_null(PD)) { + sdname = support::getQualifiedName(*ND); + dname = ND->getQualifiedNameAsString(); + } + clang::ento::PathDiagnosticLocation DLoc; + if (support::isCmsLocalFile(sfile)) { + if (D->getLocation().isMacroID()) + DLoc = clang::ento::PathDiagnosticLocation(D->getLocation(), BR.getSourceManager()); + else + DLoc = clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); + } else + DLoc = clang::ento::PathDiagnosticLocation::createBegin(DRE, BR.getSourceManager(), AC); + + std::string tname = "function-checker.txt.unsorted"; + + std::string vname = support::getQualifiedName(*D); + std::string svname = D->getNameAsString(); + if (D->getTSCSpec() == clang::ThreadStorageClassSpecifier::TSCS_thread_local) + return; + if (D->isStaticLocal() && !clangcms::support::isConst(t)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "function '" << dname << "' accesses or modifies non-const static local variable '" << svname << "'.\n"; + // BR.EmitBasicReport(D, Checker, "non-const static local variable accessed or modified","ThreadSafety",os.str(), DLoc); + std::string ostring = "function '" + sdname + "' static variable '" + vname + "'.\n"; + support::writeLog(ostring, tname); + return; + } -void FWalker::VisitChildren( clang::Stmt *S) { - for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); + if (D->isStaticDataMember() && !clangcms::support::isConst(t)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "function '" << dname << "' accesses or modifies non-const static member data variable '" << svname + << "'.\n"; + // BR.EmitBasicReport(D, Checker, "non-const static local variable accessed or modified","ThreadSafety",os.str(), DLoc); + std::string ostring = "function '" + sdname + "' static variable '" + vname + "'.\n"; + support::writeLog(ostring, tname); + return; } -} + if (D->hasGlobalStorage() && !D->isStaticDataMember() && !D->isStaticLocal() && !clangcms::support::isConst(t)) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "function '" << dname << "' accesses or modifies non-const global static variable '" << svname << "'.\n"; + // BR.EmitBasicReport(D, Checker, "non-const static local variable accessed or modified","ThreadSafety",os.str(), DLoc); + std::string ostring = "function '" + sdname + "' static variable '" + vname + "'.\n"; + support::writeLog(ostring, tname); + return; + } + } -void FWalker::VisitDeclRefExpr( clang::DeclRefExpr * DRE) { - if (const clang::VarDecl * D = llvm::dyn_cast_or_null(DRE->getDecl()) ) { - if ( D && support::isSafeClassName(D->getCanonicalDecl()->getQualifiedNameAsString() ) ) return; - ReportDeclRef(DRE); + void FunctionChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager &mgr, BugReporter &BR) const { + if (MD->hasAttr()) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(MD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; + std::string fname(sfile); + if (!support::isInterestingLocation(fname)) + return; + if (!MD->doesThisDeclarationHaveABody()) + return; + FWalker walker(this, BR, mgr.getAnalysisDeclContext(MD)); + walker.Visit(MD->getBody()); + return; } -} - -void FWalker::ReportDeclRef ( const clang::DeclRefExpr * DRE) { - - const clang::VarDecl * D = llvm::dyn_cast_or_null(DRE->getDecl()); - if ( D && ( D->hasAttr() || D->hasAttr())) return; - if ( support::isSafeClassName( D->getCanonicalDecl()->getQualifiedNameAsString() ) ) return; - - const char *sfile=BR.getSourceManager().getPresumedLoc(D->getLocation()).getFilename(); - std::string fname(sfile); - if (!support::isInterestingLocation(fname)) return; - clang::QualType t = D->getType(); - if ( support::isSafeClassName( t.getCanonicalType().getAsString() ) ) return; - const Decl * PD = AC->getDecl(); - std::string dname =""; - std::string sdname =""; - if (const NamedDecl * ND = llvm::dyn_cast_or_null(PD)) { - sdname = support::getQualifiedName(*ND); - dname = ND->getQualifiedNameAsString(); - } - clang::ento::PathDiagnosticLocation DLoc; - if (support::isCmsLocalFile(sfile)) { - if (D->getLocation().isMacroID()) - DLoc = clang::ento::PathDiagnosticLocation(D->getLocation(),BR.getSourceManager()); - else - DLoc = clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); - } else - DLoc = clang::ento::PathDiagnosticLocation::createBegin(DRE, BR.getSourceManager(), AC); - - std::string tname ="function-checker.txt.unsorted"; - - std::string vname = support::getQualifiedName(*D); - std::string svname = D->getNameAsString(); - if ( D->getTSCSpec() == clang::ThreadStorageClassSpecifier::TSCS_thread_local ) return; - if ( D->isStaticLocal() && ! clangcms::support::isConst( t ) ) - { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "function '"<isStaticDataMember() && ! clangcms::support::isConst( t ) ) - { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "function '"<hasGlobalStorage() && - !D->isStaticDataMember() && - !D->isStaticLocal() && - !clangcms::support::isConst( t ) ) - { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "function '"<hasAttr()) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(MD->getLocation()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - std::string fname(sfile); - if ( !support::isInterestingLocation(fname) ) return; - if (!MD->doesThisDeclarationHaveABody()) return; - FWalker walker(this, BR, mgr.getAnalysisDeclContext(MD)); - walker.Visit(MD->getBody()); - return; -} - -void FunctionChecker::checkASTDecl(const FunctionDecl *FD, AnalysisManager& mgr, - BugReporter &BR) const { - if ( FD->hasAttr()) return; - if (FD-> isInExternCContext()) { - std::string buf; - std::string dname = FD->getQualifiedNameAsString(); - if ( dname.compare(dname.size()-1,1,"_") == 0 ) { - llvm::raw_string_ostream os(buf); - os << "function '"<< dname << "' is in an extern \"C\" context and most likely accesses or modifies fortran variables in a 'COMMONBLOCK'.\n"; - clang::ento::PathDiagnosticLocation::createBegin(FD, BR.getSourceManager()); -// BR.EmitBasicReport(FD, "COMMONBLOCK variable accessed or modified","ThreadSafety",os.str(), FDLoc); - std::string ostring = "function '" + dname + "' static variable 'COMMONBLOCK'.\n"; - std::string tname = "function-checker.txt.unsorted"; - support::writeLog(ostring,tname); - } - } - - const char *sfile=BR.getSourceManager().getPresumedLoc(FD->getLocation ()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - std::string fname(sfile); - if ( !support::isInterestingLocation(fname) ) return; - if (FD->doesThisDeclarationHaveABody()) { - FWalker walker(this, BR, mgr.getAnalysisDeclContext(FD)); - walker.Visit(FD->getBody()); - } -} - -void FunctionChecker::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager& mgr, - BugReporter &BR) const { - - if ( TD->hasAttr()) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(TD->getLocation ()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - std::string fname(sfile); - if ( !support::isInterestingLocation(fname) ) return; - for (auto I = TD->spec_begin(), - E = TD->spec_end(); I != E; ++I) - { - if (I->doesThisDeclarationHaveABody()) { - FWalker walker(this,BR, mgr.getAnalysisDeclContext(*I)); - walker.Visit(I->getBody()); - } - } - return; -} - - - -} + + void FunctionChecker::checkASTDecl(const FunctionDecl *FD, AnalysisManager &mgr, BugReporter &BR) const { + if (FD->hasAttr()) + return; + if (FD->isInExternCContext()) { + std::string buf; + std::string dname = FD->getQualifiedNameAsString(); + if (dname.compare(dname.size() - 1, 1, "_") == 0) { + llvm::raw_string_ostream os(buf); + os << "function '" << dname + << "' is in an extern \"C\" context and most likely accesses or modifies fortran variables in a " + "'COMMONBLOCK'.\n"; + clang::ento::PathDiagnosticLocation::createBegin(FD, BR.getSourceManager()); + // BR.EmitBasicReport(FD, "COMMONBLOCK variable accessed or modified","ThreadSafety",os.str(), FDLoc); + std::string ostring = "function '" + dname + "' static variable 'COMMONBLOCK'.\n"; + std::string tname = "function-checker.txt.unsorted"; + support::writeLog(ostring, tname); + } + } + + const char *sfile = BR.getSourceManager().getPresumedLoc(FD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; + std::string fname(sfile); + if (!support::isInterestingLocation(fname)) + return; + if (FD->doesThisDeclarationHaveABody()) { + FWalker walker(this, BR, mgr.getAnalysisDeclContext(FD)); + walker.Visit(FD->getBody()); + } + } + + void FunctionChecker::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager &mgr, BugReporter &BR) const { + if (TD->hasAttr()) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(TD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; + std::string fname(sfile); + if (!support::isInterestingLocation(fname)) + return; + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + if (I->doesThisDeclarationHaveABody()) { + FWalker walker(this, BR, mgr.getAnalysisDeclContext(*I)); + walker.Visit(I->getBody()); + } + } + return; + } + +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/FunctionChecker.h b/Utilities/StaticAnalyzers/src/FunctionChecker.h index a06565d759ac0..017178213d27c 100644 --- a/Utilities/StaticAnalyzers/src/FunctionChecker.h +++ b/Utilities/StaticAnalyzers/src/FunctionChecker.h @@ -9,25 +9,23 @@ namespace clangcms { -class FunctionChecker : public clang::ento::Checker< clang::ento::check::ASTDecl, - clang::ento::check::ASTDecl, - clang::ento::check::ASTDecl > -{ - - -public: - - void checkASTDecl(const clang::CXXMethodDecl *CMD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - void checkASTDecl(const clang::FunctionDecl *FD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - void checkASTDecl(const clang::FunctionTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - -private: - CmsException m_exception; -}; - -} + class FunctionChecker : public clang::ento::Checker, + clang::ento::check::ASTDecl, + clang::ento::check::ASTDecl > { + public: + void checkASTDecl(const clang::CXXMethodDecl *CMD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + void checkASTDecl(const clang::FunctionDecl *FD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + void checkASTDecl(const clang::FunctionTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + private: + CmsException m_exception; + }; + +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/FunctionDumper.cpp b/Utilities/StaticAnalyzers/src/FunctionDumper.cpp index 6d519a42eda9b..cb78f074b8761 100644 --- a/Utilities/StaticAnalyzers/src/FunctionDumper.cpp +++ b/Utilities/StaticAnalyzers/src/FunctionDumper.cpp @@ -18,212 +18,220 @@ #include #include #include -#include +#include #include "CmsException.h" #include "CmsSupport.h" #include "FunctionDumper.h" - using namespace clang; using namespace ento; using namespace llvm; namespace clangcms { -class FDumper : public clang::StmtVisitor { - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext *AC; - const FunctionDecl *AD; + class FDumper : public clang::StmtVisitor { + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; + const FunctionDecl *AD; - enum Kind { NotVisited, - Visited }; + enum Kind { NotVisited, Visited }; - /// A DenseMap that records visited states of CallExpr. - llvm::DenseMap VisitedExpr; + /// A DenseMap that records visited states of CallExpr. + llvm::DenseMap VisitedExpr; -public: - FDumper(clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac, const FunctionDecl * fd ) - : BR(br), - AC(ac), - AD(fd) {} + public: + FDumper(clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac, const FunctionDecl *fd) + : BR(br), AC(ac), AD(fd) {} - /// This method adds a CallExpr to the worklist - void setVisited(Expr * E) { + /// This method adds a CallExpr to the worklist + void setVisited(Expr *E) { Kind &K = VisitedExpr[E]; - if ( K = NotVisited ) { - VisitedExpr[E] = Visited; - return; + if (K = NotVisited) { + VisitedExpr[E] = Visited; + return; } - } + } - bool wasVisited(Expr * E) { + bool wasVisited(Expr *E) { Kind &K = VisitedExpr[E]; - if ( K = Visited ) return true; + if (K = Visited) + return true; return false; + } + + const clang::Stmt *ParentStmt(const Stmt *S) { + const Stmt *P = AC->getParentMap().getParentIgnoreParens(S); + if (!P) + return nullptr; + return P; + } + + void fixAnonNS(std::string &name) { + const std::string anon_ns = "(anonymous namespace)"; + if (name.substr(0, anon_ns.size()) == anon_ns) { + const char *fname = BR.getSourceManager().getPresumedLoc(AD->getLocation()).getFilename(); + const char *sname = "/src/"; + const char *filename = std::strstr(fname, sname); + if (filename != nullptr) + name = name.substr(0, anon_ns.size() - 1) + " in " + filename + ")" + name.substr(anon_ns.size()); + } + return; + } + + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + void VisitCallExpr(CallExpr *CE); + void VisitCXXMemberCallExpr(CXXMemberCallExpr *CXE); + void VisitCXXConstructExpr(CXXConstructExpr *CCE); + }; + + void FDumper::VisitChildren(clang::Stmt *S) { + for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } } - const clang::Stmt * ParentStmt(const Stmt *S) { - const Stmt * P = AC->getParentMap().getParentIgnoreParens(S); - if (!P) return nullptr; - return P; + void FDumper::VisitCXXConstructExpr(CXXConstructExpr *CCE) { + std::string buf; + llvm::raw_string_ostream os(buf); + LangOptions LangOpts; + LangOpts.CPlusPlus = true; + PrintingPolicy Policy(LangOpts); + std::string mdname = support::getQualifiedName(*AD); + fixAnonNS(mdname); + CXXConstructorDecl *CCD = CCE->getConstructor(); + if (!CCD) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(CCE->getExprLoc()).getFilename(); + std::string sname(sfile); + if (!support::isInterestingLocation(sname)) + return; + std::string mname; + mname = support::getQualifiedName(*CCD); + fixAnonNS(mname); + std::string tname = "function-dumper.txt.unsorted"; + std::string ostring = "function '" + mdname + "' " + "calls function '" + mname + "'\n"; + support::writeLog(ostring, tname); + + VisitChildren(CCE); } - void fixAnonNS(std::string & name) { - const std::string anon_ns = "(anonymous namespace)"; - if (name.substr(0, anon_ns.size()) == anon_ns ) { - const char* fname = BR.getSourceManager().getPresumedLoc(AD->getLocation()).getFilename(); - const char* sname = "/src/"; - const char* filename = std::strstr(fname, sname); - if (filename != nullptr) name = name.substr(0, anon_ns.size() - 1)+" in "+filename+")"+name.substr(anon_ns.size()); - } + void FDumper::VisitCXXMemberCallExpr(CXXMemberCallExpr *CXE) { + std::string buf; + llvm::raw_string_ostream os(buf); + LangOptions LangOpts; + LangOpts.CPlusPlus = true; + PrintingPolicy Policy(LangOpts); + std::string mdname = support::getQualifiedName(*AD); + fixAnonNS(mdname); + CXXMethodDecl *MD = CXE->getMethodDecl(); + if (!MD) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(CXE->getExprLoc()).getFilename(); + std::string sname(sfile); + if (!support::isInterestingLocation(sname)) + return; + std::string mname; + mname = support::getQualifiedName(*MD); + fixAnonNS(mname); + std::string tname = "function-dumper.txt.unsorted"; + std::string ostring; + if (MD->isVirtual()) + ostring = "function '" + mdname + "' " + "calls function '" + mname + " virtual'\n"; + else + ostring = "function '" + mdname + "' " + "calls function '" + mname + "'\n"; + support::writeLog(ostring, tname); + + VisitChildren(CXE); + } + + void FDumper::VisitCallExpr(CallExpr *CE) { + std::string buf; + llvm::raw_string_ostream os(buf); + LangOptions LangOpts; + LangOpts.CPlusPlus = true; + PrintingPolicy Policy(LangOpts); + std::string mdname = support::getQualifiedName(*AD); + fixAnonNS(mdname); + FunctionDecl *FD = CE->getDirectCallee(); + if (!FD) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(CE->getExprLoc()).getFilename(); + std::string sname(sfile); + if (!support::isInterestingLocation(sname)) return; + std::string mname; + fixAnonNS(mname); + mname = support::getQualifiedName(*FD); + std::string tname = "function-dumper.txt.unsorted"; + std::string ostring; + if (FD->isVirtualAsWritten() || FD->isPure()) + ostring = "function '" + mdname + "' " + "calls function '" + mname + " virtual'\n"; + else + ostring = "function '" + mdname + "' " + "calls function '" + mname + "'\n"; + support::writeLog(ostring, tname); + + VisitChildren(CE); } + void FunctionDumper::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager &mgr, BugReporter &BR) const { + if (MD->getLocation().isInvalid()) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(MD->getLocation()).getFilename(); + std::string sname(sfile); + if (!support::isInterestingLocation(sname)) + return; + if (!support::isCmsLocalFile(sfile)) + return; + if (!MD->doesThisDeclarationHaveABody()) + return; + FDumper walker(BR, mgr.getAnalysisDeclContext(MD), MD); + walker.Visit(MD->getBody()); + std::string mname = support::getQualifiedName(*MD); + walker.fixAnonNS(mname); + std::string tname = "function-dumper.txt.unsorted"; + for (auto I = MD->begin_overridden_methods(), E = MD->end_overridden_methods(); I != E; ++I) { + std::string oname = support::getQualifiedName(*(*I)); + walker.fixAnonNS(oname); + std::string ostring = "function '" + mname + "' " + "overrides function '" + oname + " virtual'\n"; + support::writeLog(ostring, tname); + } + return; + } - void VisitChildren(clang::Stmt *S ); - void VisitStmt( clang::Stmt *S) { VisitChildren(S); } - void VisitCallExpr( CallExpr *CE ); - void VisitCXXMemberCallExpr( CXXMemberCallExpr *CXE ); - void VisitCXXConstructExpr( CXXConstructExpr *CCE ); - -}; + void FunctionDumper::checkASTDecl(const FunctionDecl *MD, AnalysisManager &mgr, BugReporter &BR) const { + if (MD->getLocation().isInvalid()) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(MD->getLocation()).getFilename(); + std::string sname(sfile); + if (!support::isInterestingLocation(sname)) + return; + if (!support::isCmsLocalFile(sfile)) + return; + if (!MD->doesThisDeclarationHaveABody()) + return; + FDumper walker(BR, mgr.getAnalysisDeclContext(MD), MD); + walker.Visit(MD->getBody()); + return; + } -void FDumper::VisitChildren( clang::Stmt *S) { - for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); - } -} - -void FDumper::VisitCXXConstructExpr( CXXConstructExpr *CCE ) { - std::string buf; - llvm::raw_string_ostream os(buf); - LangOptions LangOpts; - LangOpts.CPlusPlus = true; - PrintingPolicy Policy(LangOpts); - std::string mdname = support::getQualifiedName(*AD); - fixAnonNS(mdname); - CXXConstructorDecl * CCD = CCE->getConstructor(); - if (!CCD) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(CCE->getExprLoc()).getFilename(); - std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; - std::string mname; - mname = support::getQualifiedName(*CCD); - fixAnonNS(mname); - std::string tname = "function-dumper.txt.unsorted"; - std::string ostring = "function '"+ mdname + "' " + "calls function '" + mname + "'\n"; - support::writeLog(ostring,tname); - - VisitChildren(CCE); -} - -void FDumper::VisitCXXMemberCallExpr( CXXMemberCallExpr *CXE ) { - std::string buf; - llvm::raw_string_ostream os(buf); - LangOptions LangOpts; - LangOpts.CPlusPlus = true; - PrintingPolicy Policy(LangOpts); - std::string mdname = support::getQualifiedName(*AD); - fixAnonNS(mdname); - CXXMethodDecl * MD = CXE->getMethodDecl(); - if (!MD) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(CXE->getExprLoc()).getFilename(); - std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; - std::string mname; - mname = support::getQualifiedName(*MD); - fixAnonNS(mname); - std::string tname = "function-dumper.txt.unsorted"; - std::string ostring; - if ( MD->isVirtual()) ostring = "function '"+ mdname + "' " + "calls function '" + mname + " virtual'\n"; - else ostring = "function '"+ mdname + "' " + "calls function '" + mname + "'\n"; - support::writeLog(ostring,tname); - - VisitChildren(CXE); -} - -void FDumper::VisitCallExpr( CallExpr *CE ) { - std::string buf; - llvm::raw_string_ostream os(buf); - LangOptions LangOpts; - LangOpts.CPlusPlus = true; - PrintingPolicy Policy(LangOpts); - std::string mdname = support::getQualifiedName(*AD); - fixAnonNS(mdname); - FunctionDecl * FD = CE->getDirectCallee(); - if (!FD) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(CE->getExprLoc()).getFilename(); - std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; - std::string mname; - fixAnonNS(mname); - mname = support::getQualifiedName(*FD); - std::string tname = "function-dumper.txt.unsorted"; - std::string ostring; - if (FD->isVirtualAsWritten() || FD->isPure()) - ostring = "function '"+ mdname + "' " + "calls function '" + mname + " virtual'\n"; - else ostring = "function '"+ mdname + "' " + "calls function '" + mname + "'\n"; - support::writeLog(ostring,tname); - - VisitChildren(CE); -} - -void FunctionDumper::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager& mgr, - BugReporter &BR) const { - if (MD->getLocation().isInvalid()) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(MD->getLocation()).getFilename(); - std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; - if ( ! support::isCmsLocalFile(sfile) ) return; - if (!MD->doesThisDeclarationHaveABody()) return; - FDumper walker(BR, mgr.getAnalysisDeclContext(MD), MD); - walker.Visit(MD->getBody()); - std::string mname = support::getQualifiedName(*MD); - walker.fixAnonNS(mname); - std::string tname = "function-dumper.txt.unsorted"; - for (auto I = MD->begin_overridden_methods(), E = MD->end_overridden_methods(); I!=E; ++I) { - std::string oname = support::getQualifiedName(*(*I)); - walker.fixAnonNS(oname); - std::string ostring = "function '" + mname + "' " + "overrides function '" + oname + " virtual'\n"; - support::writeLog(ostring,tname); + void FunctionDumper::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager &mgr, BugReporter &BR) const { + if (TD->getLocation().isInvalid()) + return; + const char *sfile = BR.getSourceManager().getPresumedLoc(TD->getLocation()).getFilename(); + std::string sname(sfile); + if (!support::isInterestingLocation(sname)) + return; + if (!support::isCmsLocalFile(sfile)) + return; + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + if (I->doesThisDeclarationHaveABody()) { + FDumper walker(BR, mgr.getAnalysisDeclContext(*I), (*I)); + walker.Visit(I->getBody()); } - return; -} - -void FunctionDumper::checkASTDecl(const FunctionDecl *MD, AnalysisManager& mgr, - BugReporter &BR) const { - if (MD->getLocation().isInvalid()) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(MD->getLocation()).getFilename(); - std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; - if ( ! support::isCmsLocalFile(sfile) ) return; - if (!MD->doesThisDeclarationHaveABody()) return; - FDumper walker(BR, mgr.getAnalysisDeclContext(MD), MD); - walker.Visit(MD->getBody()); - return; -} - - - -void FunctionDumper::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager& mgr, - BugReporter &BR) const { - if (TD->getLocation().isInvalid()) return; - const char *sfile=BR.getSourceManager().getPresumedLoc(TD->getLocation ()).getFilename(); - std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; - if ( ! support::isCmsLocalFile(sfile) ) return; - for (auto I = TD->spec_begin(), - E = TD->spec_end(); I != E; ++I) - { - if (I->doesThisDeclarationHaveABody()) { - FDumper walker(BR, mgr.getAnalysisDeclContext(*I), (*I)); - walker.Visit(I->getBody()); - } - } - return; -} - - - -} + } + return; + } + +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/FunctionDumper.h b/Utilities/StaticAnalyzers/src/FunctionDumper.h index b1a90f2268bcf..1583175e03e1d 100644 --- a/Utilities/StaticAnalyzers/src/FunctionDumper.h +++ b/Utilities/StaticAnalyzers/src/FunctionDumper.h @@ -7,27 +7,25 @@ namespace clangcms { -class FunctionDumper : public clang::ento::Checker< clang::ento::check::ASTDecl, - clang::ento::check::ASTDecl , - clang::ento::check::ASTDecl > -{ - - -public: - - void checkASTDecl(const clang::CXXMethodDecl *CMD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - void checkASTDecl(const clang::FunctionDecl *MD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - void checkASTDecl(const clang::FunctionTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - -private: - CmsException m_exception; -}; - -} + class FunctionDumper : public clang::ento::Checker, + clang::ento::check::ASTDecl, + clang::ento::check::ASTDecl > { + public: + void checkASTDecl(const clang::CXXMethodDecl *CMD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + void checkASTDecl(const clang::FunctionDecl *MD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + void checkASTDecl(const clang::FunctionTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + private: + CmsException m_exception; + }; + +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/GlobalStaticChecker.cpp b/Utilities/StaticAnalyzers/src/GlobalStaticChecker.cpp index 7e3c6950ffbc9..defac9fc98388 100644 --- a/Utilities/StaticAnalyzers/src/GlobalStaticChecker.cpp +++ b/Utilities/StaticAnalyzers/src/GlobalStaticChecker.cpp @@ -12,37 +12,32 @@ using namespace clang; using namespace ento; using namespace llvm; -namespace clangcms -{ - -void GlobalStaticChecker::checkASTDecl(const clang::VarDecl *D, - clang::ento::AnalysisManager &Mgr, - clang::ento::BugReporter &BR) const -{ - if ( D->hasAttr() || D->hasAttr()) return; - if ( D->getTSCSpec() == clang::ThreadStorageClassSpecifier::TSCS_thread_local ) return; - clang::QualType t = D->getType(); - if ( D->hasGlobalStorage() && - !D->isStaticDataMember() && - !D->isStaticLocal() && - !support::isConst( t ) ) - { - clang::ento::PathDiagnosticLocation DLoc = clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); - - if ( ! m_exception.reportGlobalStaticForType( t, DLoc, BR ) ) - return; - if ( support::isSafeClassName( t.getCanonicalType().getAsString() ) ) return; - - std::string buf; - llvm::raw_string_ostream os(buf); - os << "Non-const variable '" << t.getAsString()<<" "<<*D << "' is static and might be thread-unsafe"; - - BR.EmitBasicReport(D, this, "non-const global static variable", - "ThreadSafety", os.str(), DLoc); - return; - - } - -} - -} +namespace clangcms { + + void GlobalStaticChecker::checkASTDecl(const clang::VarDecl *D, + clang::ento::AnalysisManager &Mgr, + clang::ento::BugReporter &BR) const { + if (D->hasAttr() || D->hasAttr()) + return; + if (D->getTSCSpec() == clang::ThreadStorageClassSpecifier::TSCS_thread_local) + return; + clang::QualType t = D->getType(); + if (D->hasGlobalStorage() && !D->isStaticDataMember() && !D->isStaticLocal() && !support::isConst(t)) { + clang::ento::PathDiagnosticLocation DLoc = + clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); + + if (!m_exception.reportGlobalStaticForType(t, DLoc, BR)) + return; + if (support::isSafeClassName(t.getCanonicalType().getAsString())) + return; + + std::string buf; + llvm::raw_string_ostream os(buf); + os << "Non-const variable '" << t.getAsString() << " " << *D << "' is static and might be thread-unsafe"; + + BR.EmitBasicReport(D, this, "non-const global static variable", "ThreadSafety", os.str(), DLoc); + return; + } + } + +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/GlobalStaticChecker.h b/Utilities/StaticAnalyzers/src/GlobalStaticChecker.h index 7ca20f364a6b5..12da98ca91124 100644 --- a/Utilities/StaticAnalyzers/src/GlobalStaticChecker.h +++ b/Utilities/StaticAnalyzers/src/GlobalStaticChecker.h @@ -12,19 +12,17 @@ #include "FWCore/Utilities/interface/thread_safety_macros.h" #include "CmsException.h" - namespace clangcms { -class GlobalStaticChecker : public clang::ento::Checker< clang::ento::check::ASTDecl< clang::VarDecl> > { - CMS_THREAD_SAFE mutable std::unique_ptr< clang::ento::BuiltinBug> BT; + class GlobalStaticChecker : public clang::ento::Checker > { + CMS_THREAD_SAFE mutable std::unique_ptr BT; + + public: + void checkASTDecl(const clang::VarDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const; -public: - void checkASTDecl(const clang::VarDecl *D, - clang::ento::AnalysisManager &Mgr, - clang::ento::BugReporter &BR) const; -private: - CmsException m_exception; -}; + private: + CmsException m_exception; + }; -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/MutableMemberChecker.cpp b/Utilities/StaticAnalyzers/src/MutableMemberChecker.cpp index d0b360f47bd1f..a859ac7d8380a 100644 --- a/Utilities/StaticAnalyzers/src/MutableMemberChecker.cpp +++ b/Utilities/StaticAnalyzers/src/MutableMemberChecker.cpp @@ -11,37 +11,31 @@ using namespace ento; using namespace llvm; namespace clangcms { -void MutableMemberChecker::checkASTDecl(const clang::FieldDecl *D, - clang::ento::AnalysisManager &Mgr, - clang::ento::BugReporter &BR) const -{ - if ( D->hasAttr() || D->hasAttr()) - return; - if ( D->isMutable() && - D->getDeclContext()->isRecord() ) - { - clang::QualType t = D->getType(); - clang::ento::PathDiagnosticLocation DLoc = - clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); + void MutableMemberChecker::checkASTDecl(const clang::FieldDecl *D, + clang::ento::AnalysisManager &Mgr, + clang::ento::BugReporter &BR) const { + if (D->hasAttr() || D->hasAttr()) + return; + if (D->isMutable() && D->getDeclContext()->isRecord()) { + clang::QualType t = D->getType(); + clang::ento::PathDiagnosticLocation DLoc = + clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); - if ( ! m_exception.reportMutableMember( t, DLoc, BR ) ) - return; - std::string mname = t.getCanonicalType().getAsString(); - if ( support::isSafeClassName( mname ) ) - return; - std::string buf; - llvm::raw_string_ostream os(buf); - std::string pname = D->getParent()->getQualifiedNameAsString(); - os << "Mutable member '" <<*D << "' in class '"<getQualifiedNameAsString(); - support::writeLog(ostring,tname); - } - -} - - -} + if (!m_exception.reportMutableMember(t, DLoc, BR)) + return; + std::string mname = t.getCanonicalType().getAsString(); + if (support::isSafeClassName(mname)) + return; + std::string buf; + llvm::raw_string_ostream os(buf); + std::string pname = D->getParent()->getQualifiedNameAsString(); + os << "Mutable member '" << *D << "' in class '" << pname + << "', might be thread-unsafe when accessing via a const pointer."; + BR.EmitBasicReport(D, this, "mutable member if accessed via const pointer", "ConstThreadSafety", os.str(), DLoc); + std::string tname = "mutablemember-checker.txt.unsorted"; + std::string ostring = "flagged class '" + pname + "' mutable member '" + D->getQualifiedNameAsString(); + support::writeLog(ostring, tname); + } + } +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/MutableMemberChecker.h b/Utilities/StaticAnalyzers/src/MutableMemberChecker.h index e951be84e3b15..df70e360048e8 100644 --- a/Utilities/StaticAnalyzers/src/MutableMemberChecker.h +++ b/Utilities/StaticAnalyzers/src/MutableMemberChecker.h @@ -15,16 +15,15 @@ #include "CmsSupport.h" namespace clangcms { -class MutableMemberChecker : public clang::ento::Checker< clang::ento::check::ASTDecl< clang::FieldDecl> > { - CMS_THREAD_SAFE mutable std::unique_ptr< clang::ento::BuiltinBug> BT; + class MutableMemberChecker : public clang::ento::Checker > { + CMS_THREAD_SAFE mutable std::unique_ptr BT; -public: - void checkASTDecl(const clang::FieldDecl *D, - clang::ento::AnalysisManager &Mgr, - clang::ento::BugReporter &BR) const; -private: - CmsException m_exception; -}; -} + public: + void checkASTDecl(const clang::FieldDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const; + + private: + CmsException m_exception; + }; +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/StaticLocalChecker.cpp b/Utilities/StaticAnalyzers/src/StaticLocalChecker.cpp index 0c6152b2b6ba6..cdd1de7630e5b 100644 --- a/Utilities/StaticAnalyzers/src/StaticLocalChecker.cpp +++ b/Utilities/StaticAnalyzers/src/StaticLocalChecker.cpp @@ -15,32 +15,31 @@ using namespace llvm; namespace clangcms { - -void StaticLocalChecker::checkASTDecl(const clang::VarDecl *D, - clang::ento::AnalysisManager &Mgr, - clang::ento::BugReporter &BR) const -{ - clang::QualType t = D->getType(); - if ( D->hasAttr() || D->hasAttr()) return; - if ( ( (D->isStaticLocal() || D->isStaticDataMember() ) && D->getTSCSpec() != clang::ThreadStorageClassSpecifier::TSCS_thread_local) && ! support::isConst( t ) ) - { - clang::ento::PathDiagnosticLocation DLoc = clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); - - if ( ! m_exception.reportGlobalStaticForType( t, DLoc, BR ) ) - return; - if ( support::isSafeClassName( t.getCanonicalType().getAsString() ) ) return; - - std::string buf; - llvm::raw_string_ostream os(buf); - os << "Non-const variable '" <getType(); + if (D->hasAttr() || D->hasAttr()) + return; + if (((D->isStaticLocal() || D->isStaticDataMember()) && + D->getTSCSpec() != clang::ThreadStorageClassSpecifier::TSCS_thread_local) && + !support::isConst(t)) { + clang::ento::PathDiagnosticLocation DLoc = + clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); + + if (!m_exception.reportGlobalStaticForType(t, DLoc, BR)) + return; + if (support::isSafeClassName(t.getCanonicalType().getAsString())) + return; + + std::string buf; + llvm::raw_string_ostream os(buf); + os << "Non-const variable '" << t.getAsString() << " " << *D + << "' is static local or static member data and might be thread-unsafe"; + + BR.EmitBasicReport(D, this, "non-const static variable", "ThreadSafety", os.str(), DLoc); + return; + } + } + +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/StaticLocalChecker.h b/Utilities/StaticAnalyzers/src/StaticLocalChecker.h index 88342f3a68588..6d68dc7573324 100644 --- a/Utilities/StaticAnalyzers/src/StaticLocalChecker.h +++ b/Utilities/StaticAnalyzers/src/StaticLocalChecker.h @@ -13,18 +13,16 @@ #include "FWCore/Utilities/interface/thread_safety_macros.h" #include "CmsException.h" - namespace clangcms { -class StaticLocalChecker : public clang::ento::Checker< clang::ento::check::ASTDecl< clang::VarDecl> > { - CMS_THREAD_SAFE mutable std::unique_ptr BT; + class StaticLocalChecker : public clang::ento::Checker > { + CMS_THREAD_SAFE mutable std::unique_ptr BT; + + public: + void checkASTDecl(const clang::VarDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const; -public: - void checkASTDecl(const clang::VarDecl *D, - clang::ento::AnalysisManager &Mgr, - clang::ento::BugReporter &BR) const; -private: - CmsException m_exception; -}; -} + private: + CmsException m_exception; + }; +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.cpp b/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.cpp index 4e393c464f644..52f96309f5553 100644 --- a/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.cpp +++ b/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.cpp @@ -4,92 +4,85 @@ #include #include - using namespace clang; using namespace ento; using namespace llvm; namespace clangcms { -class TUFWalker : public clang::StmtVisitor { - const CheckerBase *Checker; - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext *AC; - -public: - TUFWalker(const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac ) - : Checker(checker), - BR(br), - AC(ac) {} - - void VisitChildren(clang::Stmt *S ); - void VisitStmt( clang::Stmt *S) { VisitChildren(S); } - void VisitCXXMemberCallExpr( clang::CXXMemberCallExpr *CE ); - -}; - -void TUFWalker::VisitChildren( clang::Stmt *S) { - for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); + class TUFWalker : public clang::StmtVisitor { + const CheckerBase *Checker; + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; + + public: + TUFWalker(const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac) + : Checker(checker), BR(br), AC(ac) {} + + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + void VisitCXXMemberCallExpr(clang::CXXMemberCallExpr *CE); + }; + + void TUFWalker::VisitChildren(clang::Stmt *S) { + for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } + } + + void TUFWalker::VisitCXXMemberCallExpr(CXXMemberCallExpr *CE) { + CXXMethodDecl *MD = CE->getMethodDecl(); + if (!MD) + return; + const CXXMethodDecl *PD = llvm::dyn_cast_or_null(AC->getDecl()); + if (!PD) + return; + std::string mname = support::getQualifiedName(*MD); + std::string pname = support::getQualifiedName(*PD); + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + if (support::isKnownThrUnsafeFunc(mname)) { + os << "Known thread unsafe function " << mname << " is called in function " << pname; + PathDiagnosticLocation CELoc = PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + BugType *BT = new BugType(Checker, "known thread unsafe function called", "ThreadSafety"); + std::unique_ptr R = llvm::make_unique(*BT, os.str(), CELoc); + R->setDeclWithIssue(AC->getDecl()); + R->addRange(CE->getSourceRange()); + BR.emitReport(std::move(R)); + std::string tname = "function-checker.txt.unsorted"; + std::string ostring = "function '" + pname + "' known thread unsafe function '" + mname + "'.\n"; + support::writeLog(ostring, tname); } -} - - - -void TUFWalker::VisitCXXMemberCallExpr( CXXMemberCallExpr *CE ) { - CXXMethodDecl * MD = CE->getMethodDecl(); - if (!MD) return; - const CXXMethodDecl * PD = llvm::dyn_cast_or_null(AC->getDecl()); - if (!PD) return; - std::string mname = support::getQualifiedName(*MD); - std::string pname = support::getQualifiedName(*PD); - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - if ( support::isKnownThrUnsafeFunc(mname) ) { - os << "Known thread unsafe function " << mname << " is called in function " << pname; - PathDiagnosticLocation CELoc = PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(),AC); - BugType * BT = new BugType(Checker, "known thread unsafe function called","ThreadSafety"); - std::unique_ptr R = llvm::make_unique(*BT,os.str(),CELoc); - R->setDeclWithIssue(AC->getDecl()); - R->addRange(CE->getSourceRange()); - BR.emitReport(std::move(R)); - std::string tname = "function-checker.txt.unsorted"; - std::string ostring = "function '"+ pname + "' known thread unsafe function '" + mname + "'.\n"; - support::writeLog(ostring,tname); - } - - -} - -void ThrUnsafeFCallChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager& mgr, - BugReporter &BR) const { - const SourceManager &SM = BR.getSourceManager(); - PathDiagnosticLocation DLoc =PathDiagnosticLocation::createBegin( MD, SM ); - if ( SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation()) ) return; - if (!MD->doesThisDeclarationHaveABody()) return; - clangcms::TUFWalker walker(this,BR, mgr.getAnalysisDeclContext(MD)); - walker.Visit(MD->getBody()); - return; -} - -void ThrUnsafeFCallChecker::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager& mgr, - BugReporter &BR) const { - const clang::SourceManager &SM = BR.getSourceManager(); - clang::ento::PathDiagnosticLocation DLoc =clang::ento::PathDiagnosticLocation::createBegin( TD, SM ); - if ( SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation()) ) return; - - for (auto I = TD->spec_begin(), - E = TD->spec_end(); I != E; ++I) - { - if (I->doesThisDeclarationHaveABody()) { - clangcms::TUFWalker walker(this,BR, mgr.getAnalysisDeclContext(*I)); - walker.Visit(I->getBody()); - } - } - return; -} - - + } + + void ThrUnsafeFCallChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager &mgr, BugReporter &BR) const { + const SourceManager &SM = BR.getSourceManager(); + PathDiagnosticLocation DLoc = PathDiagnosticLocation::createBegin(MD, SM); + if (SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation())) + return; + if (!MD->doesThisDeclarationHaveABody()) + return; + clangcms::TUFWalker walker(this, BR, mgr.getAnalysisDeclContext(MD)); + walker.Visit(MD->getBody()); + return; + } + + void ThrUnsafeFCallChecker::checkASTDecl(const FunctionTemplateDecl *TD, + AnalysisManager &mgr, + BugReporter &BR) const { + const clang::SourceManager &SM = BR.getSourceManager(); + clang::ento::PathDiagnosticLocation DLoc = clang::ento::PathDiagnosticLocation::createBegin(TD, SM); + if (SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation())) + return; + + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + if (I->doesThisDeclarationHaveABody()) { + clangcms::TUFWalker walker(this, BR, mgr.getAnalysisDeclContext(*I)); + walker.Visit(I->getBody()); + } + } + return; + } -} +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.h b/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.h index 8fbeb5df5e8cf..8cde356410446 100644 --- a/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.h +++ b/Utilities/StaticAnalyzers/src/ThrUnsafeFCallChecker.h @@ -20,23 +20,20 @@ namespace clangcms { -class ThrUnsafeFCallChecker : public clang::ento::Checker< clang::ento::check::ASTDecl, - clang::ento::check::ASTDecl > -{ - - -public: - - void checkASTDecl(const clang::CXXMethodDecl *CMD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - void checkASTDecl(const clang::FunctionTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - -private: - CmsException m_exception; -}; - -} + class ThrUnsafeFCallChecker : public clang::ento::Checker, + clang::ento::check::ASTDecl > { + public: + void checkASTDecl(const clang::CXXMethodDecl *CMD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + void checkASTDecl(const clang::FunctionTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + private: + CmsException m_exception; + }; + +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/TrunCastChecker.cpp b/Utilities/StaticAnalyzers/src/TrunCastChecker.cpp index 65f125f139e71..b81d44d51a83e 100644 --- a/Utilities/StaticAnalyzers/src/TrunCastChecker.cpp +++ b/Utilities/StaticAnalyzers/src/TrunCastChecker.cpp @@ -16,150 +16,200 @@ #include #include #include "TrunCastChecker.h" -#include "CmsSupport.h" +#include "CmsSupport.h" using namespace clang; using namespace ento; using namespace llvm; -namespace clangcms -{ +namespace clangcms { -class ICEVisitor: public clang::StmtVisitor { - const clang::ento::CheckerBase *Checker; - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext* AC; + class ICEVisitor : public clang::StmtVisitor { + const clang::ento::CheckerBase *Checker; + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; - public: - ICEVisitor(const clang::ento::CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac) - : Checker(checker), BR(br), AC(ac) {} - -void VisitImplicitCastExpr( ImplicitCastExpr *CE ); -void VisitBinaryOperator( BinaryOperator *BO ); -void VisitChildren( clang::Stmt *S ); -void VisitStmt( clang::Stmt *S ) { VisitChildren(S); } -}; + public: + ICEVisitor(const clang::ento::CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac) + : Checker(checker), BR(br), AC(ac) {} -void ICEVisitor::VisitChildren( clang::Stmt *S ) { - for ( auto I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); - } -} + void VisitImplicitCastExpr(ImplicitCastExpr *CE); + void VisitBinaryOperator(BinaryOperator *BO); + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + }; + void ICEVisitor::VisitChildren(clang::Stmt *S) { + for (auto I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } + } -void ICEVisitor::VisitBinaryOperator( BinaryOperator *BO ) -{ - const NamedDecl * ACD = dyn_cast_or_null(AC->getDecl()); - VisitChildren(BO); - std::string ename = "EventNumber_t"; - clang::Expr * LHS = BO->getLHS(); - clang::Expr * RHS = BO->getRHS(); - if (!LHS || !RHS) return; - std::string lname = LHS->getType().getAsString(); - std::string rname = RHS->getType().getAsString(); - if (IntegerLiteral::classof(LHS->IgnoreCasts()) || IntegerLiteral::classof(RHS->IgnoreCasts())) return; - if (!(lname == ename || rname == ename)) return; - if ( lname == ename && rname == ename ) return; - clang::QualType OTy; - clang::QualType TTy; - if (lname == ename && ImplicitCastExpr::classof(RHS) ) { - ImplicitCastExpr * ICE = dyn_cast_or_null(RHS); - TTy = BR.getContext().getCanonicalType(LHS->getType()); - OTy = BR.getContext().getCanonicalType(ICE->getSubExprAsWritten()->getType()); - } - if (rname == ename && ImplicitCastExpr::classof(LHS) ) { - ImplicitCastExpr * ICE = dyn_cast_or_null(LHS); - TTy = BR.getContext().getCanonicalType(RHS->getType()); - OTy = BR.getContext().getCanonicalType(ICE->getSubExprAsWritten()->getType()); - } - if ( TTy.isNull() || OTy.isNull() ) return; - QualType ToTy = TTy.getUnqualifiedType(); - QualType OrigTy = OTy.getUnqualifiedType(); - if (!(ToTy->isIntegerType()||ToTy->isFloatingType()) ) return; - if ( ToTy->isBooleanType() ) return; - CharUnits size_otype = BR.getContext().getTypeSizeInChars(OrigTy); - CharUnits size_ttype = BR.getContext().getTypeSizeInChars(ToTy); - std::string oname = OrigTy.getAsString(); - std::string tname = ToTy.getAsString(); - if ( ToTy->isFloatingType() ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os <<"Cast-to type, "<getSourceRange()); - } - if ( (size_otype > size_ttype) ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os <<"Cast-to type, "<getSourceRange()); - } - if ( ( size_otype == size_ttype ) && (ToTy->hasSignedIntegerRepresentation() && OrigTy->hasUnsignedIntegerRepresentation() || - ToTy->hasUnsignedIntegerRepresentation() && OrigTy->hasSignedIntegerRepresentation() ) ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os <<"Cast-to type, "<getSourceRange()); - } - return; -return; -} + void ICEVisitor::VisitBinaryOperator(BinaryOperator *BO) { + const NamedDecl *ACD = dyn_cast_or_null(AC->getDecl()); + VisitChildren(BO); + std::string ename = "EventNumber_t"; + clang::Expr *LHS = BO->getLHS(); + clang::Expr *RHS = BO->getRHS(); + if (!LHS || !RHS) + return; + std::string lname = LHS->getType().getAsString(); + std::string rname = RHS->getType().getAsString(); + if (IntegerLiteral::classof(LHS->IgnoreCasts()) || IntegerLiteral::classof(RHS->IgnoreCasts())) + return; + if (!(lname == ename || rname == ename)) + return; + if (lname == ename && rname == ename) + return; + clang::QualType OTy; + clang::QualType TTy; + if (lname == ename && ImplicitCastExpr::classof(RHS)) { + ImplicitCastExpr *ICE = dyn_cast_or_null(RHS); + TTy = BR.getContext().getCanonicalType(LHS->getType()); + OTy = BR.getContext().getCanonicalType(ICE->getSubExprAsWritten()->getType()); + } + if (rname == ename && ImplicitCastExpr::classof(LHS)) { + ImplicitCastExpr *ICE = dyn_cast_or_null(LHS); + TTy = BR.getContext().getCanonicalType(RHS->getType()); + OTy = BR.getContext().getCanonicalType(ICE->getSubExprAsWritten()->getType()); + } + if (TTy.isNull() || OTy.isNull()) + return; + QualType ToTy = TTy.getUnqualifiedType(); + QualType OrigTy = OTy.getUnqualifiedType(); + if (!(ToTy->isIntegerType() || ToTy->isFloatingType())) + return; + if (ToTy->isBooleanType()) + return; + CharUnits size_otype = BR.getContext().getTypeSizeInChars(OrigTy); + CharUnits size_ttype = BR.getContext().getTypeSizeInChars(ToTy); + std::string oname = OrigTy.getAsString(); + std::string tname = ToTy.getAsString(); + if (ToTy->isFloatingType()) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << "Cast-to type, " << tname << ". Cast-from type, " << oname << " . " << support::getQualifiedName(*(ACD)); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(BO, BR.getSourceManager(), AC); + BR.EmitBasicReport(ACD, + CheckName(), + "implicit cast of int type to float type", + "CMS code rules", + os.str(), + CELoc, + BO->getSourceRange()); + } + if ((size_otype > size_ttype)) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << "Cast-to type, " << tname << ". Cast-from type, " << oname << ". Cast may result in truncation. " + << support::getQualifiedName(*(ACD)); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(BO, BR.getSourceManager(), AC); + BR.EmitBasicReport(ACD, + CheckName(), + "implicit cast of int type to smaller int type could truncate", + "CMS code rules", + os.str(), + CELoc, + BO->getSourceRange()); + } + if ((size_otype == size_ttype) && + (ToTy->hasSignedIntegerRepresentation() && OrigTy->hasUnsignedIntegerRepresentation() || + ToTy->hasUnsignedIntegerRepresentation() && OrigTy->hasSignedIntegerRepresentation())) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << "Cast-to type, " << tname << ". Cast-from type, " << oname << ". Changes int sign type. " + << support::getQualifiedName(*(ACD)); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(BO, BR.getSourceManager(), AC); + BR.EmitBasicReport( + ACD, CheckName(), "implicit cast ins sign type", "CMS code rules", os.str(), CELoc, BO->getSourceRange()); + } + return; + return; + } -void ICEVisitor::VisitImplicitCastExpr( ImplicitCastExpr *CE ) -{ - const NamedDecl * ACD = dyn_cast(AC->getDecl()); - VisitChildren(CE); - const Expr * SE = CE->getSubExprAsWritten(); - std::string sename = SE->getType().getAsString(); - const clang::Expr *E = CE->getSubExpr(); - if (!(sename=="EventNumber_t")) return; - QualType OTy = BR.getContext().getCanonicalType(E->getType()); - QualType TTy = BR.getContext().getCanonicalType(CE->getType()); - QualType ToTy = TTy.getUnqualifiedType(); - QualType OrigTy = OTy.getUnqualifiedType(); - if (!(ToTy->isIntegerType()||ToTy->isFloatingType()) ) return; - if ( ToTy->isBooleanType() ) return; - CharUnits size_otype = BR.getContext().getTypeSizeInChars(OrigTy); - CharUnits size_ttype = BR.getContext().getTypeSizeInChars(ToTy); - std::string oname = OrigTy.getAsString(); - std::string tname = ToTy.getAsString(); - if ( ToTy->isFloatingType() ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os <<"Cast-to type, "<getSourceRange()); - } - if ( (size_otype > size_ttype) ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os <<"Cast-to type, "<getSourceRange()); - } - if ( ToTy->hasSignedIntegerRepresentation() && OrigTy->hasUnsignedIntegerRepresentation() || - ToTy->hasUnsignedIntegerRepresentation() && OrigTy->hasSignedIntegerRepresentation() ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os <<"Cast-to type, "<getSourceRange()); - } - return; -} + void ICEVisitor::VisitImplicitCastExpr(ImplicitCastExpr *CE) { + const NamedDecl *ACD = dyn_cast(AC->getDecl()); + VisitChildren(CE); + const Expr *SE = CE->getSubExprAsWritten(); + std::string sename = SE->getType().getAsString(); + const clang::Expr *E = CE->getSubExpr(); + if (!(sename == "EventNumber_t")) + return; + QualType OTy = BR.getContext().getCanonicalType(E->getType()); + QualType TTy = BR.getContext().getCanonicalType(CE->getType()); + QualType ToTy = TTy.getUnqualifiedType(); + QualType OrigTy = OTy.getUnqualifiedType(); + if (!(ToTy->isIntegerType() || ToTy->isFloatingType())) + return; + if (ToTy->isBooleanType()) + return; + CharUnits size_otype = BR.getContext().getTypeSizeInChars(OrigTy); + CharUnits size_ttype = BR.getContext().getTypeSizeInChars(ToTy); + std::string oname = OrigTy.getAsString(); + std::string tname = ToTy.getAsString(); + if (ToTy->isFloatingType()) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << "Cast-to type, " << tname << ". Cast-from type, " << oname << " . " << support::getQualifiedName(*(ACD)); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + BR.EmitBasicReport(ACD, + CheckName(), + "implicit cast of int type to float type", + "CMS code rules", + os.str(), + CELoc, + CE->getSourceRange()); + } + if ((size_otype > size_ttype)) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << "Cast-to type, " << tname << ". Cast-from type, " << oname << ". Cast may result in truncation. " + << support::getQualifiedName(*(ACD)); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + BR.EmitBasicReport(ACD, + CheckName(), + "implicit cast of int type to smaller int type could truncate", + "CMS code rules", + os.str(), + CELoc, + CE->getSourceRange()); + } + if (ToTy->hasSignedIntegerRepresentation() && OrigTy->hasUnsignedIntegerRepresentation() || + ToTy->hasUnsignedIntegerRepresentation() && OrigTy->hasSignedIntegerRepresentation()) { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << "Cast-to type, " << tname << ". Cast-from type, " << oname << ". Changes int sign type. " + << support::getQualifiedName(*(ACD)); + clang::ento::PathDiagnosticLocation CELoc = + clang::ento::PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), AC); + BR.EmitBasicReport(ACD, + CheckName(), + "implicit cast changes int sign type", + "CMS code rules", + os.str(), + CELoc, + CE->getSourceRange()); + } + return; + } -void TrunCastChecker::checkASTDecl(const CXXRecordDecl *D, AnalysisManager& Mgr,BugReporter &BR) const { - for ( auto I = D->method_begin(), E = D->method_end(); I != E; ++I) { - if ( !llvm::isa((*I)) ) continue; - clang::CXXMethodDecl * MD = llvm::cast((*I)->getMostRecentDecl()); - if ( ! MD->hasBody() ) continue; - clang::Stmt *Body = MD->getBody(); - ICEVisitor icevisitor(this, BR, Mgr.getAnalysisDeclContext(MD)); - icevisitor.Visit(Body); - } -} + void TrunCastChecker::checkASTDecl(const CXXRecordDecl *D, AnalysisManager &Mgr, BugReporter &BR) const { + for (auto I = D->method_begin(), E = D->method_end(); I != E; ++I) { + if (!llvm::isa((*I))) + continue; + clang::CXXMethodDecl *MD = llvm::cast((*I)->getMostRecentDecl()); + if (!MD->hasBody()) + continue; + clang::Stmt *Body = MD->getBody(); + ICEVisitor icevisitor(this, BR, Mgr.getAnalysisDeclContext(MD)); + icevisitor.Visit(Body); + } + } -} +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/TrunCastChecker.h b/Utilities/StaticAnalyzers/src/TrunCastChecker.h index d74cfc98dc64a..375c81b77d536 100644 --- a/Utilities/StaticAnalyzers/src/TrunCastChecker.h +++ b/Utilities/StaticAnalyzers/src/TrunCastChecker.h @@ -4,21 +4,22 @@ #include #include #include -#include "CmsException.h" +#include "CmsException.h" #include "FWCore/Utilities/interface/thread_safety_macros.h" namespace clangcms { -class TrunCastChecker: public clang::ento::Checker< clang::ento::check::ASTDecl > { -public: - CMS_THREAD_SAFE mutable std::unique_ptr BT; - void checkASTDecl(const clang::CXXRecordDecl *D, clang::ento::AnalysisManager& Mgr, clang::ento::BugReporter &BR) const; + class TrunCastChecker : public clang::ento::Checker > { + public: + CMS_THREAD_SAFE mutable std::unique_ptr BT; + void checkASTDecl(const clang::CXXRecordDecl *D, + clang::ento::AnalysisManager &Mgr, + clang::ento::BugReporter &BR) const; -private: - CmsException m_exception; + private: + CmsException m_exception; + }; -}; - -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/UsingNamespace.cpp b/Utilities/StaticAnalyzers/src/UsingNamespace.cpp index 8401813b9d3c6..8e0cb54aecaba 100644 --- a/Utilities/StaticAnalyzers/src/UsingNamespace.cpp +++ b/Utilities/StaticAnalyzers/src/UsingNamespace.cpp @@ -9,36 +9,41 @@ #include "CmsSupport.h" using namespace clangcms; -void UsingNamespace::checkASTDecl (const clang::UsingDirectiveDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const -{ - if (isDeclOK(D,BR)) return; - reportBug("'using namespace '",D,BR); +void UsingNamespace::checkASTDecl(const clang::UsingDirectiveDecl *D, + clang::ento::AnalysisManager &Mgr, + clang::ento::BugReporter &BR) const { + if (isDeclOK(D, BR)) + return; + reportBug("'using namespace '", D, BR); } -void UsingNamespace::checkASTDecl (const clang::UsingDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const -{ - if (isDeclOK(D,BR)) return; - std::string NS = D->getQualifier ()->getAsNamespace ()->getNameAsString (); - if (strcmp(NS.c_str(),"std")!=0) return; - reportBug("'using std:: '",D,BR); +void UsingNamespace::checkASTDecl(const clang::UsingDecl *D, + clang::ento::AnalysisManager &Mgr, + clang::ento::BugReporter &BR) const { + if (isDeclOK(D, BR)) + return; + std::string NS = D->getQualifier()->getAsNamespace()->getNameAsString(); + if (strcmp(NS.c_str(), "std") != 0) + return; + reportBug("'using std:: '", D, BR); } -bool UsingNamespace::isDeclOK (const clang::NamedDecl *D, clang::ento::BugReporter &BR) const -{ - if (D->getDeclContext ()->getParent()!=nullptr) return true; - const char *sfile=BR.getSourceManager().getPresumedLoc(D->getLocation ()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return true; +bool UsingNamespace::isDeclOK(const clang::NamedDecl *D, clang::ento::BugReporter &BR) const { + if (D->getDeclContext()->getParent() != nullptr) + return true; + const char *sfile = BR.getSourceManager().getPresumedLoc(D->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return true; size_t flen = strlen(sfile); - if ((sfile[flen-2] != '.') || (sfile[flen-1] != 'h')) return true; + if ((sfile[flen - 2] != '.') || (sfile[flen - 1] != 'h')) + return true; return false; } -void UsingNamespace::reportBug (const char* bug, const clang::Decl *D, clang::ento::BugReporter &BR) const -{ +void UsingNamespace::reportBug(const char *bug, const clang::Decl *D, clang::ento::BugReporter &BR) const { clang::ento::PathDiagnosticLocation DLoc = clang::ento::PathDiagnosticLocation::createBegin(D, BR.getSourceManager()); std::string buf; llvm::raw_string_ostream os(buf); os << bug << " in headers files."; - BR.EmitBasicReport(D, this, "using namespace in header files","CMS code rules",os.str(), DLoc); + BR.EmitBasicReport(D, this, "using namespace in header files", "CMS code rules", os.str(), DLoc); } - diff --git a/Utilities/StaticAnalyzers/src/UsingNamespace.h b/Utilities/StaticAnalyzers/src/UsingNamespace.h index f06a6251798a0..84821376601b0 100644 --- a/Utilities/StaticAnalyzers/src/UsingNamespace.h +++ b/Utilities/StaticAnalyzers/src/UsingNamespace.h @@ -12,14 +12,17 @@ #include namespace clangcms { - class UsingNamespace : public clang::ento::Checker< clang::ento::check::ASTDecl ,clang::ento::check::ASTDecl > - { + class UsingNamespace : public clang::ento::Checker, + clang::ento::check::ASTDecl > { public: - void checkASTDecl (const clang::UsingDirectiveDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const; - void checkASTDecl (const clang::UsingDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const; + void checkASTDecl(const clang::UsingDirectiveDecl *D, + clang::ento::AnalysisManager &Mgr, + clang::ento::BugReporter &BR) const; + void checkASTDecl(const clang::UsingDecl *D, clang::ento::AnalysisManager &Mgr, clang::ento::BugReporter &BR) const; + private: bool isDeclOK(const clang::NamedDecl *D, clang::ento::BugReporter &BR) const; - void reportBug(const char* bug, const clang::Decl *D, clang::ento::BugReporter &BR) const; + void reportBug(const char *bug, const clang::Decl *D, clang::ento::BugReporter &BR) const; }; -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/edmChecker.cpp b/Utilities/StaticAnalyzers/src/edmChecker.cpp index afd74165eab6a..93584c76d059f 100644 --- a/Utilities/StaticAnalyzers/src/edmChecker.cpp +++ b/Utilities/StaticAnalyzers/src/edmChecker.cpp @@ -5,23 +5,33 @@ using namespace llvm; namespace clangcms { -void edmChecker::checkASTDecl(const clang::CXXRecordDecl *RD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const { - if (!RD->hasDefinition()) return; - const clang::SourceManager &SM = BR.getSourceManager(); - for ( auto J=RD->bases_begin(), F=RD->bases_end();J != F; ++J) { - auto BRD = J->getType()->getAsCXXRecordDecl(); - if (!BRD) continue; - std::string bname = BRD->getQualifiedNameAsString(); - if (bname=="edm::EDProducer" || bname=="edm::EDFilter" || bname=="edm::EDAnalyzer" || bname=="edm::OutputModule" ) { - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - os << RD->getQualifiedNameAsString() << " inherits from edm::EDProducer,edm::EDFilter,edm::EDAnalyzer, or edm::OutputModule"; - os << "\n"; - clang::ento::PathDiagnosticLocation ELoc =clang::ento::PathDiagnosticLocation::createBegin( RD, SM ); - BR.EmitBasicReport(RD, this, "inherits from edm::EDProducer,edm::EDFilter,edm::EDAnalyzer, or edm::OutputModule","ThreadSafety",os.str(),ELoc); - } - } -} //end of class + void edmChecker::checkASTDecl(const clang::CXXRecordDecl *RD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + if (!RD->hasDefinition()) + return; + const clang::SourceManager &SM = BR.getSourceManager(); + for (auto J = RD->bases_begin(), F = RD->bases_end(); J != F; ++J) { + auto BRD = J->getType()->getAsCXXRecordDecl(); + if (!BRD) + continue; + std::string bname = BRD->getQualifiedNameAsString(); + if (bname == "edm::EDProducer" || bname == "edm::EDFilter" || bname == "edm::EDAnalyzer" || + bname == "edm::OutputModule") { + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + os << RD->getQualifiedNameAsString() + << " inherits from edm::EDProducer,edm::EDFilter,edm::EDAnalyzer, or edm::OutputModule"; + os << "\n"; + clang::ento::PathDiagnosticLocation ELoc = clang::ento::PathDiagnosticLocation::createBegin(RD, SM); + BR.EmitBasicReport(RD, + this, + "inherits from edm::EDProducer,edm::EDFilter,edm::EDAnalyzer, or edm::OutputModule", + "ThreadSafety", + os.str(), + ELoc); + } + } + } //end of class -} +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/edmChecker.h b/Utilities/StaticAnalyzers/src/edmChecker.h index b0650b8e2e122..325878d2c2e6d 100644 --- a/Utilities/StaticAnalyzers/src/edmChecker.h +++ b/Utilities/StaticAnalyzers/src/edmChecker.h @@ -18,16 +18,15 @@ namespace clangcms { -class edmChecker : public clang::ento::Checker > { + class edmChecker : public clang::ento::Checker > { + public: + void checkASTDecl(const clang::CXXRecordDecl *CRD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + private: + CmsException m_exception; + }; -public: - void checkASTDecl(const clang::CXXRecordDecl *CRD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - -private: - CmsException m_exception; -}; - -} +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/getByChecker.cpp b/Utilities/StaticAnalyzers/src/getByChecker.cpp index 63f1a891e53b3..9d2964c91dbbe 100644 --- a/Utilities/StaticAnalyzers/src/getByChecker.cpp +++ b/Utilities/StaticAnalyzers/src/getByChecker.cpp @@ -5,172 +5,159 @@ using namespace llvm; namespace clangcms { -class Walker : public clang::StmtVisitor { - const CheckerBase *Checker; - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext *AC; + class Walker : public clang::StmtVisitor { + const CheckerBase *Checker; + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; -public: - Walker( const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac ) - : Checker(checker), - BR(br), - AC(ac) {} + public: + Walker(const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac) + : Checker(checker), BR(br), AC(ac) {} - void VisitChildren(clang::Stmt *S ); - void VisitStmt( clang::Stmt *S) { VisitChildren(S); } - void VisitCXXMemberCallExpr( clang::CXXMemberCallExpr *CE ); - -}; + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + void VisitCXXMemberCallExpr(clang::CXXMemberCallExpr *CE); + }; -void Walker::VisitChildren( clang::Stmt *S) { - for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); - } -} - - - -void Walker::VisitCXXMemberCallExpr( CXXMemberCallExpr *CE ) { - LangOptions LangOpts; - LangOpts.CPlusPlus = true; - PrintingPolicy Policy(LangOpts); - const Decl * D = AC->getDecl(); - std::string dname =""; - if (const NamedDecl * ND = llvm::dyn_cast_or_null(D)) dname = ND->getQualifiedNameAsString(); - CXXMethodDecl * MD = CE->getMethodDecl(); - if (!MD) return; - std::string mname = MD->getQualifiedNameAsString(); -// llvm::errs()<<"Parent Decl: '"<printPretty(llvm::errs(),0,Policy); -// llvm::errs()<<"'\n"; -// if (!MD) return; - llvm::SmallString<100> buf; - llvm::raw_svector_ostream os(buf); - if ( mname == "edm::Event::getByLabel" || mname == "edm::Event::getManyByType" ) { -// if (const CXXRecordDecl * RD = llvm::dyn_cast_or_null(D)->getParent() ) { -// llvm::errs()<<"class "<getQualifiedNameAsString()<<"\n"; -// llvm::errs()<<"\n"; -// } - os<<"function '"; - llvm::dyn_cast(D)->getNameForDiagnostic(os,Policy,true); - os<<"' "; -// os<<"call expression '"; -// CE->printPretty(os,0,Policy); -// os<<"' "; - if (mname == "edm::Event::getByLabel") { - os <<"calls edm::Event::getByLabel with arguments '"; - QualType QT; - for ( auto I=CE->arg_begin(), E=CE->arg_end(); I != E; ++I) { - QT=(*I)->getType(); - std::string qtname = QT.getCanonicalType().getAsString(); - if ( qtname.substr(0,17)=="class edm::Handle" ) { -// os<<"argument name '"; -// (*I)->printPretty(os,0,Policy); -// os<<"' "; - const CXXRecordDecl * RD = QT->getAsCXXRecordDecl(); - std::string rname = RD->getQualifiedNameAsString(); - os << rname<<" "; - const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); - for (unsigned J = 0, F = SD->getTemplateArgs().size(); J!=F; ++J) { - SD->getTemplateArgs().data()[J].print(Policy,os); - os<<", "; - } - } - else { - os<<" "<< qtname <<" "; - (*I)->printPretty(os,nullptr,Policy); - os <<", "; - } - } - os <<"'\n"; - } else { - os <<"calls edm::Event::getManyByType with argument '"; - QualType QT = (*CE->arg_begin())->getType(); - const CXXRecordDecl * RD = QT->getAsCXXRecordDecl(); - os << "getManyByType , "; - const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); - const TemplateArgument TA = SD->getTemplateArgs().data()[0]; - const QualType AQT = TA.getAsType(); - const CXXRecordDecl * SRD = AQT->getAsCXXRecordDecl(); - os << SRD->getQualifiedNameAsString()<<" "; - const ClassTemplateSpecializationDecl *SVD = dyn_cast(SRD); - for (unsigned J = 0, F = SVD->getTemplateArgs().size(); J!=F; ++J) { - SVD->getTemplateArgs().data()[J].print(Policy,os); - os<<", "; - } - - } + void Walker::VisitChildren(clang::Stmt *S) { + for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } + } -// llvm::errs()< R = llvm::make_unique(*BT,os.str(),CELoc); - R->addRange(CE->getSourceRange()); - BR.emitReport(std::move(R)); - } - else { - for (auto I=CE->arg_begin(), E=CE->arg_end(); I != E; ++I) { - QualType QT = (*I)->getType(); - std::string qtname = QT.getAsString(); -// if (qtname.find(" edm::Event") != std::string::npos ) llvm::errs()<<"arg type '" << qtname <<"'\n"; - if ( qtname=="edm::Event" || qtname=="const edm::Event" || - qtname=="edm::Event *" || qtname=="const edm::Event *" ) { - std::string tname; - os<<"function '"<getNameForDiagnostic(os,Policy,true); - os<<"' with argument of type '"<printPretty(llvm::errs(),0,Policy); -// llvm::errs()<<" argument name "; -// (*I)->printPretty(llvm::errs(),0,Policy); -// llvm::errs()<<" "< R = llvm::make_unique(*BT,os.str(),CELoc); - R->addRange(CE->getSourceRange()); - BR.emitReport(std::move(R)); - } - } - } - - -} + void Walker::VisitCXXMemberCallExpr(CXXMemberCallExpr *CE) { + LangOptions LangOpts; + LangOpts.CPlusPlus = true; + PrintingPolicy Policy(LangOpts); + const Decl *D = AC->getDecl(); + std::string dname = ""; + if (const NamedDecl *ND = llvm::dyn_cast_or_null(D)) + dname = ND->getQualifiedNameAsString(); + CXXMethodDecl *MD = CE->getMethodDecl(); + if (!MD) + return; + std::string mname = MD->getQualifiedNameAsString(); + // llvm::errs()<<"Parent Decl: '"<printPretty(llvm::errs(),0,Policy); + // llvm::errs()<<"'\n"; + // if (!MD) return; + llvm::SmallString<100> buf; + llvm::raw_svector_ostream os(buf); + if (mname == "edm::Event::getByLabel" || mname == "edm::Event::getManyByType") { + // if (const CXXRecordDecl * RD = llvm::dyn_cast_or_null(D)->getParent() ) { + // llvm::errs()<<"class "<getQualifiedNameAsString()<<"\n"; + // llvm::errs()<<"\n"; + // } + os << "function '"; + llvm::dyn_cast(D)->getNameForDiagnostic(os, Policy, true); + os << "' "; + // os<<"call expression '"; + // CE->printPretty(os,0,Policy); + // os<<"' "; + if (mname == "edm::Event::getByLabel") { + os << "calls edm::Event::getByLabel with arguments '"; + QualType QT; + for (auto I = CE->arg_begin(), E = CE->arg_end(); I != E; ++I) { + QT = (*I)->getType(); + std::string qtname = QT.getCanonicalType().getAsString(); + if (qtname.substr(0, 17) == "class edm::Handle") { + // os<<"argument name '"; + // (*I)->printPretty(os,0,Policy); + // os<<"' "; + const CXXRecordDecl *RD = QT->getAsCXXRecordDecl(); + std::string rname = RD->getQualifiedNameAsString(); + os << rname << " "; + const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); + for (unsigned J = 0, F = SD->getTemplateArgs().size(); J != F; ++J) { + SD->getTemplateArgs().data()[J].print(Policy, os); + os << ", "; + } + } else { + os << " " << qtname << " "; + (*I)->printPretty(os, nullptr, Policy); + os << ", "; + } + } + os << "'\n"; + } else { + os << "calls edm::Event::getManyByType with argument '"; + QualType QT = (*CE->arg_begin())->getType(); + const CXXRecordDecl *RD = QT->getAsCXXRecordDecl(); + os << "getManyByType , "; + const ClassTemplateSpecializationDecl *SD = dyn_cast(RD); + const TemplateArgument TA = SD->getTemplateArgs().data()[0]; + const QualType AQT = TA.getAsType(); + const CXXRecordDecl *SRD = AQT->getAsCXXRecordDecl(); + os << SRD->getQualifiedNameAsString() << " "; + const ClassTemplateSpecializationDecl *SVD = dyn_cast(SRD); + for (unsigned J = 0, F = SVD->getTemplateArgs().size(); J != F; ++J) { + SVD->getTemplateArgs().data()[J].print(Policy, os); + os << ", "; + } + } -void getByChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager& mgr, - BugReporter &BR) const { - const SourceManager &SM = BR.getSourceManager(); - PathDiagnosticLocation DLoc =PathDiagnosticLocation::createBegin( MD, SM ); - if ( SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation()) ) return; - if (!MD->doesThisDeclarationHaveABody()) return; - clangcms::Walker walker(this,BR, mgr.getAnalysisDeclContext(MD)); - walker.Visit(MD->getBody()); - return; -} - -void getByChecker::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager& mgr, - BugReporter &BR) const { - const clang::SourceManager &SM = BR.getSourceManager(); - clang::ento::PathDiagnosticLocation DLoc =clang::ento::PathDiagnosticLocation::createBegin( TD, SM ); - if ( SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation()) ) return; + // llvm::errs()< R = llvm::make_unique(*BT, os.str(), CELoc); + R->addRange(CE->getSourceRange()); + BR.emitReport(std::move(R)); + } else { + for (auto I = CE->arg_begin(), E = CE->arg_end(); I != E; ++I) { + QualType QT = (*I)->getType(); + std::string qtname = QT.getAsString(); + // if (qtname.find(" edm::Event") != std::string::npos ) llvm::errs()<<"arg type '" << qtname <<"'\n"; + if (qtname == "edm::Event" || qtname == "const edm::Event" || qtname == "edm::Event *" || + qtname == "const edm::Event *") { + std::string tname; + os << "function '" << dname << "' "; + os << "calls '"; + MD->getNameForDiagnostic(os, Policy, true); + os << "' with argument of type '" << qtname << "'\n"; + // llvm::errs()<<"\n"; + // llvm::errs()<<"call expression passed edm::Event "; + // CE->printPretty(llvm::errs(),0,Policy); + // llvm::errs()<<" argument name "; + // (*I)->printPretty(llvm::errs(),0,Policy); + // llvm::errs()<<" "< R = llvm::make_unique(*BT, os.str(), CELoc); + R->addRange(CE->getSourceRange()); + BR.emitReport(std::move(R)); + } + } + } + } - for (auto I = TD->spec_begin(), - E = TD->spec_end(); I != E; ++I) - { - if (I->doesThisDeclarationHaveABody()) { - clangcms::Walker walker(this,BR, mgr.getAnalysisDeclContext(*I)); - walker.Visit(I->getBody()); - } - } - return; -} + void getByChecker::checkASTDecl(const CXXMethodDecl *MD, AnalysisManager &mgr, BugReporter &BR) const { + const SourceManager &SM = BR.getSourceManager(); + PathDiagnosticLocation DLoc = PathDiagnosticLocation::createBegin(MD, SM); + if (SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation())) + return; + if (!MD->doesThisDeclarationHaveABody()) + return; + clangcms::Walker walker(this, BR, mgr.getAnalysisDeclContext(MD)); + walker.Visit(MD->getBody()); + return; + } + void getByChecker::checkASTDecl(const FunctionTemplateDecl *TD, AnalysisManager &mgr, BugReporter &BR) const { + const clang::SourceManager &SM = BR.getSourceManager(); + clang::ento::PathDiagnosticLocation DLoc = clang::ento::PathDiagnosticLocation::createBegin(TD, SM); + if (SM.isInSystemHeader(DLoc.asLocation()) || SM.isInExternCSystemHeader(DLoc.asLocation())) + return; + for (auto I = TD->spec_begin(), E = TD->spec_end(); I != E; ++I) { + if (I->doesThisDeclarationHaveABody()) { + clangcms::Walker walker(this, BR, mgr.getAnalysisDeclContext(*I)); + walker.Visit(I->getBody()); + } + } + return; + } -} +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/getByChecker.h b/Utilities/StaticAnalyzers/src/getByChecker.h index b8655f4d9056c..1a94ed308c131 100644 --- a/Utilities/StaticAnalyzers/src/getByChecker.h +++ b/Utilities/StaticAnalyzers/src/getByChecker.h @@ -20,22 +20,20 @@ namespace clangcms { -class getByChecker : public clang::ento::Checker< clang::ento::check::ASTDecl, - clang::ento::check::ASTDecl > -{ - -public: - - void checkASTDecl(const clang::CXXMethodDecl *CMD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - void checkASTDecl(const clang::FunctionTemplateDecl *TD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; - - -private: - CmsException m_exception; -}; - -} + class getByChecker : public clang::ento::Checker, + clang::ento::check::ASTDecl > { + public: + void checkASTDecl(const clang::CXXMethodDecl *CMD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + void checkASTDecl(const clang::FunctionTemplateDecl *TD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + + private: + CmsException m_exception; + }; + +} // namespace clangcms #endif diff --git a/Utilities/StaticAnalyzers/src/getParamDumper.cpp b/Utilities/StaticAnalyzers/src/getParamDumper.cpp index 66dcef0bbb78a..8b7aa4d843d67 100644 --- a/Utilities/StaticAnalyzers/src/getParamDumper.cpp +++ b/Utilities/StaticAnalyzers/src/getParamDumper.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include "CmsException.h" #include "CmsSupport.h" @@ -33,263 +33,265 @@ using namespace llvm; namespace clangcms { -void getParamDumper::analyzerEval(const clang::CallExpr *CE, clang::ento::CheckerContext &C) const { + void getParamDumper::analyzerEval(const clang::CallExpr *CE, clang::ento::CheckerContext &C) const { + if (!C.getSourceManager().isInMainFile(CE->getExprLoc())) + return; - if ( ! C.getSourceManager().isInMainFile(CE->getExprLoc()) ) return; + const FunctionDecl *FD = CE->getDirectCallee(); - const FunctionDecl * FD = CE->getDirectCallee(); - - if (!FD) return; + if (!FD) + return; std::string mname = support::getQualifiedName(*FD); - const char *sfile=C.getSourceManager().getPresumedLoc(CE->getExprLoc()).getFilename(); + const char *sfile = C.getSourceManager().getPresumedLoc(CE->getExprLoc()).getFilename(); std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; + if (!support::isInterestingLocation(sname)) + return; std::string mdname; - const FunctionDecl * MD = C.getCurrentAnalysisDeclContext()->getDecl()->getAsFunction(); - if (!MD) return; + const FunctionDecl *MD = C.getCurrentAnalysisDeclContext()->getDecl()->getAsFunction(); + if (!MD) + return; mdname = MD->getQualifiedNameAsString(); - for ( unsigned I=0, E=MD->getNumParams(); I != E; ++I) { - std::string ps = "const class edm::ParameterSet "; - std::string ups = "const class edm::UntrackedParameterSet "; - std::string pname = MD->getParamDecl(I)->getQualifiedNameAsString(); - std::string qname = MD->getParamDecl(I)->getType().getCanonicalType().getAsString(); -// if (qname.substr(0,ps.length()) == ps || qname.substr(0,ups.length()) == ups) { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "in function decl '"<< mdname << "' with parameter '"<< qname << " " << pname <<"'\n"; -// } - } - const CXXMemberCallExpr * CXE = llvm::dyn_cast_or_null(CE); - if (!CXE) return; - const Expr * IOA = CXE->getImplicitObjectArgument(); + for (unsigned I = 0, E = MD->getNumParams(); I != E; ++I) { + std::string ps = "const class edm::ParameterSet "; + std::string ups = "const class edm::UntrackedParameterSet "; + std::string pname = MD->getParamDecl(I)->getQualifiedNameAsString(); + std::string qname = MD->getParamDecl(I)->getType().getCanonicalType().getAsString(); + // if (qname.substr(0,ps.length()) == ps || qname.substr(0,ups.length()) == ups) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "in function decl '" << mdname << "' with parameter '" << qname << " " << pname << "'\n"; + // } + } + const CXXMemberCallExpr *CXE = llvm::dyn_cast_or_null(CE); + if (!CXE) + return; + const Expr *IOA = CXE->getImplicitObjectArgument(); std::string tname = "getparam-dumper.txt.unsorted"; std::string gp = "edm::ParameterSet::getParameter"; std::string gup = "edm::ParameterSet::getUntrackedParameter"; - if (mname.substr(0,gp.length()) == gp || mname.substr(0,gup.length()) == gup ) { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "in function decl '" << mdname << "' member function call '"; - clang::LangOptions LangOpts; - LangOpts.CPlusPlus = true; - clang::PrintingPolicy Policy(LangOpts); - os << support::getQualifiedName(*(CXE->getMethodDecl())); - os << "' with args '"; - for ( unsigned I=0, E=CE->getNumArgs(); I != E; ++I) { - if (I) os <<", "; - CE->getArg(I)->printPretty(os,nullptr,Policy); - } - os << "' with implicit object '"; - const Expr * E = IOA->IgnoreParenNoopCasts(C.getASTContext()); - QualType QE = E->getType().getCanonicalType(); - os << QE.getAsString()<<" "; - switch( E->getStmtClass() ) { - case Stmt::MemberExprClass: - os << dyn_cast(E)->getMemberDecl()->getQualifiedNameAsString(); - break; - case Stmt::DeclRefExprClass: - os << dyn_cast(E)->getDecl()->getQualifiedNameAsString(); - break; - case Stmt::CXXOperatorCallExprClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - case Stmt::CXXBindTemporaryExprClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - case Stmt::CXXMemberCallExprClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - case Stmt::UnaryOperatorClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - default: - E->printPretty(os,nullptr,Policy); - os << " unhandled expr class " <getStmtClassName(); - } - os<<"'\n"; - - support::writeLog(os.str(),tname); + if (mname.substr(0, gp.length()) == gp || mname.substr(0, gup.length()) == gup) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "in function decl '" << mdname << "' member function call '"; + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + os << support::getQualifiedName(*(CXE->getMethodDecl())); + os << "' with args '"; + for (unsigned I = 0, E = CE->getNumArgs(); I != E; ++I) { + if (I) + os << ", "; + CE->getArg(I)->printPretty(os, nullptr, Policy); + } + os << "' with implicit object '"; + const Expr *E = IOA->IgnoreParenNoopCasts(C.getASTContext()); + QualType QE = E->getType().getCanonicalType(); + os << QE.getAsString() << " "; + switch (E->getStmtClass()) { + case Stmt::MemberExprClass: + os << dyn_cast(E)->getMemberDecl()->getQualifiedNameAsString(); + break; + case Stmt::DeclRefExprClass: + os << dyn_cast(E)->getDecl()->getQualifiedNameAsString(); + break; + case Stmt::CXXOperatorCallExprClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + case Stmt::CXXBindTemporaryExprClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + case Stmt::CXXMemberCallExprClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + case Stmt::UnaryOperatorClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + default: + E->printPretty(os, nullptr, Policy); + os << " unhandled expr class " << E->getStmtClassName(); + } + os << "'\n"; + + support::writeLog(os.str(), tname); + } + return; } - return ; -} - - -bool getParamDumper::evalCall(const CallExpr *CE, CheckerContext &C) const { - - FnCheck Handler = llvm::StringSwitch(C.getCalleeName(CE)) - .Case("getParameter",&getParamDumper::analyzerEval) - .Case("getUntrackedParameter",&getParamDumper::analyzerEval) - .Default(nullptr); - - if (!Handler) return false; - (this->*Handler)(CE,C); + bool getParamDumper::evalCall(const CallExpr *CE, CheckerContext &C) const { + FnCheck Handler = llvm::StringSwitch(C.getCalleeName(CE)) + .Case("getParameter", &getParamDumper::analyzerEval) + .Case("getUntrackedParameter", &getParamDumper::analyzerEval) + .Default(nullptr); - return true; -} - - -class gpWalkAST : public clang::StmtVisitor { - const CheckerBase *Checker; - clang::ento::BugReporter &BR; - clang::AnalysisDeclContext *AC; - const NamedDecl *ND; - -public: - gpWalkAST(const CheckerBase *checker, clang::ento::BugReporter &br, clang::AnalysisDeclContext *ac, const NamedDecl * nd) - : Checker(checker), - BR(br), - AC(ac), - ND(nd) {} - - // Stmt visitor methods. - void VisitChildren(clang::Stmt *S); - void VisitStmt( clang::Stmt *S) { VisitChildren(S); } - void VisitCXXMemberCallExpr( clang::CXXMemberCallExpr *CE ) ; -}; - - -void gpWalkAST::VisitChildren( clang::Stmt *S) { - for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I!=E; ++I) - if (clang::Stmt *child = *I) { - Visit(child); - } -} + if (!Handler) + return false; + (this->*Handler)(CE, C); -void gpWalkAST::VisitCXXMemberCallExpr( clang::CXXMemberCallExpr *CE ) { + return true; + } + class gpWalkAST : public clang::StmtVisitor { + const CheckerBase *Checker; + clang::ento::BugReporter &BR; + clang::AnalysisDeclContext *AC; + const NamedDecl *ND; + + public: + gpWalkAST(const CheckerBase *checker, + clang::ento::BugReporter &br, + clang::AnalysisDeclContext *ac, + const NamedDecl *nd) + : Checker(checker), BR(br), AC(ac), ND(nd) {} + + // Stmt visitor methods. + void VisitChildren(clang::Stmt *S); + void VisitStmt(clang::Stmt *S) { VisitChildren(S); } + void VisitCXXMemberCallExpr(clang::CXXMemberCallExpr *CE); + }; + + void gpWalkAST::VisitChildren(clang::Stmt *S) { + for (clang::Stmt::child_iterator I = S->child_begin(), E = S->child_end(); I != E; ++I) + if (clang::Stmt *child = *I) { + Visit(child); + } + } - const FunctionDecl * FD = CE->getMethodDecl(); - if (!FD) return; + void gpWalkAST::VisitCXXMemberCallExpr(clang::CXXMemberCallExpr *CE) { + const FunctionDecl *FD = CE->getMethodDecl(); + if (!FD) + return; std::string mname = FD->getQualifiedNameAsString(); - const char *sfile=BR.getSourceManager().getPresumedLoc(CE->getExprLoc()).getFilename(); + const char *sfile = BR.getSourceManager().getPresumedLoc(CE->getExprLoc()).getFilename(); std::string sname(sfile); - if ( ! support::isInterestingLocation(sname) ) return; + if (!support::isInterestingLocation(sname)) + return; std::string mdname = ND->getQualifiedNameAsString(); - const Expr * IOA = CE->getImplicitObjectArgument(); + const Expr *IOA = CE->getImplicitObjectArgument(); std::string tname = "getparam-dumper.txt.unsorted"; std::string ps = "const class edm::ParameterSet "; std::string ups = "const class edm::UntrackedParameterSet "; std::string gp = "edm::ParameterSet::getParameter"; std::string gup = "edm::ParameterSet::getUntrackedParameter"; - if (mname.substr(0,gp.length()) == gp || mname.substr(0,gup.length()) == gup ) { - std::string buf; - llvm::raw_string_ostream os(buf); - const NamedDecl * nd = llvm::dyn_cast(AC->getDecl()); - if ( FunctionDecl::classof(ND) ) { - os << "function decl '" << nd->getQualifiedNameAsString() ; - os << "' this '"<getQualifiedNameAsString() ; - os << "' initializer for member decl '"<getNumArgs(); I != E; ++I) { - if (I) os <<", "; - os << CE->getType().getCanonicalType().getAsString()<<" "; - CE->getArg(I)->printPretty(os,nullptr,Policy); - } - os << "' with implicit object '"; - const Expr * E = IOA->IgnoreParenCasts(); - QualType QE = E->getType().getCanonicalType(); - os << QE.getAsString()<<" "; - switch( E->getStmtClass() ) { - case Stmt::MemberExprClass: - os << dyn_cast(E)->getMemberDecl()->getQualifiedNameAsString(); - break; - case Stmt::DeclRefExprClass: - os << dyn_cast(E)->getDecl()->getQualifiedNameAsString(); - break; - case Stmt::CXXOperatorCallExprClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - case Stmt::CXXBindTemporaryExprClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - case Stmt::CXXMemberCallExprClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - case Stmt::UnaryOperatorClass: - dyn_cast(E)->printPretty(os,nullptr,Policy); - break; - default: - E->printPretty(os,nullptr,Policy); - os << " unhandled expr class " <getStmtClassName(); - } - os<<"'\n"; - - support::writeLog(os.str(),tname); + if (mname.substr(0, gp.length()) == gp || mname.substr(0, gup.length()) == gup) { + std::string buf; + llvm::raw_string_ostream os(buf); + const NamedDecl *nd = llvm::dyn_cast(AC->getDecl()); + if (FunctionDecl::classof(ND)) { + os << "function decl '" << nd->getQualifiedNameAsString(); + os << "' this '" << mdname; + } else { + os << "constructor decl '" << nd->getQualifiedNameAsString(); + os << "' initializer for member decl '" << mdname; + } + clang::LangOptions LangOpts; + LangOpts.CPlusPlus = true; + clang::PrintingPolicy Policy(LangOpts); + os << "' with call args '"; + for (unsigned I = 0, E = CE->getNumArgs(); I != E; ++I) { + if (I) + os << ", "; + os << CE->getType().getCanonicalType().getAsString() << " "; + CE->getArg(I)->printPretty(os, nullptr, Policy); + } + os << "' with implicit object '"; + const Expr *E = IOA->IgnoreParenCasts(); + QualType QE = E->getType().getCanonicalType(); + os << QE.getAsString() << " "; + switch (E->getStmtClass()) { + case Stmt::MemberExprClass: + os << dyn_cast(E)->getMemberDecl()->getQualifiedNameAsString(); + break; + case Stmt::DeclRefExprClass: + os << dyn_cast(E)->getDecl()->getQualifiedNameAsString(); + break; + case Stmt::CXXOperatorCallExprClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + case Stmt::CXXBindTemporaryExprClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + case Stmt::CXXMemberCallExprClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + case Stmt::UnaryOperatorClass: + dyn_cast(E)->printPretty(os, nullptr, Policy); + break; + default: + E->printPretty(os, nullptr, Policy); + os << " unhandled expr class " << E->getStmtClassName(); + } + os << "'\n"; + + support::writeLog(os.str(), tname); + } + return; } - return ; -} + void getParamDumper::checkASTDecl(const clang::CXXRecordDecl *RD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const { + const clang::SourceManager &SM = BR.getSourceManager(); + const char *sfile = SM.getPresumedLoc(RD->getLocation()).getFilename(); + if (!support::isCmsLocalFile(sfile)) + return; -void getParamDumper::checkASTDecl(const clang::CXXRecordDecl *RD, clang::ento::AnalysisManager& mgr, clang::ento::BugReporter &BR) const { - - - const clang::SourceManager &SM = BR.getSourceManager(); - const char *sfile=SM.getPresumedLoc(RD->getLocation()).getFilename(); - if (!support::isCmsLocalFile(sfile)) return; - - std::string tname = "getparam-dumper.txt.unsorted"; - std::string ps = "const class edm::ParameterSet "; - std::string ups = "const class edm::UntrackedParameterSet "; + std::string tname = "getparam-dumper.txt.unsorted"; + std::string ps = "const class edm::ParameterSet "; + std::string ups = "const class edm::UntrackedParameterSet "; - for (clang::CXXRecordDecl::ctor_iterator - I = RD->ctor_begin(), E = RD->ctor_end(); I != E; ++I) { - clang::CXXConstructorDecl * CD = llvm::dyn_cast((*I)->getMostRecentDecl()); - for ( unsigned I=0, E=CD->getNumParams(); I != E; ++I) { - std::string pname = CD->getParamDecl(I)->getQualifiedNameAsString(); - std::string qname = CD->getParamDecl(I)->getType().getCanonicalType().getAsString(); - if (qname.substr(0,ps.length()) == ps || qname.substr(0,ups.length()) == ups) { - std::string buf; - llvm::raw_string_ostream os(buf); - os << "constructor decl '"<getQualifiedNameAsString()<< "' with parameter '"<< qname << " " << pname <<"'\n"; - support::writeLog(os.str(),tname); - for ( CXXConstructorDecl::init_iterator J = CD->init_begin(), E=CD->init_end(); J != E; ++J ) { - if (FieldDecl * fd = (*J)->getAnyMember()) { - std::string fname = fd->getQualifiedNameAsString(); - std::string fqname = fd->getType().getCanonicalType().getAsString(); - os << "constructor decl '"<getQualifiedNameAsString()<< "' initializer for member decl '" << fname << "' of type '" <getInit(); - if ( e ) { - gpWalkAST walker(this,BR, mgr.getAnalysisDeclContext(CD),fd); - walker.Visit(e); - } - } - } - support::writeLog(os.str(),tname); + for (clang::CXXRecordDecl::ctor_iterator I = RD->ctor_begin(), E = RD->ctor_end(); I != E; ++I) { + clang::CXXConstructorDecl *CD = llvm::dyn_cast((*I)->getMostRecentDecl()); + for (unsigned I = 0, E = CD->getNumParams(); I != E; ++I) { + std::string pname = CD->getParamDecl(I)->getQualifiedNameAsString(); + std::string qname = CD->getParamDecl(I)->getType().getCanonicalType().getAsString(); + if (qname.substr(0, ps.length()) == ps || qname.substr(0, ups.length()) == ups) { + std::string buf; + llvm::raw_string_ostream os(buf); + os << "constructor decl '" << CD->getQualifiedNameAsString() << "' with parameter '" << qname << " " << pname + << "'\n"; + support::writeLog(os.str(), tname); + for (CXXConstructorDecl::init_iterator J = CD->init_begin(), E = CD->init_end(); J != E; ++J) { + if (FieldDecl *fd = (*J)->getAnyMember()) { + std::string fname = fd->getQualifiedNameAsString(); + std::string fqname = fd->getType().getCanonicalType().getAsString(); + os << "constructor decl '" << CD->getQualifiedNameAsString() << "' initializer for member decl '" << fname + << "' of type '" << fqname << "'\n"; + Expr *e = (*J)->getInit(); + if (e) { + gpWalkAST walker(this, BR, mgr.getAnalysisDeclContext(CD), fd); + walker.Visit(e); } - } - } - - for (clang::CXXRecordDecl::method_iterator - I = RD->method_begin(), E = RD->method_end(); I != E; ++I) { - clang::CXXMethodDecl * MD = llvm::cast((*I)->getMostRecentDecl()); - for ( unsigned I=0, E=MD->getNumParams(); I != E; ++I) { - std::string pname = MD->getParamDecl(I)->getQualifiedNameAsString(); - std::string qname = MD->getParamDecl(I)->getType().getCanonicalType().getAsString(); - if (qname.substr(0,ps.length()) == ps || qname.substr(0,ups.length()) == ups) { - std::string buf; - llvm::raw_string_ostream os(buf); - clang::Stmt *Body = MD->getBody(); - if ( Body ) { - os << "function decl '"<getQualifiedNameAsString()<< "' with parameter '"<< qname << " " << pname <<"'\n"; - gpWalkAST walker(this,BR, mgr.getAnalysisDeclContext(MD),MD); - walker.Visit(Body); - } - support::writeLog(os.str(),tname); - } + } } - } + support::writeLog(os.str(), tname); + } + } + } - return; -} + for (clang::CXXRecordDecl::method_iterator I = RD->method_begin(), E = RD->method_end(); I != E; ++I) { + clang::CXXMethodDecl *MD = llvm::cast((*I)->getMostRecentDecl()); + for (unsigned I = 0, E = MD->getNumParams(); I != E; ++I) { + std::string pname = MD->getParamDecl(I)->getQualifiedNameAsString(); + std::string qname = MD->getParamDecl(I)->getType().getCanonicalType().getAsString(); + if (qname.substr(0, ps.length()) == ps || qname.substr(0, ups.length()) == ups) { + std::string buf; + llvm::raw_string_ostream os(buf); + clang::Stmt *Body = MD->getBody(); + if (Body) { + os << "function decl '" << MD->getQualifiedNameAsString() << "' with parameter '" << qname << " " << pname + << "'\n"; + gpWalkAST walker(this, BR, mgr.getAnalysisDeclContext(MD), MD); + walker.Visit(Body); + } + support::writeLog(os.str(), tname); + } + } + } -} + return; + } +} // namespace clangcms diff --git a/Utilities/StaticAnalyzers/src/getParamDumper.h b/Utilities/StaticAnalyzers/src/getParamDumper.h index 1e1228338c569..034534faca6fc 100644 --- a/Utilities/StaticAnalyzers/src/getParamDumper.h +++ b/Utilities/StaticAnalyzers/src/getParamDumper.h @@ -4,24 +4,21 @@ #include #include - namespace clangcms { -class getParamDumper : public clang::ento::Checker, - clang::ento::eval::Call > { - - void analyzerEval(const clang::CallExpr *CE, clang::ento::CheckerContext &C) const; - - typedef void (getParamDumper::*FnCheck)(const clang::CallExpr *, clang::ento::CheckerContext &C) const; - -public: + class getParamDumper + : public clang::ento::Checker, clang::ento::eval::Call> { + void analyzerEval(const clang::CallExpr *CE, clang::ento::CheckerContext &C) const; - bool evalCall(const clang::CallExpr *CE, clang::ento::CheckerContext &C) const; + typedef void (getParamDumper::*FnCheck)(const clang::CallExpr *, clang::ento::CheckerContext &C) const; - void checkASTDecl(const clang::CXXRecordDecl *CRD, clang::ento::AnalysisManager& mgr, - clang::ento::BugReporter &BR) const ; + public: + bool evalCall(const clang::CallExpr *CE, clang::ento::CheckerContext &C) const; -}; + void checkASTDecl(const clang::CXXRecordDecl *CRD, + clang::ento::AnalysisManager &mgr, + clang::ento::BugReporter &BR) const; + }; -} - #endif +} // namespace clangcms +#endif diff --git a/Utilities/StorageFactory/bin/anycp.cpp b/Utilities/StorageFactory/bin/anycp.cpp index 537f086b61235..95377a9caa4be 100644 --- a/Utilities/StorageFactory/bin/anycp.cpp +++ b/Utilities/StorageFactory/bin/anycp.cpp @@ -23,87 +23,78 @@ typedef std::lock_guard ScopedLock; //<<<<<< PUBLIC FUNCTION DEFINITIONS >>>>>> //<<<<<< MEMBER FUNCTION DEFINITIONS >>>>>> -struct IsDoneWriting -{ +struct IsDoneWriting { int &flag_; - IsDoneWriting(int &flag) : flag_(flag) { } + IsDoneWriting(int &flag) : flag_(flag) {} bool operator()() { return flag_ == 0; } }; -class OutputDropBox -{ +class OutputDropBox { public: - OutputDropBox() : outbuf(1000000), nout(0), ce(""), writing(0) {} + OutputDropBox() : outbuf(1000000), nout(0), ce(""), writing(0) {} - // called by main - bool set (std::vector &ibuf, IOSize n) - { + // called by main + bool set(std::vector &ibuf, IOSize n) { // wait that all threads finish write to swap std::unique_lock gl(lock); done.wait(gl, IsDoneWriting(writing)); bool ret = true; // if error in previous write return.... - if (ce.empty()) - { + if (ce.empty()) { outbuf.swap(ibuf); nout = n; - } - else + } else ret = false; - undo(); // clean al bits, set writing to its size... + undo(); // clean al bits, set writing to its size... // notify threads buffer is ready doit.notify_all(); return ret; - } + } // called by thread - bool write (Storage *os, int id) - { + bool write(Storage *os, int id) { std::unique_lock gl(lock); // wait if box empty or this thread already consumed... if (m_done[id]) doit.wait(gl); - bool ret=true; - if (nout==0) + bool ret = true; + if (nout == 0) // notify thread to exit.... - ret=false; - else - { - try { os->write(&outbuf[0], nout); } - catch (cms::Exception &lce) - { - ce = lce.explainSelf(); - } + ret = false; + else { + try { + os->write(&outbuf[0], nout); + } catch (cms::Exception &lce) { + ce = lce.explainSelf(); + } } { // declare it finishes ScopedLock wl(wlock); - m_done[id]=true; + m_done[id] = true; writing--; - } - done.notify_all(); + } + done.notify_all(); return ret; } // add a writer (called by thread itself), return thread index.... - int addWriter (void) - { + int addWriter(void) { ScopedLock wl(wlock); m_done.push_back(true); - return m_done.size()-1; + return m_done.size() - 1; } // clear bits (declare box ready...) - void undo() - { + void undo() { ScopedLock wl(wlock); writing = m_done.size(); - std::fill(m_done.begin(),m_done.end(),false); + std::fill(m_done.begin(), m_done.end(), false); } std::vector m_done; @@ -119,14 +110,12 @@ class OutputDropBox mutable std::condition_variable done; }; -class InputDropBox -{ +class InputDropBox { public: - InputDropBox() : end(false), inbuf(1000000), nin(0), ce("") {} + InputDropBox() : end(false), inbuf(1000000), nin(0), ce("") {} - // called by main - IOSize get(std::vector &ibuf) - { + // called by main + IOSize get(std::vector &ibuf) { std::unique_lock gl(lock); if (end) // if thread is over return... @@ -138,8 +127,7 @@ class InputDropBox IOSize ret = 0; // if error in previous write return.... - if (ce.empty()) - { + if (ce.empty()) { inbuf.swap(ibuf); ret = nin; } @@ -147,42 +135,34 @@ class InputDropBox // notify threads buffer is ready doit.notify_all(); return ret; - } + } // called by thread - bool read (Storage *os) - { + bool read(Storage *os) { std::unique_lock gl(lock); if (nin != 0) // wait if box empty or this thread already consumed... doit.wait(gl); - bool ret=true; + bool ret = true; - if (inbuf.empty()) - { + if (inbuf.empty()) { // inbuf empty notify thread to exit.... - end=true; - ret=false; - } - else - { - try - { - nin = os->read(&inbuf[0],inbuf.size()); - if (nin==0) - { - end=true; - ret=false; // stop thread - } + end = true; + ret = false; + } else { + try { + nin = os->read(&inbuf[0], inbuf.size()); + if (nin == 0) { + end = true; + ret = false; // stop thread + } + } catch (cms::Exception &e) { + ce = e.explainSelf(); } - catch(cms::Exception &e) - { - ce = e.explainSelf(); - } } - done.notify_all(); + done.notify_all(); return ret; } @@ -196,11 +176,10 @@ class InputDropBox mutable std::condition_variable done; }; -static InputDropBox inbox; +static InputDropBox inbox; static OutputDropBox dropbox; -static void writeThread (Storage* os) -{ +static void writeThread(Storage *os) { int myid = dropbox.addWriter(); std::cout << "start writing thread " << myid << std::endl; @@ -209,85 +188,70 @@ static void writeThread (Storage* os) std::cout << "end writing thread " << myid << std::endl; } -static void readThread (Storage* os) -{ +static void readThread(Storage *os) { std::cout << "start reading thread" << std::endl; while (inbox.read(os)) ; std::cout << "end reading thread" << std::endl; } -int main (int argc, char **argv) try -{ +int main(int argc, char **argv) try { edmplugin::PluginManager::configure(edmplugin::standard::config()); - if (argc < 3) - { + if (argc < 3) { std::cerr << "usage: " << argv[0] << " INFILE OUTFILE...\n"; return EXIT_FAILURE; } - std::shared_ptr is; - std::vector > os(argc-2); + std::shared_ptr is; + std::vector > os(argc - 2); std::vector threads; - bool readThreadActive = true; - bool writeThreadActive = true; - IOOffset size = -1; + bool readThreadActive = true; + bool writeThreadActive = true; + IOOffset size = -1; - StorageFactory::getToModify ()->enableAccounting(true); - bool exists = StorageFactory::getToModify ()->check(argv [1], &size); + StorageFactory::getToModify()->enableAccounting(true); + bool exists = StorageFactory::getToModify()->check(argv[1], &size); std::cerr << "input file exists = " << exists << ", size = " << size << "\n"; - if (! exists) return EXIT_SUCCESS; - - try - { - is = StorageFactory::getToModify ()->open (argv [1]); - if (readThreadActive) - threads.emplace_back(&readThread,is.get()); - } - catch (cms::Exception &e) - { - std::cerr << "error in opening input file " << argv[1] - << ":\n" << e.explainSelf() << std::endl; + if (!exists) + return EXIT_SUCCESS; + + try { + is = StorageFactory::getToModify()->open(argv[1]); + if (readThreadActive) + threads.emplace_back(&readThread, is.get()); + } catch (cms::Exception &e) { + std::cerr << "error in opening input file " << argv[1] << ":\n" << e.explainSelf() << std::endl; return EXIT_FAILURE; } // open output files and create threads, one thread per output - for (int i=0; i < argc-2; i++) - try - { - os[i]= StorageFactory::getToModify ()->open (argv[i+2], - IOFlags::OpenWrite - | IOFlags::OpenCreate - | IOFlags::OpenTruncate); - if (writeThreadActive) - threads.emplace_back(&writeThread,os[i].get()); - } - catch (cms::Exception &e) - { - std::cerr << "error in opening output file " << argv[i+2] - << ":\n" << e.explainSelf() << std::endl; - return EXIT_FAILURE; - } + for (int i = 0; i < argc - 2; i++) + try { + os[i] = StorageFactory::getToModify()->open(argv[i + 2], + IOFlags::OpenWrite | IOFlags::OpenCreate | IOFlags::OpenTruncate); + if (writeThreadActive) + threads.emplace_back(&writeThread, os[i].get()); + } catch (cms::Exception &e) { + std::cerr << "error in opening output file " << argv[i + 2] << ":\n" << e.explainSelf() << std::endl; + return EXIT_FAILURE; + } - std::vector inbuf (1048576); + std::vector inbuf(1048576); std::vector outbuf(1048576); - IOSize n; + IOSize n; - while ((n = readThreadActive ? inbox.get(inbuf) : is->read(&inbuf[0],inbuf.size()))) - { + while ((n = readThreadActive ? inbox.get(inbuf) : is->read(&inbuf[0], inbuf.size()))) { //free reading thread inbuf.swap(outbuf); // wait threads have finished to write // drop buffer in thread - if (writeThreadActive) - { - if (! dropbox.set(outbuf,n)) - break; - } - else + if (writeThreadActive) { + if (!dropbox.set(outbuf, n)) + break; + } else for (size_t i = 0; i < os.size(); i++) - os[i]->write(&outbuf[0],n); + os[i]->write(&outbuf[0], n); } std::cout << "main end reading" << std::endl; @@ -301,17 +265,17 @@ int main (int argc, char **argv) try dropbox.set(outbuf, 0); if (writeThreadActive || readThreadActive) { - for(auto& t: threads) { + for (auto &t : threads) { t.join(); } } std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const &e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const &e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/interface/File.h b/Utilities/StorageFactory/interface/File.h index 3d18cdbcbca5c..aaa8e7a97756a 100644 --- a/Utilities/StorageFactory/interface/File.h +++ b/Utilities/StorageFactory/interface/File.h @@ -1,77 +1,67 @@ #ifndef STORAGE_FACTORY_FILE_H -# define STORAGE_FACTORY_FILE_H +#define STORAGE_FACTORY_FILE_H -# include "Utilities/StorageFactory/interface/IOTypes.h" -# include "Utilities/StorageFactory/interface/IOFlags.h" -# include "Utilities/StorageFactory/interface/Storage.h" -# include "Utilities/StorageFactory/interface/IOChannel.h" -# include +#include "Utilities/StorageFactory/interface/IOTypes.h" +#include "Utilities/StorageFactory/interface/IOFlags.h" +#include "Utilities/StorageFactory/interface/Storage.h" +#include "Utilities/StorageFactory/interface/IOChannel.h" +#include /** Basic file-related functions. Nicked from SEAL. */ -class File : public IOChannel, public Storage -{ +class File : public IOChannel, public Storage { public: - File (void); - File (IOFD fd, bool autoclose = true); - File (const char *name, int flags = IOFlags::OpenRead, int perms = 0666); - File (const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); - ~File (void) override; + File(void); + File(IOFD fd, bool autoclose = true); + File(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + File(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + ~File(void) override; // implicit copy constructor // implicit assignment operator - virtual void create (const char *name, - bool exclusive = false, - int perms = 0666); - virtual void create (const std::string &name, - bool exclusive = false, - int perms = 0666); - virtual void open (const char *name, - int flags = IOFlags::OpenRead, - int perms = 0666); - virtual void open (const std::string &name, - int flags = IOFlags::OpenRead, - int perms = 0666); - virtual void attach (IOFD fd); + virtual void create(const char *name, bool exclusive = false, int perms = 0666); + virtual void create(const std::string &name, bool exclusive = false, int perms = 0666); + virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + virtual void attach(IOFD fd); + using Storage::position; using Storage::read; using Storage::readv; using Storage::write; using Storage::writev; - using Storage::position; - bool prefetch (const IOPosBuffer *what, IOSize n) override; - IOSize read (void *into, IOSize n) override; - IOSize read (void *into, IOSize n, IOOffset pos) override; - IOSize readv (IOBuffer *into, IOSize length) override; + bool prefetch(const IOPosBuffer *what, IOSize n) override; + IOSize read(void *into, IOSize n) override; + IOSize read(void *into, IOSize n, IOOffset pos) override; + IOSize readv(IOBuffer *into, IOSize length) override; - IOSize write (const void *from, IOSize n) override; - IOSize write (const void *from, IOSize n, IOOffset pos) override; - IOSize writev (const IOBuffer *from, IOSize length) override; + IOSize write(const void *from, IOSize n) override; + IOSize write(const void *from, IOSize n, IOOffset pos) override; + IOSize writev(const IOBuffer *from, IOSize length) override; - IOOffset size (void) const override; - IOOffset position (IOOffset offset, Relative whence = SET) override; + IOOffset size(void) const override; + IOOffset position(IOOffset offset, Relative whence = SET) override; - void resize (IOOffset size) override; + void resize(IOOffset size) override; - void flush (void) override; - void close (void) override; - virtual void abort (void); + void flush(void) override; + void close(void) override; + virtual void abort(void); - virtual void setAutoClose (bool closeit); + virtual void setAutoClose(bool closeit); private: - enum { InternalAutoClose = 4096 }; //< Close on delete + enum { InternalAutoClose = 4096 }; //< Close on delete - File (IOFD fd, unsigned flags); + File(IOFD fd, unsigned flags); - File * duplicate (bool copy) const; - File * duplicate (File *child) const; - static IOFD sysduplicate (IOFD fd); - static void sysopen (const char *name, int flags, int perms, - IOFD &newfd, unsigned &newflags); - static bool sysclose (IOFD fd, int *error = nullptr); + File *duplicate(bool copy) const; + File *duplicate(File *child) const; + static IOFD sysduplicate(IOFD fd); + static void sysopen(const char *name, int flags, int perms, IOFD &newfd, unsigned &newflags); + static bool sysclose(IOFD fd, int *error = nullptr); - unsigned m_flags; + unsigned m_flags; }; -#endif // STORAGE_FACTORY_FILE_H +#endif // STORAGE_FACTORY_FILE_H diff --git a/Utilities/StorageFactory/interface/IOBuffer.h b/Utilities/StorageFactory/interface/IOBuffer.h index 6faadfe55d8fe..e2b6faacbd0f4 100644 --- a/Utilities/StorageFactory/interface/IOBuffer.h +++ b/Utilities/StorageFactory/interface/IOBuffer.h @@ -1,53 +1,36 @@ #ifndef STORAGE_FACTORY_IO_BUFFER_H -# define STORAGE_FACTORY_IO_BUFFER_H +#define STORAGE_FACTORY_IO_BUFFER_H -# include "Utilities/StorageFactory/interface/IOTypes.h" +#include "Utilities/StorageFactory/interface/IOTypes.h" /** Buffer for I/O operations. */ -class IOBuffer -{ +class IOBuffer { public: - IOBuffer (void); - IOBuffer (void *data, IOSize length); - IOBuffer (const void *data, IOSize length); + IOBuffer(void); + IOBuffer(void *data, IOSize length); + IOBuffer(const void *data, IOSize length); - void * data (void) const; - IOSize size (void) const; + void *data(void) const; + IOSize size(void) const; private: - void *m_data; //< Data - IOSize m_length; //< Length of data in bytes. + void *m_data; //< Data + IOSize m_length; //< Length of data in bytes. }; /** Construct a null I/O buffer. */ -inline -IOBuffer::IOBuffer (void) - : m_data (nullptr), - m_length (0) -{} +inline IOBuffer::IOBuffer(void) : m_data(nullptr), m_length(0) {} /** Construct a I/O buffer for reading. */ -inline -IOBuffer::IOBuffer (void *data, IOSize length) - : m_data (data), - m_length (length) -{} +inline IOBuffer::IOBuffer(void *data, IOSize length) : m_data(data), m_length(length) {} /** Construct a I/O buffer for writing. */ -inline -IOBuffer::IOBuffer (const void *data, IOSize length) - : m_data (const_cast (data)), - m_length (length) -{} +inline IOBuffer::IOBuffer(const void *data, IOSize length) : m_data(const_cast(data)), m_length(length) {} /** Return a pointer to the beginning of the buffer's data area. */ -inline void * -IOBuffer::data (void) const -{ return m_data; } +inline void *IOBuffer::data(void) const { return m_data; } /** Return the buffer's size. */ -inline IOSize -IOBuffer::size (void) const -{ return m_length; } +inline IOSize IOBuffer::size(void) const { return m_length; } -#endif // STORAGE_FACTORY_IO_BUFFER_H +#endif // STORAGE_FACTORY_IO_BUFFER_H diff --git a/Utilities/StorageFactory/interface/IOChannel.h b/Utilities/StorageFactory/interface/IOChannel.h index 7c2bfc0d6df31..2d7c573647c19 100644 --- a/Utilities/StorageFactory/interface/IOChannel.h +++ b/Utilities/StorageFactory/interface/IOChannel.h @@ -1,42 +1,41 @@ #ifndef STORAGE_FACTORY_IO_CHANNEL_H -# define STORAGE_FACTORY_IO_CHANNEL_H +#define STORAGE_FACTORY_IO_CHANNEL_H -# include "Utilities/StorageFactory/interface/IOInput.h" -# include "Utilities/StorageFactory/interface/IOOutput.h" +#include "Utilities/StorageFactory/interface/IOInput.h" +#include "Utilities/StorageFactory/interface/IOOutput.h" /** Base class for stream-oriented I/O sources and targets, based on the operating system file descriptor. */ -class IOChannel : public virtual IOInput, public virtual IOOutput -{ +class IOChannel : public virtual IOInput, public virtual IOOutput { public: - IOChannel (IOFD fd = EDM_IOFD_INVALID); - ~IOChannel (void) override; + IOChannel(IOFD fd = EDM_IOFD_INVALID); + ~IOChannel(void) override; // implicit copy constructor // implicit assignment operator using IOInput::read; using IOOutput::write; - - IOSize read (void *into, IOSize n) override; - IOSize readv (IOBuffer *into, IOSize buffers) override; - IOSize write (const void *from, IOSize n) override; - IOSize writev (const IOBuffer *from, IOSize buffers) override; + IOSize read(void *into, IOSize n) override; + IOSize readv(IOBuffer *into, IOSize buffers) override; - virtual IOFD fd (void) const; - virtual void fd (IOFD value); // FIXME: dangerous? + IOSize write(const void *from, IOSize n) override; + IOSize writev(const IOBuffer *from, IOSize buffers) override; - virtual void close (void); + virtual IOFD fd(void) const; + virtual void fd(IOFD value); // FIXME: dangerous? - virtual void setBlocking (bool value); - virtual bool isBlocking (void) const; + virtual void close(void); + + virtual void setBlocking(bool value); + virtual bool isBlocking(void) const; protected: // System implementation - bool sysclose (IOFD fd, int *error = nullptr); + bool sysclose(IOFD fd, int *error = nullptr); private: - IOFD m_fd; /*< System file descriptor. */ + IOFD m_fd; /*< System file descriptor. */ }; -#endif // STORAGE_FACTORY_IO_CHANNEL_H +#endif // STORAGE_FACTORY_IO_CHANNEL_H diff --git a/Utilities/StorageFactory/interface/IOFlags.h b/Utilities/StorageFactory/interface/IOFlags.h index f6ac2cb5f0b25..2b445a6d7794d 100644 --- a/Utilities/StorageFactory/interface/IOFlags.h +++ b/Utilities/StorageFactory/interface/IOFlags.h @@ -1,14 +1,12 @@ #ifndef STORAGE_FACTORY_IO_FLAGS_H -# define STORAGE_FACTORY_IO_FLAGS_H +#define STORAGE_FACTORY_IO_FLAGS_H -namespace IOFlags -{ +namespace IOFlags { /// File open mode flags. - enum OpenMode - { - OpenRead = 1, /*< Open for read access. */ - OpenWrite = 2, /*< Open for write access. */ - OpenNonBlock = 4, /*< Open in non-blocking mode. + enum OpenMode { + OpenRead = 1, /*< Open for read access. */ + OpenWrite = 2, /*< Open for write access. */ + OpenNonBlock = 4, /*< Open in non-blocking mode. Neither the #open() nor any subsequent operation on the returned descriptor cause the @@ -17,26 +15,26 @@ namespace IOFlags allows the pipe to be opened without waiting for the other end to be connected. */ - OpenAppend = 8, /*< Writes to the file should always + OpenAppend = 8, /*< Writes to the file should always append to the end of the file. */ - OpenUnbuffered = 16, /*< No write caching should be done on + OpenUnbuffered = 16, /*< No write caching should be done on this file; may result in blocking the process until the data has actually been written. */ - OpenCreate = 32, /*< Create the file if it does not exist. */ - OpenExclusive = 64, /*< With #ModeCreate specifies that + OpenCreate = 32, /*< Create the file if it does not exist. */ + OpenExclusive = 64, /*< With #ModeCreate specifies that the creation should fali if the file exists. */ - OpenTruncate = 128, /*< If the file exists, truncate it to + OpenTruncate = 128, /*< If the file exists, truncate it to zero size. */ - OpenNotCTTY = 256, /*< If the specified file is a + OpenNotCTTY = 256, /*< If the specified file is a terminal device, do not make it the controlling terminal for the process even if the process does not have one yet. */ - OpenWrap = 512 /*< Wrap the file if at all possible + OpenWrap = 512 /*< Wrap the file if at all possible with a local file. */ }; -} // namespace IOFlags +} // namespace IOFlags -#endif // STORAGE_FACTORY_IO_FLAGS_H +#endif // STORAGE_FACTORY_IO_FLAGS_H diff --git a/Utilities/StorageFactory/interface/IOInput.h b/Utilities/StorageFactory/interface/IOInput.h index ddb118d00cde8..8bdff37b7cd04 100644 --- a/Utilities/StorageFactory/interface/IOInput.h +++ b/Utilities/StorageFactory/interface/IOInput.h @@ -1,25 +1,24 @@ #ifndef STORAGE_FACTORY_IO_INPUT_H -# define STORAGE_FACTORY_IO_INPUT_H +#define STORAGE_FACTORY_IO_INPUT_H -# include "Utilities/StorageFactory/interface/IOBuffer.h" +#include "Utilities/StorageFactory/interface/IOBuffer.h" /** Abstract base class for stream-oriented input sources. */ -class IOInput -{ +class IOInput { public: - virtual ~IOInput (void); + virtual ~IOInput(void); // implicit constructor // implicit copy constructor // implicit assignment operator - int read (void); - IOSize read (IOBuffer into); - virtual IOSize read (void *into, IOSize n) = 0; - virtual IOSize readv (IOBuffer *into, IOSize buffers); + int read(void); + IOSize read(IOBuffer into); + virtual IOSize read(void *into, IOSize n) = 0; + virtual IOSize readv(IOBuffer *into, IOSize buffers); - IOSize xread (IOBuffer into); - IOSize xread (void *into, IOSize n); - IOSize xreadv (IOBuffer *into, IOSize buffers); + IOSize xread(IOBuffer into); + IOSize xread(void *into, IOSize n); + IOSize xreadv(IOBuffer *into, IOSize buffers); }; -#endif // STORAGE_FACTORY_IO_INPUT_H +#endif // STORAGE_FACTORY_IO_INPUT_H diff --git a/Utilities/StorageFactory/interface/IOOutput.h b/Utilities/StorageFactory/interface/IOOutput.h index ddd7e65ba4cb6..dc4a4bec2f3b4 100644 --- a/Utilities/StorageFactory/interface/IOOutput.h +++ b/Utilities/StorageFactory/interface/IOOutput.h @@ -1,25 +1,24 @@ #ifndef STORAGE_FACTORY_IO_OUTPUT_H -# define STORAGE_FACTORY_IO_OUTPUT_H +#define STORAGE_FACTORY_IO_OUTPUT_H -# include "Utilities/StorageFactory/interface/IOBuffer.h" +#include "Utilities/StorageFactory/interface/IOBuffer.h" /** Abstract base class for stream-oriented output targets. */ -class IOOutput -{ +class IOOutput { public: - virtual ~IOOutput (void); + virtual ~IOOutput(void); // implicit constructor // implicit copy constructor // implicit assignment operator - IOSize write (unsigned char byte); - IOSize write (IOBuffer from); - virtual IOSize write (const void *from, IOSize n) = 0; - virtual IOSize writev (const IOBuffer *from, IOSize buffers); + IOSize write(unsigned char byte); + IOSize write(IOBuffer from); + virtual IOSize write(const void *from, IOSize n) = 0; + virtual IOSize writev(const IOBuffer *from, IOSize buffers); - IOSize xwrite (const void *from, IOSize n); - IOSize xwrite (IOBuffer from); - IOSize xwritev (const IOBuffer *from, IOSize buffers); + IOSize xwrite(const void *from, IOSize n); + IOSize xwrite(IOBuffer from); + IOSize xwritev(const IOBuffer *from, IOSize buffers); }; -#endif // STORAGE_FACTORY_IO_OUTPUT_H +#endif // STORAGE_FACTORY_IO_OUTPUT_H diff --git a/Utilities/StorageFactory/interface/IOPosBuffer.h b/Utilities/StorageFactory/interface/IOPosBuffer.h index 6af95fe72c5c4..353fac697adbd 100644 --- a/Utilities/StorageFactory/interface/IOPosBuffer.h +++ b/Utilities/StorageFactory/interface/IOPosBuffer.h @@ -1,82 +1,56 @@ #ifndef STORAGE_FACTORY_IO_POS_BUFFER_H -# define STORAGE_FACTORY_IO_POS_BUFFER_H +#define STORAGE_FACTORY_IO_POS_BUFFER_H -# include "Utilities/StorageFactory/interface/IOTypes.h" +#include "Utilities/StorageFactory/interface/IOTypes.h" /** Buffer for I/O operations. */ -struct IOPosBuffer -{ +struct IOPosBuffer { public: - IOPosBuffer (void); - IOPosBuffer (IOOffset offset, void *data, IOSize length); - IOPosBuffer (IOOffset offset, const void *data, IOSize length); + IOPosBuffer(void); + IOPosBuffer(IOOffset offset, void *data, IOSize length); + IOPosBuffer(IOOffset offset, const void *data, IOSize length); - IOOffset offset (void) const; - void * data (void) const; - IOSize size (void) const; + IOOffset offset(void) const; + void *data(void) const; + IOSize size(void) const; - void set_offset (IOOffset new_offset); - void set_data (void * new_buffer); - void set_size (IOSize new_size); + void set_offset(IOOffset new_offset); + void set_data(void *new_buffer); + void set_size(IOSize new_size); private: - IOOffset m_offset; //< File offset. - void *m_data; //< Data - IOSize m_length; //< Length of data in bytes. + IOOffset m_offset; //< File offset. + void *m_data; //< Data + IOSize m_length; //< Length of data in bytes. }; /** Construct a null I/O buffer. */ -inline -IOPosBuffer::IOPosBuffer (void) - : m_offset (0), - m_data (nullptr), - m_length (0) -{} +inline IOPosBuffer::IOPosBuffer(void) : m_offset(0), m_data(nullptr), m_length(0) {} /** Construct a I/O buffer for reading. */ -inline -IOPosBuffer::IOPosBuffer (IOOffset offset, void *data, IOSize length) - : m_offset (offset), - m_data (data), - m_length (length) -{} +inline IOPosBuffer::IOPosBuffer(IOOffset offset, void *data, IOSize length) + : m_offset(offset), m_data(data), m_length(length) {} /** Construct a I/O buffer for writing. */ -inline -IOPosBuffer::IOPosBuffer (IOOffset offset, const void *data, IOSize length) - : m_offset (offset), - m_data (const_cast (data)), - m_length (length) -{} +inline IOPosBuffer::IOPosBuffer(IOOffset offset, const void *data, IOSize length) + : m_offset(offset), m_data(const_cast(data)), m_length(length) {} /** Return the file offset where I/O is expected to occur. */ -inline IOOffset -IOPosBuffer::offset (void) const -{ return m_offset; } +inline IOOffset IOPosBuffer::offset(void) const { return m_offset; } /** Return a pointer to the beginning of the buffer's data area. */ -inline void * -IOPosBuffer::data (void) const -{ return m_data; } +inline void *IOPosBuffer::data(void) const { return m_data; } /** Return the buffer's size. */ -inline IOSize -IOPosBuffer::size (void) const -{ return m_length; } +inline IOSize IOPosBuffer::size(void) const { return m_length; } /** Update the file offset */ -inline void -IOPosBuffer::set_offset(IOOffset new_offset) -{ m_offset = new_offset; } +inline void IOPosBuffer::set_offset(IOOffset new_offset) { m_offset = new_offset; } /** Update the buffer's data area */ -inline void -IOPosBuffer::set_data(void * new_data) -{ m_data = new_data; } +inline void IOPosBuffer::set_data(void *new_data) { m_data = new_data; } /** Update the buffer's size */ -inline void -IOPosBuffer::set_size(IOSize new_length) -{ m_length = new_length; } +inline void IOPosBuffer::set_size(IOSize new_length) { m_length = new_length; } -#endif // STORAGE_FACTORY_IO_POS_BUFFER_H +#endif // STORAGE_FACTORY_IO_POS_BUFFER_H diff --git a/Utilities/StorageFactory/interface/IOTypes.h b/Utilities/StorageFactory/interface/IOTypes.h index f4befbf97d549..aacdf130a6232 100644 --- a/Utilities/StorageFactory/interface/IOTypes.h +++ b/Utilities/StorageFactory/interface/IOTypes.h @@ -1,8 +1,8 @@ #ifndef STORAGE_FACTORY_IO_TYPES_H -# define STORAGE_FACTORY_IO_TYPES_H +#define STORAGE_FACTORY_IO_TYPES_H -# include -# include +#include +#include /** Invalid channel descriptor constant. */ #define EDM_IOFD_INVALID -1 @@ -23,24 +23,21 @@ typedef int IOFD; /** I/O operation mask. */ enum IOMask { - IORead = 0x01, //< Read - IOWrite = 0x02, //< Write - IOUrgent = 0x04, //< Exceptional or urgent condition - IOAccept = 0x08, //< Socket accept - IOConnect = 0x10 //< Socket connect + IORead = 0x01, //< Read + IOWrite = 0x02, //< Write + IOUrgent = 0x04, //< Exceptional or urgent condition + IOAccept = 0x08, //< Socket accept + IOConnect = 0x10 //< Socket connect }; /** Safely convert IOOffset @a n into a #IOSize quantity. If @a n is larger than what #IOSize can hold, it is truncated to maximum value that can be held in #IOSize. */ -inline IOSize -IOSized (IOOffset n) -{ +inline IOSize IOSized(IOOffset n) { // If IOSize and IOOffset are the same size, largest positive value // is half of maximum IOSize. Otherwise all bits of IOSize work. - IOOffset largest = (sizeof (IOOffset) == sizeof (IOSize) - ? ~IOSize(0)/2 : ~IOSize(0)); - return IOSize (n > largest ? largest : n); + IOOffset largest = (sizeof(IOOffset) == sizeof(IOSize) ? ~IOSize(0) / 2 : ~IOSize(0)); + return IOSize(n > largest ? largest : n); } -#endif // STORAGE_FACTORY_IO_TYPES_H +#endif // STORAGE_FACTORY_IO_TYPES_H diff --git a/Utilities/StorageFactory/interface/LocalCacheFile.h b/Utilities/StorageFactory/interface/LocalCacheFile.h index 9ac42c8670b55..a7986c221b9a9 100644 --- a/Utilities/StorageFactory/interface/LocalCacheFile.h +++ b/Utilities/StorageFactory/interface/LocalCacheFile.h @@ -1,48 +1,47 @@ #ifndef STORAGE_FACTORY_LOCAL_CACHE_FILE_H -# define STORAGE_FACTORY_LOCAL_CACHE_FILE_H +#define STORAGE_FACTORY_LOCAL_CACHE_FILE_H -# include "Utilities/StorageFactory/interface/Storage.h" -# include "Utilities/StorageFactory/interface/File.h" +#include "Utilities/StorageFactory/interface/Storage.h" +#include "Utilities/StorageFactory/interface/File.h" #include "FWCore/Utilities/interface/propagate_const.h" -# include -# include -# include +#include +#include +#include /** Proxy class to copy a file locally in large chunks. */ -class LocalCacheFile : public Storage -{ +class LocalCacheFile : public Storage { public: - LocalCacheFile (std::unique_ptr base, const std::string &tmpdir = ""); - ~LocalCacheFile (void) override; + LocalCacheFile(std::unique_ptr base, const std::string &tmpdir = ""); + ~LocalCacheFile(void) override; using Storage::read; using Storage::write; - bool prefetch (const IOPosBuffer *what, IOSize n) override; - IOSize read (void *into, IOSize n) override; - IOSize read (void *into, IOSize n, IOOffset pos) override; - IOSize readv (IOBuffer *into, IOSize n) override; - IOSize readv (IOPosBuffer *into, IOSize n) override; - IOSize write (const void *from, IOSize n) override; - IOSize write (const void *from, IOSize n, IOOffset pos) override; - IOSize writev (const IOBuffer *from, IOSize n) override; - IOSize writev (const IOPosBuffer *from, IOSize n) override; - - IOOffset position (IOOffset offset, Relative whence = SET) override; - void resize (IOOffset size) override; - void flush (void) override; - void close (void) override; + bool prefetch(const IOPosBuffer *what, IOSize n) override; + IOSize read(void *into, IOSize n) override; + IOSize read(void *into, IOSize n, IOOffset pos) override; + IOSize readv(IOBuffer *into, IOSize n) override; + IOSize readv(IOPosBuffer *into, IOSize n) override; + IOSize write(const void *from, IOSize n) override; + IOSize write(const void *from, IOSize n, IOOffset pos) override; + IOSize writev(const IOBuffer *from, IOSize n) override; + IOSize writev(const IOPosBuffer *from, IOSize n) override; + + IOOffset position(IOOffset offset, Relative whence = SET) override; + void resize(IOOffset size) override; + void flush(void) override; + void close(void) override; private: - void cache (IOOffset start, IOOffset end); + void cache(IOOffset start, IOOffset end); - IOOffset image_; - std::vector present_; - edm::propagate_const> file_; + IOOffset image_; + std::vector present_; + edm::propagate_const> file_; edm::propagate_const> storage_; - bool closedFile_; - unsigned int cacheCount_; - unsigned int cacheTotal_; + bool closedFile_; + unsigned int cacheCount_; + unsigned int cacheTotal_; }; -#endif // STORAGE_FACTORY_LOCAL_CACHE_FILE_H +#endif // STORAGE_FACTORY_LOCAL_CACHE_FILE_H diff --git a/Utilities/StorageFactory/interface/LocalFileSystem.h b/Utilities/StorageFactory/interface/LocalFileSystem.h index cdb971d81a196..94544783e3586 100644 --- a/Utilities/StorageFactory/interface/LocalFileSystem.h +++ b/Utilities/StorageFactory/interface/LocalFileSystem.h @@ -1,29 +1,29 @@ #ifndef STORAGE_FACTORY_LOCAL_FILE_SYSTEM_H -# define STORAGE_FACTORY_LOCAL_FILE_SYSTEM_H -# include -# include +#define STORAGE_FACTORY_LOCAL_FILE_SYSTEM_H +#include +#include #include struct stat; struct statfs; struct mntent; -class LocalFileSystem -{ +class LocalFileSystem { struct FSInfo; + public: LocalFileSystem(void); ~LocalFileSystem(void); - bool isLocalPath(const std::string &path) const; - std::pair findCachePath(const std::vector &paths, double minFreeSpace) const; + bool isLocalPath(const std::string &path) const; + std::pair findCachePath(const std::vector &paths, double minFreeSpace) const; private: - int readFSTypes(void); - FSInfo * initFSInfo(void *p); - int initFSList(void); - int statFSInfo(FSInfo *i) const; - FSInfo * findMount(const char *path, struct statfs *sfs, struct stat *s, std::vector &) const; + int readFSTypes(void); + FSInfo *initFSInfo(void *p); + int initFSList(void); + int statFSInfo(FSInfo *i) const; + FSInfo *findMount(const char *path, struct statfs *sfs, struct stat *s, std::vector &) const; std::vector fs_; std::vector fstypes_; @@ -33,4 +33,4 @@ class LocalFileSystem void operator=(LocalFileSystem &) = delete; }; -#endif // STORAGE_FACTORY_LOCAL_FILE_SYSTEM_H +#endif // STORAGE_FACTORY_LOCAL_FILE_SYSTEM_H diff --git a/Utilities/StorageFactory/interface/RemoteFile.h b/Utilities/StorageFactory/interface/RemoteFile.h index 0b4e13938e623..b5796eadb75cc 100644 --- a/Utilities/StorageFactory/interface/RemoteFile.h +++ b/Utilities/StorageFactory/interface/RemoteFile.h @@ -1,27 +1,25 @@ #ifndef STORAGE_FACTORY_REMOTE_FILE_H -# define STORAGE_FACTORY_REMOTE_FILE_H +#define STORAGE_FACTORY_REMOTE_FILE_H -# include "Utilities/StorageFactory/interface/File.h" -# include +#include "Utilities/StorageFactory/interface/File.h" +#include #include -class RemoteFile : protected File -{ +class RemoteFile : protected File { public: - ~RemoteFile (void) override { remove (); } + ~RemoteFile(void) override { remove(); } - static int local (const std::string &tmpdir, std::string &temp); - static std::unique_ptr get (int localfd, const std::string &name, - char **cmd, int mode); + static int local(const std::string &tmpdir, std::string &temp); + static std::unique_ptr get(int localfd, const std::string &name, char **cmd, int mode); protected: - void close (void) override; - void abort (void) override; + void close(void) override; + void abort(void) override; private: - RemoteFile (IOFD fd, const std::string &name); - void remove (void); + RemoteFile(IOFD fd, const std::string &name); + void remove(void); std::string name_; }; -#endif // STORAGE_FACTORY_REMOTE_FILE_H +#endif // STORAGE_FACTORY_REMOTE_FILE_H diff --git a/Utilities/StorageFactory/interface/StatisticsSenderService.h b/Utilities/StorageFactory/interface/StatisticsSenderService.h index fd316899a43a5..24651dcc9f9c2 100644 --- a/Utilities/StorageFactory/interface/StatisticsSenderService.h +++ b/Utilities/StorageFactory/interface/StatisticsSenderService.h @@ -9,56 +9,55 @@ namespace edm { - class ParameterSet; + class ParameterSet; class ActivityRegistry; namespace storage { class StatisticsSenderService { - public: - StatisticsSenderService(edm::ParameterSet const& pset, edm::ActivityRegistry& ar); + public: + StatisticsSenderService(edm::ParameterSet const &pset, edm::ActivityRegistry &ar); - void setSize(size_t size); - void setCurrentServer(const std::string &servername); - void filePreCloseEvent(std::string const& lfn, bool usedFallback); - static const char * getJobID(); - static bool getX509Subject(std::string &); - private: + void setSize(size_t size); + void setCurrentServer(const std::string &servername); + void filePreCloseEvent(std::string const &lfn, bool usedFallback); + static const char *getJobID(); + static bool getX509Subject(std::string &); - class FileStatistics { - public: - FileStatistics(); - void fillUDP(std::ostringstream &os); - private: - ssize_t m_read_single_operations; - ssize_t m_read_single_bytes; - ssize_t m_read_single_square; - ssize_t m_read_vector_operations; - ssize_t m_read_vector_bytes; - ssize_t m_read_vector_square; - ssize_t m_read_vector_count_sum; - ssize_t m_read_vector_count_square; - time_t m_start_time; - }; + private: + class FileStatistics { + public: + FileStatistics(); + void fillUDP(std::ostringstream &os); - void determineHostnames(void); - void fillUDP(const std::string&, bool, std::string &); - std::string m_clienthost; - std::string m_clientdomain; - std::string m_serverhost; - std::string m_serverdomain; - std::string m_filelfn; - FileStatistics m_filestats; - std::string m_guid; - size_t m_counter; - std::atomic m_size; - std::string m_userdn; - std::mutex m_servermutex; + private: + ssize_t m_read_single_operations; + ssize_t m_read_single_bytes; + ssize_t m_read_single_square; + ssize_t m_read_vector_operations; + ssize_t m_read_vector_bytes; + ssize_t m_read_vector_square; + ssize_t m_read_vector_count_sum; + ssize_t m_read_vector_count_square; + time_t m_start_time; + }; + + void determineHostnames(void); + void fillUDP(const std::string &, bool, std::string &); + std::string m_clienthost; + std::string m_clientdomain; + std::string m_serverhost; + std::string m_serverdomain; + std::string m_filelfn; + FileStatistics m_filestats; + std::string m_guid; + size_t m_counter; + std::atomic m_size; + std::string m_userdn; + std::mutex m_servermutex; }; - - } -} + } // namespace storage +} // namespace edm #endif - diff --git a/Utilities/StorageFactory/interface/Storage.h b/Utilities/StorageFactory/interface/Storage.h index 2441867bef194..073f55bbc8cc1 100644 --- a/Utilities/StorageFactory/interface/Storage.h +++ b/Utilities/StorageFactory/interface/Storage.h @@ -1,9 +1,9 @@ #ifndef STORAGE_FACTORY_STORAGE_H -# define STORAGE_FACTORY_STORAGE_H +#define STORAGE_FACTORY_STORAGE_H -# include "Utilities/StorageFactory/interface/IOInput.h" -# include "Utilities/StorageFactory/interface/IOOutput.h" -# include "Utilities/StorageFactory/interface/IOPosBuffer.h" +#include "Utilities/StorageFactory/interface/IOInput.h" +#include "Utilities/StorageFactory/interface/IOOutput.h" +#include "Utilities/StorageFactory/interface/IOPosBuffer.h" // // ROOT will probe for prefetching support by calling @@ -15,45 +15,44 @@ // implementation, but prefers it not to be used by default, it // should detect the probe and return true. // -# define PREFETCH_PROBE_LENGTH 4096 +#define PREFETCH_PROBE_LENGTH 4096 -class Storage : public virtual IOInput, public virtual IOOutput -{ +class Storage : public virtual IOInput, public virtual IOOutput { public: enum Relative { SET, CURRENT, END }; - Storage (void); - ~Storage (void) override; + Storage(void); + ~Storage(void) override; using IOInput::read; using IOInput::readv; using IOOutput::write; using IOOutput::writev; - virtual bool prefetch (const IOPosBuffer *what, IOSize n); - virtual IOSize read (void *into, IOSize n, IOOffset pos); - IOSize read (IOBuffer into, IOOffset pos); - virtual IOSize readv (IOPosBuffer *into, IOSize buffers); - virtual IOSize write (const void *from, IOSize n, IOOffset pos); - IOSize write (IOBuffer from, IOOffset pos); - virtual IOSize writev (const IOPosBuffer *from, IOSize buffers); + virtual bool prefetch(const IOPosBuffer *what, IOSize n); + virtual IOSize read(void *into, IOSize n, IOOffset pos); + IOSize read(IOBuffer into, IOOffset pos); + virtual IOSize readv(IOPosBuffer *into, IOSize buffers); + virtual IOSize write(const void *from, IOSize n, IOOffset pos); + IOSize write(IOBuffer from, IOOffset pos); + virtual IOSize writev(const IOPosBuffer *from, IOSize buffers); - virtual bool eof (void) const; - virtual IOOffset size (void) const; - virtual IOOffset position (void) const; - virtual IOOffset position (IOOffset offset, Relative whence = SET) = 0; + virtual bool eof(void) const; + virtual IOOffset size(void) const; + virtual IOOffset position(void) const; + virtual IOOffset position(IOOffset offset, Relative whence = SET) = 0; - virtual void rewind (void); + virtual void rewind(void); - virtual void resize (IOOffset size) = 0; + virtual void resize(IOOffset size) = 0; - virtual void flush (void); - virtual void close (void); + virtual void flush(void); + virtual void close(void); private: // undefined, no semantics - Storage (const Storage &) = delete; - Storage &operator= (const Storage &) = delete; + Storage(const Storage &) = delete; + Storage &operator=(const Storage &) = delete; }; -#endif // STORAGE_FACTORY_STORAGE_H +#endif // STORAGE_FACTORY_STORAGE_H diff --git a/Utilities/StorageFactory/interface/StorageAccount.h b/Utilities/StorageFactory/interface/StorageAccount.h index 07abed8b62cdf..89cc39cc4c860 100644 --- a/Utilities/StorageFactory/interface/StorageAccount.h +++ b/Utilities/StorageFactory/interface/StorageAccount.h @@ -1,17 +1,16 @@ #ifndef STORAGE_FACTORY_STORAGE_ACCOUNT_H -# define STORAGE_FACTORY_STORAGE_ACCOUNT_H +#define STORAGE_FACTORY_STORAGE_ACCOUNT_H -# include -# include -# include -# include -# include -# include -# include "tbb/concurrent_unordered_map.h" +#include +#include +#include +#include +#include +#include +#include "tbb/concurrent_unordered_map.h" class StorageAccount { public: - enum class Operation { check, close, @@ -36,103 +35,102 @@ class StorageAccount { writeViaCache, writev }; - + struct Counter { - Counter(): - attempts{0}, - successes{0}, - amount{0}, - amount_square{0.}, - vector_count{0}, - vector_square{0}, - timeTotal{0.}, - timeMin{0.}, - timeMax{0.} {} - - Counter(Counter&& ) = default; + Counter() + : attempts{0}, + successes{0}, + amount{0}, + amount_square{0.}, + vector_count{0}, + vector_square{0}, + timeTotal{0.}, + timeMin{0.}, + timeMax{0.} {} + + Counter(Counter&&) = default; //NOTE: This is needed by tbb::concurrent_unordered_map when it // is constructing a new one. This would not give correct results // if the object being passed were being updated, but that is not // the case for operator[] - Counter( Counter const& iOther): - attempts{iOther.attempts.load()}, - successes{iOther.successes.load()}, - amount{iOther.amount.load()}, - amount_square{iOther.amount_square.load()}, - vector_count{iOther.vector_count.load()}, - vector_square{iOther.vector_square.load()}, - timeTotal{iOther.timeTotal.load()}, - timeMin{iOther.timeMin.load()}, - timeMax{iOther.timeMax.load()} {} - + Counter(Counter const& iOther) + : attempts{iOther.attempts.load()}, + successes{iOther.successes.load()}, + amount{iOther.amount.load()}, + amount_square{iOther.amount_square.load()}, + vector_count{iOther.vector_count.load()}, + vector_square{iOther.vector_square.load()}, + timeTotal{iOther.timeTotal.load()}, + timeMin{iOther.timeMin.load()}, + timeMax{iOther.timeMax.load()} {} + //Use atomics to allow concurrent read/write for intermediate // output of the statics while running. The values obtained // won't be completely consistent but should be good enough for // monitoring. The values obtained once the program is shutting // down should be completely consistent. - + std::atomic attempts; std::atomic successes; std::atomic amount; // NOTE: Significant risk exists for underflow in this value. // However, the use cases are marginal so I let it pass. - std::atomic amount_square; - std::atomic vector_count; - std::atomic vector_square; - std::atomic timeTotal; - std::atomic timeMin; - std::atomic timeMax; - + std::atomic amount_square; + std::atomic vector_count; + std::atomic vector_square; + std::atomic timeTotal; + std::atomic timeMin; + std::atomic timeMax; + static void addTo(std::atomic& iAtomic, double iToAdd) { double oldValue = iAtomic.load(); double newValue = oldValue + iToAdd; - while( not iAtomic.compare_exchange_weak(oldValue, newValue)) { - newValue = oldValue+iToAdd; + while (not iAtomic.compare_exchange_weak(oldValue, newValue)) { + newValue = oldValue + iToAdd; } } }; class Stamp { public: - Stamp (Counter &counter); + Stamp(Counter& counter); + + void tick(uint64_t amount = 0, int64_t tick = 0) const; - void tick (uint64_t amount = 0, int64_t tick = 0) const; protected: - Counter &m_counter; + Counter& m_counter; std::chrono::time_point m_start; }; class StorageClassToken { public: StorageClassToken(StorageClassToken const&) = default; - int value() const { return m_value;} + int value() const { return m_value; } friend class StorageAccount; + private: StorageClassToken() = delete; explicit StorageClassToken(int iValue) : m_value{iValue} {} int m_value; - }; - + typedef tbb::concurrent_unordered_map OperationStats; - typedef tbb::concurrent_unordered_map StorageStats; + typedef tbb::concurrent_unordered_map StorageStats; static char const* operationName(Operation operation); - static StorageClassToken tokenForStorageClassName( std::string const& iName); - static const std::string& nameForToken( StorageClassToken); - + static StorageClassToken tokenForStorageClassName(std::string const& iName); + static const std::string& nameForToken(StorageClassToken); + static const StorageStats& summary(void); - static std::string summaryText(bool banner=false); - static void fillSummary(std::map &summary); - static Counter& counter (StorageClassToken token, - Operation operation); + static std::string summaryText(bool banner = false); + static void fillSummary(std::map& summary); + static Counter& counter(StorageClassToken token, Operation operation); private: static StorageStats m_stats; - }; -#endif // STORAGE_FACTORY_STORAGE_ACCOUNT_H +#endif // STORAGE_FACTORY_STORAGE_ACCOUNT_H diff --git a/Utilities/StorageFactory/interface/StorageAccountProxy.h b/Utilities/StorageFactory/interface/StorageAccountProxy.h index 3051d4d0b49ce..3364fde2eb678 100644 --- a/Utilities/StorageFactory/interface/StorageAccountProxy.h +++ b/Utilities/StorageFactory/interface/StorageAccountProxy.h @@ -1,10 +1,10 @@ #ifndef STORAGE_FACTORY_STORAGE_ACCOUNT_PROXY_H -# define STORAGE_FACTORY_STORAGE_ACCOUNT_PROXY_H +#define STORAGE_FACTORY_STORAGE_ACCOUNT_PROXY_H -# include "Utilities/StorageFactory/interface/StorageAccount.h" -# include "Utilities/StorageFactory/interface/Storage.h" -# include "FWCore/Utilities/interface/get_underlying_safe.h" -# include +#include "Utilities/StorageFactory/interface/StorageAccount.h" +#include "Utilities/StorageFactory/interface/Storage.h" +#include "FWCore/Utilities/interface/get_underlying_safe.h" +#include #include /** Proxy class that wraps SEAL's #Storage class with one that ticks @@ -15,32 +15,31 @@ Future improvement would be to implement more methods so that the wrapper itself doesn't cause peroformance degradation if the base storage does actually implement "sophisticated" features. */ -class StorageAccountProxy : public Storage -{ +class StorageAccountProxy : public Storage { public: - StorageAccountProxy (const std::string &storageClass, std::unique_ptr baseStorage); - ~StorageAccountProxy (void) override; + StorageAccountProxy(const std::string &storageClass, std::unique_ptr baseStorage); + ~StorageAccountProxy(void) override; using Storage::read; using Storage::write; - bool prefetch (const IOPosBuffer *what, IOSize n) override; - IOSize read (void *into, IOSize n) override; - IOSize read (void *into, IOSize n, IOOffset pos) override; - IOSize readv (IOBuffer *into, IOSize n) override; - IOSize readv (IOPosBuffer *into, IOSize n) override; - IOSize write (const void *from, IOSize n) override; - IOSize write (const void *from, IOSize n, IOOffset pos) override; - IOSize writev (const IOBuffer *from, IOSize n) override; - IOSize writev (const IOPosBuffer *from, IOSize n) override; - - IOOffset position (IOOffset offset, Relative whence = SET) override; - void resize (IOOffset size) override; - void flush (void) override; - void close (void) override; + bool prefetch(const IOPosBuffer *what, IOSize n) override; + IOSize read(void *into, IOSize n) override; + IOSize read(void *into, IOSize n, IOOffset pos) override; + IOSize readv(IOBuffer *into, IOSize n) override; + IOSize readv(IOPosBuffer *into, IOSize n) override; + IOSize write(const void *from, IOSize n) override; + IOSize write(const void *from, IOSize n, IOOffset pos) override; + IOSize writev(const IOBuffer *from, IOSize n) override; + IOSize writev(const IOPosBuffer *from, IOSize n) override; + + IOOffset position(IOOffset offset, Relative whence = SET) override; + void resize(IOOffset size) override; + void flush(void) override; + void close(void) override; protected: - void releaseStorage() {get_underlying_safe(m_baseStorage).release();} + void releaseStorage() { get_underlying_safe(m_baseStorage).release(); } edm::propagate_const> m_baseStorage; @@ -53,4 +52,4 @@ class StorageAccountProxy : public Storage StorageAccount::Counter &m_statsPrefetch; }; -#endif // STORAGE_FACTORY_STORAGE_ACCOUNT_PROXY_H +#endif // STORAGE_FACTORY_STORAGE_ACCOUNT_PROXY_H diff --git a/Utilities/StorageFactory/interface/StorageFactory.h b/Utilities/StorageFactory/interface/StorageFactory.h index 13dd7f8ba2b1b..c082a3f8d6e9f 100644 --- a/Utilities/StorageFactory/interface/StorageFactory.h +++ b/Utilities/StorageFactory/interface/StorageFactory.h @@ -1,93 +1,77 @@ #ifndef STORAGE_FACTORY_STORAGE_FACTORY_H -# define STORAGE_FACTORY_STORAGE_FACTORY_H +#define STORAGE_FACTORY_STORAGE_FACTORY_H -# include "Utilities/StorageFactory/interface/StorageMaker.h" -# include "Utilities/StorageFactory/interface/LocalFileSystem.h" -# include "Utilities/StorageFactory/interface/IOTypes.h" -# include "Utilities/StorageFactory/interface/IOFlags.h" -# include +#include "Utilities/StorageFactory/interface/StorageMaker.h" +#include "Utilities/StorageFactory/interface/LocalFileSystem.h" +#include "Utilities/StorageFactory/interface/IOTypes.h" +#include "Utilities/StorageFactory/interface/IOFlags.h" +#include #include #include "tbb/concurrent_unordered_map.h" class Storage; -class StorageFactory -{ +class StorageFactory { public: - enum CacheHint - { - CACHE_HINT_APPLICATION, - CACHE_HINT_STORAGE, - CACHE_HINT_LAZY_DOWNLOAD, - CACHE_HINT_AUTO_DETECT - }; - - enum ReadHint - { - READ_HINT_UNBUFFERED, - READ_HINT_READAHEAD, - READ_HINT_AUTO - }; - - static const StorageFactory *get (void); - static StorageFactory *getToModify (void); - - ~StorageFactory (void); + enum CacheHint { CACHE_HINT_APPLICATION, CACHE_HINT_STORAGE, CACHE_HINT_LAZY_DOWNLOAD, CACHE_HINT_AUTO_DETECT }; + + enum ReadHint { READ_HINT_UNBUFFERED, READ_HINT_READAHEAD, READ_HINT_AUTO }; + + static const StorageFactory *get(void); + static StorageFactory *getToModify(void); + + ~StorageFactory(void); // implicit copy constructor // implicit assignment operator - void setCacheHint(CacheHint value); - CacheHint cacheHint(void) const; + void setCacheHint(CacheHint value); + CacheHint cacheHint(void) const; - void setReadHint(ReadHint value); - ReadHint readHint(void) const; + void setReadHint(ReadHint value); + ReadHint readHint(void) const; - bool enableAccounting (bool enabled); - bool accounting (void) const; + bool enableAccounting(bool enabled); + bool accounting(void) const; - void setTimeout(unsigned int timeout); - unsigned int timeout(void) const; + void setTimeout(unsigned int timeout); + unsigned int timeout(void) const; - void setDebugLevel(unsigned int level); - unsigned int debugLevel(void) const; + void setDebugLevel(unsigned int level); + unsigned int debugLevel(void) const; - void setTempDir (const std::string &s, double minFreeSpace); - std::string tempDir (void) const; - std::string tempPath (void) const; - double tempMinFree (void) const; + void setTempDir(const std::string &s, double minFreeSpace); + std::string tempDir(void) const; + std::string tempPath(void) const; + double tempMinFree(void) const; - void stagein (const std::string &url) const; - std::unique_ptr open (const std::string &url, - int mode = IOFlags::OpenRead) const; - bool check (const std::string &url, - IOOffset *size = nullptr) const; + void stagein(const std::string &url) const; + std::unique_ptr open(const std::string &url, int mode = IOFlags::OpenRead) const; + bool check(const std::string &url, IOOffset *size = nullptr) const; - std::unique_ptr wrapNonLocalFile (std::unique_ptr s, - const std::string &proto, - const std::string &path, - int mode) const; + std::unique_ptr wrapNonLocalFile(std::unique_ptr s, + const std::string &proto, + const std::string &path, + int mode) const; private: typedef tbb::concurrent_unordered_map> MakerTable; - StorageFactory (void); - StorageMaker *getMaker (const std::string &proto) const; - StorageMaker *getMaker (const std::string &url, - std::string &protocol, - std::string &rest) const; - - mutable MakerTable m_makers; - CacheHint m_cacheHint; - ReadHint m_readHint; - bool m_accounting; - double m_tempfree; - std::string m_temppath; - std::string m_tempdir; + StorageFactory(void); + StorageMaker *getMaker(const std::string &proto) const; + StorageMaker *getMaker(const std::string &url, std::string &protocol, std::string &rest) const; + + mutable MakerTable m_makers; + CacheHint m_cacheHint; + ReadHint m_readHint; + bool m_accounting; + double m_tempfree; + std::string m_temppath; + std::string m_tempdir; std::string m_unusableDirWarnings; - unsigned int m_timeout; - unsigned int m_debugLevel; + unsigned int m_timeout; + unsigned int m_debugLevel; LocalFileSystem m_lfs; static StorageFactory s_instance; }; -#endif // STORAGE_FACTORY_STORAGE_FACTORY_H +#endif // STORAGE_FACTORY_STORAGE_FACTORY_H diff --git a/Utilities/StorageFactory/interface/StorageMaker.h b/Utilities/StorageFactory/interface/StorageMaker.h index 4b18b319cb5c9..231da848ac5e8 100644 --- a/Utilities/StorageFactory/interface/StorageMaker.h +++ b/Utilities/StorageFactory/interface/StorageMaker.h @@ -1,45 +1,42 @@ #ifndef STORAGE_FACTORY_STORAGE_MAKER_H -# define STORAGE_FACTORY_STORAGE_MAKER_H +#define STORAGE_FACTORY_STORAGE_MAKER_H -# include "Utilities/StorageFactory/interface/IOTypes.h" -# include +#include "Utilities/StorageFactory/interface/IOTypes.h" +#include #include class Storage; -class StorageMaker -{ +class StorageMaker { public: struct AuxSettings { unsigned int timeout = 0; unsigned int debugLevel = 0; - - AuxSettings& setDebugLevel(unsigned int iLevel) { + + AuxSettings &setDebugLevel(unsigned int iLevel) { debugLevel = iLevel; return *this; } - - AuxSettings& setTimeout(unsigned int iTime) { + + AuxSettings &setTimeout(unsigned int iTime) { timeout = iTime; return *this; } }; - + StorageMaker() = default; - virtual ~StorageMaker () = default; + virtual ~StorageMaker() = default; // implicit copy constructor // implicit assignment operator - virtual void stagein (const std::string &proto, - const std::string &path, - const AuxSettings& aux) const; - virtual std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - const AuxSettings& aux) const = 0; - virtual bool check (const std::string &proto, - const std::string &path, - const AuxSettings& aux, - IOOffset *size = nullptr) const; + virtual void stagein(const std::string &proto, const std::string &path, const AuxSettings &aux) const; + virtual std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + const AuxSettings &aux) const = 0; + virtual bool check(const std::string &proto, + const std::string &path, + const AuxSettings &aux, + IOOffset *size = nullptr) const; }; -#endif // STORAGE_FACTORY_STORAGE_MAKER_H +#endif // STORAGE_FACTORY_STORAGE_MAKER_H diff --git a/Utilities/StorageFactory/interface/StorageMakerFactory.h b/Utilities/StorageFactory/interface/StorageMakerFactory.h index 98cef9297855f..85465fa431c6e 100644 --- a/Utilities/StorageFactory/interface/StorageMakerFactory.h +++ b/Utilities/StorageFactory/interface/StorageMakerFactory.h @@ -1,9 +1,9 @@ #ifndef STORAGE_FACTORY_STORAGE_MAKER_FACTORY_H -# define STORAGE_FACTORY_STORAGE_MAKER_FACTORY_H +#define STORAGE_FACTORY_STORAGE_MAKER_FACTORY_H -# include "FWCore/PluginManager/interface/PluginFactory.h" +#include "FWCore/PluginManager/interface/PluginFactory.h" class StorageMaker; typedef edmplugin::PluginFactory StorageMakerFactory; -#endif // STORAGE_FACTORY_STORAGE_MAKER_FACTORY_H +#endif // STORAGE_FACTORY_STORAGE_MAKER_FACTORY_H diff --git a/Utilities/StorageFactory/plugins/GsiFTPStorageMaker.cc b/Utilities/StorageFactory/plugins/GsiFTPStorageMaker.cc index 296322664139b..ffccd0125afb0 100644 --- a/Utilities/StorageFactory/plugins/GsiFTPStorageMaker.cc +++ b/Utilities/StorageFactory/plugins/GsiFTPStorageMaker.cc @@ -3,23 +3,21 @@ #include "Utilities/StorageFactory/interface/StorageFactory.h" #include "Utilities/StorageFactory/interface/RemoteFile.h" -class GsiFTPStorageMaker : public StorageMaker -{ +class GsiFTPStorageMaker : public StorageMaker { public: - std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - const AuxSettings&) const override - { - std::string temp; + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + const AuxSettings &) const override { + std::string temp; const StorageFactory *f = StorageFactory::get(); - int localfd = RemoteFile::local (f->tempDir(), temp); - std::string lurl = "file://" + temp; - std::string newurl ((proto == "sfn" ? "gsiftp" : proto) + ":" + path); - const char *ftpopts [] = { "globus-url-copy", newurl.c_str (), lurl.c_str (), nullptr }; - return RemoteFile::get (localfd, temp, (char **) ftpopts, mode); + int localfd = RemoteFile::local(f->tempDir(), temp); + std::string lurl = "file://" + temp; + std::string newurl((proto == "sfn" ? "gsiftp" : proto) + ":" + path); + const char *ftpopts[] = {"globus-url-copy", newurl.c_str(), lurl.c_str(), nullptr}; + return RemoteFile::get(localfd, temp, (char **)ftpopts, mode); } }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, GsiFTPStorageMaker, "gsiftp"); -DEFINE_EDM_PLUGIN (StorageMakerFactory, GsiFTPStorageMaker, "sfn"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, GsiFTPStorageMaker, "gsiftp"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, GsiFTPStorageMaker, "sfn"); diff --git a/Utilities/StorageFactory/plugins/HttpStorageMaker.cc b/Utilities/StorageFactory/plugins/HttpStorageMaker.cc index 1ae77a9bfa77f..ab2e40ae7ce3f 100644 --- a/Utilities/StorageFactory/plugins/HttpStorageMaker.cc +++ b/Utilities/StorageFactory/plugins/HttpStorageMaker.cc @@ -3,25 +3,20 @@ #include "Utilities/StorageFactory/interface/StorageFactory.h" #include "Utilities/StorageFactory/interface/RemoteFile.h" -class HttpStorageMaker : public StorageMaker -{ +class HttpStorageMaker : public StorageMaker { public: - std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - const AuxSettings&) const override - { - std::string temp; + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + const AuxSettings &) const override { + std::string temp; const StorageFactory *f = StorageFactory::get(); - int localfd = RemoteFile::local (f->tempDir(), temp); - std::string newurl ((proto == "web" ? "http" : proto) + ":" + path); - const char *curlopts [] = { - "curl", "-L", "-f", "-o", temp.c_str(), "-q", "-s", "--url", - newurl.c_str (), nullptr - }; + int localfd = RemoteFile::local(f->tempDir(), temp); + std::string newurl((proto == "web" ? "http" : proto) + ":" + path); + const char *curlopts[] = {"curl", "-L", "-f", "-o", temp.c_str(), "-q", "-s", "--url", newurl.c_str(), nullptr}; - return RemoteFile::get (localfd, temp, (char **) curlopts, mode); + return RemoteFile::get(localfd, temp, (char **)curlopts, mode); } }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, HttpStorageMaker, "ftp"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, HttpStorageMaker, "ftp"); diff --git a/Utilities/StorageFactory/plugins/LocalStorageMaker.cc b/Utilities/StorageFactory/plugins/LocalStorageMaker.cc index 8d7d46603ea10..1f24e855902e1 100644 --- a/Utilities/StorageFactory/plugins/LocalStorageMaker.cc +++ b/Utilities/StorageFactory/plugins/LocalStorageMaker.cc @@ -8,42 +8,38 @@ #include #include -class LocalStorageMaker : public StorageMaker -{ +class LocalStorageMaker : public StorageMaker { public: - std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - const AuxSettings&) const override - { - const StorageFactory *f = StorageFactory::get(); - StorageFactory::ReadHint readHint = f->readHint(); - StorageFactory::CacheHint cacheHint = f->cacheHint(); + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + const AuxSettings &) const override { + const StorageFactory *f = StorageFactory::get(); + StorageFactory::ReadHint readHint = f->readHint(); + StorageFactory::CacheHint cacheHint = f->cacheHint(); - if (readHint != StorageFactory::READ_HINT_UNBUFFERED - || cacheHint == StorageFactory::CACHE_HINT_STORAGE) - mode &= ~IOFlags::OpenUnbuffered; - else - mode |= IOFlags::OpenUnbuffered; + if (readHint != StorageFactory::READ_HINT_UNBUFFERED || cacheHint == StorageFactory::CACHE_HINT_STORAGE) + mode &= ~IOFlags::OpenUnbuffered; + else + mode |= IOFlags::OpenUnbuffered; - auto file = std::make_unique (path, mode); - return f->wrapNonLocalFile (std::move(file), proto, path, mode); - } + auto file = std::make_unique(path, mode); + return f->wrapNonLocalFile(std::move(file), proto, path, mode); + } - bool check (const std::string &/*proto*/, - const std::string &path, - const AuxSettings&, - IOOffset *size = nullptr) const override - { - struct stat st; - if (stat (path.c_str(), &st) != 0) - return false; + bool check(const std::string & /*proto*/, + const std::string &path, + const AuxSettings &, + IOOffset *size = nullptr) const override { + struct stat st; + if (stat(path.c_str(), &st) != 0) + return false; - if (size) - *size = st.st_size; + if (size) + *size = st.st_size; - return true; - } + return true; + } }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, LocalStorageMaker, "file"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, LocalStorageMaker, "file"); diff --git a/Utilities/StorageFactory/plugins/StatisticsSenderServiceMaker.cc b/Utilities/StorageFactory/plugins/StatisticsSenderServiceMaker.cc index 23b2807b76056..e9b3dc7e5c832 100644 --- a/Utilities/StorageFactory/plugins/StatisticsSenderServiceMaker.cc +++ b/Utilities/StorageFactory/plugins/StatisticsSenderServiceMaker.cc @@ -7,4 +7,3 @@ using edm::storage::StatisticsSenderService; typedef edm::serviceregistry::AllArgsMaker StatisticsSenderServiceMaker; DEFINE_FWK_SERVICE_MAKER(StatisticsSenderService, StatisticsSenderServiceMaker); - diff --git a/Utilities/StorageFactory/plugins/StormLCGStorageMaker.cc b/Utilities/StorageFactory/plugins/StormLCGStorageMaker.cc index 0a69830ba42f9..e4b3dc88b0e07 100644 --- a/Utilities/StorageFactory/plugins/StormLCGStorageMaker.cc +++ b/Utilities/StorageFactory/plugins/StormLCGStorageMaker.cc @@ -9,28 +9,24 @@ #include #include -class StormLcgGtStorageMaker : public StorageMaker -{ +class StormLcgGtStorageMaker : public StorageMaker { /* getTURL: Executes lcg-gt and extracts the physical file path */ - std::string getTURL (const std::string &surl) const - { - // PrepareToGet timeout + std::string getTURL(const std::string &surl) const { + // PrepareToGet timeout std::string timeout("300"); - if(char *p = getenv("CMS_STORM_LCG_GT_TIMEOUT")) + if (char *p = getenv("CMS_STORM_LCG_GT_TIMEOUT")) timeout = p; /* Build the command line: -b => no BDII contacted -T srmv2 => necessary with -b -t timeout */ - std::string comm("lcg-gt -b -T srmv2 -t " + timeout + " srm:" + surl + " file 2>&1"); + std::string comm("lcg-gt -b -T srmv2 -t " + timeout + " srm:" + surl + " file 2>&1"); LogDebug("StormLCGStorageMaker") << "command: " << comm << std::endl; FILE *pipe = popen(comm.c_str(), "r"); - if(! pipe) - throw cms::Exception("StormLCGStorageMaker") - << "failed to execute lcg-gt command: " - << comm; + if (!pipe) + throw cms::Exception("StormLCGStorageMaker") << "failed to execute lcg-gt command: " << comm; // Get output int ch; @@ -40,48 +36,43 @@ class StormLcgGtStorageMaker : public StorageMaker pclose(pipe); LogDebug("StormLCGStorageMaker") << "output: " << output << std::endl; - + // Extract TURL if possible. size_t start = output.find("file:", 0); if (start == std::string::npos) - throw cms::Exception("StormLCGStorageMaker") - << "no turl found in command '" << comm << "' output:\n" << output; + throw cms::Exception("StormLCGStorageMaker") << "no turl found in command '" << comm << "' output:\n" << output; start += 5; - std::string turl(output, start, output.find_first_of("\n", start) - start); + std::string turl(output, start, output.find_first_of("\n", start) - start); LogDebug("StormLCGStorageMaker") << "file to open: " << turl << std::endl; return turl; } - public: - std::unique_ptr open (const std::string &proto, - const std::string &surl, - int mode, - const AuxSettings& ) const override - { + std::unique_ptr open(const std::string &proto, + const std::string &surl, + int mode, + const AuxSettings &) const override { const StorageFactory *f = StorageFactory::get(); StorageFactory::ReadHint readHint = f->readHint(); StorageFactory::CacheHint cacheHint = f->cacheHint(); - if (readHint != StorageFactory::READ_HINT_UNBUFFERED - || cacheHint == StorageFactory::CACHE_HINT_STORAGE) + if (readHint != StorageFactory::READ_HINT_UNBUFFERED || cacheHint == StorageFactory::CACHE_HINT_STORAGE) mode &= ~IOFlags::OpenUnbuffered; else mode |= IOFlags::OpenUnbuffered; std::string path = getTURL(surl); - auto file = std::make_unique (path, mode); - return f->wrapNonLocalFile (std::move(file), proto, path, mode); + auto file = std::make_unique(path, mode); + return f->wrapNonLocalFile(std::move(file), proto, path, mode); } - bool check (const std::string &/*proto*/, - const std::string &path, - const AuxSettings&, - IOOffset *size = nullptr) const override - { + bool check(const std::string & /*proto*/, + const std::string &path, + const AuxSettings &, + IOOffset *size = nullptr) const override { struct stat st; - if (stat (getTURL(path).c_str(), &st) != 0) + if (stat(getTURL(path).c_str(), &st) != 0) return false; if (size) @@ -91,4 +82,4 @@ class StormLcgGtStorageMaker : public StorageMaker } }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, StormLcgGtStorageMaker, "storm-lcg"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, StormLcgGtStorageMaker, "storm-lcg"); diff --git a/Utilities/StorageFactory/plugins/StormStorageMaker.cc b/Utilities/StorageFactory/plugins/StormStorageMaker.cc index 7d16aaaca3bfb..2c7d46e34db78 100644 --- a/Utilities/StorageFactory/plugins/StormStorageMaker.cc +++ b/Utilities/StorageFactory/plugins/StormStorageMaker.cc @@ -9,27 +9,22 @@ #include #include -class StormStorageMaker : public StorageMaker -{ +class StormStorageMaker : public StorageMaker { /* getTURL: Executes a prepare to get script and extracts the physical file path */ - std::string getTURL (const std::string &surl) const - { + std::string getTURL(const std::string &surl) const { std::string client; if (char *p = getenv("CMS_STORM_PTG_CLIENT")) client = p; else - throw cms::Exception("StormStorageMaker") - << "$CMS_STORM_PTG_CLIENT has no value"; + throw cms::Exception("StormStorageMaker") << "$CMS_STORM_PTG_CLIENT has no value"; // Command - std::string comm(client + " srm:" + surl + " 2>&1"); + std::string comm(client + " srm:" + surl + " 2>&1"); LogDebug("StormStorageMaker") << "command: " << comm << std::endl; FILE *pipe = popen(comm.c_str(), "r"); - if(! pipe) - throw cms::Exception("StormStorageMaker") - << "failed to execute PtG command: " - << comm; + if (!pipe) + throw cms::Exception("StormStorageMaker") << "failed to execute PtG command: " << comm; // Get output int ch; @@ -39,48 +34,43 @@ class StormStorageMaker : public StorageMaker pclose(pipe); LogDebug("StormStorageMaker") << "output: " << output << std::endl; - + // Extract TURL if possible. size_t start = output.find("FilePath:", 0); if (start == std::string::npos) - throw cms::Exception("StormStorageMaker") - << "no turl found in command '" << comm << "' output:\n" << output; + throw cms::Exception("StormStorageMaker") << "no turl found in command '" << comm << "' output:\n" << output; start += 9; - std::string turl(output, start, output.find_first_of("\n", start) - start); + std::string turl(output, start, output.find_first_of("\n", start) - start); LogDebug("StormStorageMaker") << "file to open: " << turl << std::endl; return turl; } - public: - std::unique_ptr open (const std::string &proto, - const std::string &surl, - int mode, - const AuxSettings&) const override - { + std::unique_ptr open(const std::string &proto, + const std::string &surl, + int mode, + const AuxSettings &) const override { const StorageFactory *f = StorageFactory::get(); StorageFactory::ReadHint readHint = f->readHint(); StorageFactory::CacheHint cacheHint = f->cacheHint(); - if (readHint != StorageFactory::READ_HINT_UNBUFFERED - || cacheHint == StorageFactory::CACHE_HINT_STORAGE) + if (readHint != StorageFactory::READ_HINT_UNBUFFERED || cacheHint == StorageFactory::CACHE_HINT_STORAGE) mode &= ~IOFlags::OpenUnbuffered; else mode |= IOFlags::OpenUnbuffered; std::string path = getTURL(surl); - auto file = std::make_unique (path, mode); - return f->wrapNonLocalFile (std::move(file), proto, path, mode); + auto file = std::make_unique(path, mode); + return f->wrapNonLocalFile(std::move(file), proto, path, mode); } - bool check (const std::string &/*proto*/, - const std::string &path, - const AuxSettings&, - IOOffset *size = nullptr) const override - { + bool check(const std::string & /*proto*/, + const std::string &path, + const AuxSettings &, + IOOffset *size = nullptr) const override { struct stat st; - if (stat (getTURL(path).c_str(), &st) != 0) + if (stat(getTURL(path).c_str(), &st) != 0) return false; if (size) @@ -90,4 +80,4 @@ class StormStorageMaker : public StorageMaker } }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, StormStorageMaker, "storm"); +DEFINE_EDM_PLUGIN(StorageMakerFactory, StormStorageMaker, "storm"); diff --git a/Utilities/StorageFactory/src/File.cc b/Utilities/StorageFactory/src/File.cc index 07ad0e9daba73..4e7d7297eaec8 100644 --- a/Utilities/StorageFactory/src/File.cc +++ b/Utilities/StorageFactory/src/File.cc @@ -54,43 +54,39 @@ using namespace IOFlags; ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// /** Create a new file object without a file attached to it. */ -File::File (void) -{ - fd (EDM_IOFD_INVALID); +File::File(void) { + fd(EDM_IOFD_INVALID); m_flags = 0; } /** Create a new file object from a file descriptor. The descriptor will be closed automatically when the file object is destructed if @a autoclose is @c true (the default). */ -File::File (IOFD fd, bool autoclose /* = true */) -{ - this->fd (fd); +File::File(IOFD fd, bool autoclose /* = true */) { + this->fd(fd); m_flags = autoclose ? InternalAutoClose : 0; } /** Internal function for copying file objects to retain the state flags. */ -File::File (IOFD fd, unsigned flags) -{ - this->fd (fd); +File::File(IOFD fd, unsigned flags) { + this->fd(fd); m_flags = flags; } /** Create a new file object by calling #open() with the given arguments. */ -File::File (const char *name, int flags /*= OpenRead*/, int perms /*= 0666*/) -{ open (name, flags, perms); } +File::File(const char *name, int flags /*= OpenRead*/, int perms /*= 0666*/) { open(name, flags, perms); } /** Create a new file object by calling #open() with the given arguments. */ -File::File (const std::string &name, int flags /*= OpenRead*/, int perms /*= 0666*/) -{ open (name.c_str (), flags, perms); } +File::File(const std::string &name, int flags /*= OpenRead*/, int perms /*= 0666*/) { + open(name.c_str(), flags, perms); +} /** Release the resources held by the file object. If the object holds a valid file descriptor given to it through the constructor or obtained by calling #open(), the descriptor will be closed. */ -File::~File (void) -{ +File::~File(void) { if (m_flags & InternalAutoClose) - abort (); + abort(); } ////////////////////////////////////////////////////////////////////// @@ -99,9 +95,7 @@ File::~File (void) descriptor. Otherwise the file descriptor will be left open. Set the flag off if the file descriptor is originally owned by someone else. */ -void -File::setAutoClose (bool autoclose) -{ +void File::setAutoClose(bool autoclose) { m_flags &= ~InternalAutoClose; if (autoclose) m_flags |= InternalAutoClose; @@ -113,22 +107,18 @@ File::setAutoClose (bool autoclose) same descriptor. If the file descriptor is not copied, the copy will not close its file descriptor on destruction, the original object (@c this) will. */ -File * -File::duplicate (bool copy) const -{ - File *dup = new File (fd (), copy ? m_flags : 0); - return copy ? this->duplicate (dup) : dup; +File *File::duplicate(bool copy) const { + File *dup = new File(fd(), copy ? m_flags : 0); + return copy ? this->duplicate(dup) : dup; } /** Internal implementation of #duplicate() to actually duplicate the file handle into @a child. */ -File * -File::duplicate (File *child) const -{ - IOFD fd = this->fd (); - assert (fd != EDM_IOFD_INVALID); - assert (child); - child->fd (sysduplicate (fd)); +File *File::duplicate(File *child) const { + IOFD fd = this->fd(); + assert(fd != EDM_IOFD_INVALID); + assert(child); + child->fd(sysduplicate(fd)); child->m_flags = m_flags; return child; } @@ -138,26 +128,16 @@ File::duplicate (File *child) const the creation fails if the file already exists, otherwise if the file exists, it will be truncated. The new file will have the permissions @a perms. */ -void -File::create (const char *name, bool exclusive /*=false*/, int perms/*=0666*/) -{ - open (name, - (OpenCreate | OpenWrite | OpenTruncate - | (exclusive ? OpenExclusive : 0)), - perms); +void File::create(const char *name, bool exclusive /*=false*/, int perms /*=0666*/) { + open(name, (OpenCreate | OpenWrite | OpenTruncate | (exclusive ? OpenExclusive : 0)), perms); } /** Create and open the file @a name in write mode. If @a exclusive, the creation fails if the file already exists, otherwise if the file exists, it will be truncated. The new file will have the permissions @a perms. */ -void -File::create (const std::string &name, bool exclusive /*=false*/, int perms/*=0666*/) -{ - open (name.c_str (), - (OpenCreate | OpenWrite | OpenTruncate - | (exclusive ? OpenExclusive : 0)), - perms); +void File::create(const std::string &name, bool exclusive /*=false*/, int perms /*=0666*/) { + open(name.c_str(), (OpenCreate | OpenWrite | OpenTruncate | (exclusive ? OpenExclusive : 0)), perms); } /** Open or possibly create the file @a name with options specified in @@ -165,53 +145,46 @@ File::create (const std::string &name, bool exclusive /*=false*/, int perms/*=06 permissions @a perms. If this object already has a file open, it is closed first. Redirected to the overloaded method taking a "const char *" argument. */ -void -File::open (const std::string &name, int flags /*= OpenRead*/, int perms /*= 0666*/) -{ open (name.c_str (), flags, perms); } +void File::open(const std::string &name, int flags /*= OpenRead*/, int perms /*= 0666*/) { + open(name.c_str(), flags, perms); +} /** Open or possibly create the file @a name with options specified in @a flags. If the file is to be created, it will be given the permissions @a perms. If this object already has a file open, it is closed first. */ -void -File::open (const char *name, int flags /*= OpenRead*/, int perms /*= 0666*/) -{ +void File::open(const char *name, int flags /*= OpenRead*/, int perms /*= 0666*/) { // is zero and always implied. OTOH, existence check should be // done with Filename::exists() -- see comments there about what // can happen on a WIN32 remote share even if the file doesn't // exist. For now make sure that read or write was asked for. - assert (name && *name); - assert (flags & (OpenRead | OpenWrite)); + assert(name && *name); + assert(flags & (OpenRead | OpenWrite)); // If I am already open, close the old file first. - if (fd () != EDM_IOFD_INVALID && (m_flags & InternalAutoClose)) - close (); - - IOFD newfd = EDM_IOFD_INVALID; - unsigned newflags = InternalAutoClose; + if (fd() != EDM_IOFD_INVALID && (m_flags & InternalAutoClose)) + close(); - sysopen (name, flags, perms, newfd, newflags); + IOFD newfd = EDM_IOFD_INVALID; + unsigned newflags = InternalAutoClose; - fd (newfd); + sysopen(name, flags, perms, newfd, newflags); + + fd(newfd); m_flags = newflags; } -void -File::attach (IOFD fd) -{ - this->fd (fd); +void File::attach(IOFD fd) { + this->fd(fd); m_flags = 0; } ////////////////////////////////////////////////////////////////////// /** Prefetch data for the file. */ -bool -File::prefetch (const IOPosBuffer *what, IOSize n) -{ +bool File::prefetch(const IOPosBuffer *what, IOSize n) { IOFD fd = this->fd(); - for (IOSize i = 0; i < n; ++i) - { + for (IOSize i = 0; i < n; ++i) { #if F_RDADVISE radvisory info; info.ra_offset = what[i].offset(); @@ -220,83 +193,69 @@ File::prefetch (const IOPosBuffer *what, IOSize n) #elif _POSIX_ADVISORY_INFO > 0 posix_fadvise(fd, what[i].offset(), what[i].size(), POSIX_FADV_WILLNEED); #else -# error advisory read ahead not available on this platform +#error advisory read ahead not available on this platform #endif } return true; } /** Read from the file. */ -IOSize -File::read (void *into, IOSize n) -{ return IOChannel::read (into, n); } +IOSize File::read(void *into, IOSize n) { return IOChannel::read(into, n); } /** Read from the file. */ -IOSize -File::readv (IOBuffer *into, IOSize length) -{ return IOChannel::readv (into, length); } +IOSize File::readv(IOBuffer *into, IOSize length) { return IOChannel::readv(into, length); } /** Write to the file. */ -IOSize -File::write (const void *from, IOSize n) -{ +IOSize File::write(const void *from, IOSize n) { // FIXME: This may create a race condition or cause trouble on // remote files. Should be currently needed only on WIN32. if (m_flags & OpenAppend) - position (0, END); + position(0, END); - IOSize s = IOChannel::write (from, n); + IOSize s = IOChannel::write(from, n); if (m_flags & OpenUnbuffered) // FIXME: Exception handling? - flush (); + flush(); return s; } /** Write to the file. */ -IOSize -File::writev (const IOBuffer *from, IOSize length) -{ +IOSize File::writev(const IOBuffer *from, IOSize length) { // FIXME: This may create a race condition or cause trouble on // remote files. Should be currently needed only on WIN32. if (m_flags & OpenAppend) - position (0, END); + position(0, END); - IOSize s = IOChannel::writev (from, length); + IOSize s = IOChannel::writev(from, length); if (m_flags & OpenUnbuffered) // FIXME: Exception handling? - flush (); + flush(); return s; } /** Close the file. */ -void -File::close (void) -{ - IOFD fd = this->fd (); - assert (fd != EDM_IOFD_INVALID); +void File::close(void) { + IOFD fd = this->fd(); + assert(fd != EDM_IOFD_INVALID); int error; - if (! sysclose (fd, &error)) - throwStorageError("FileCloseError", "Calling File::close()", - "sysclose", error); + if (!sysclose(fd, &error)) + throwStorageError("FileCloseError", "Calling File::close()", "sysclose", error); m_flags &= ~InternalAutoClose; - this->fd (EDM_IOFD_INVALID); + this->fd(EDM_IOFD_INVALID); } /** Close the file and ignore all errors. */ -void -File::abort (void) -{ - IOFD fd = this->fd (); - if (fd != EDM_IOFD_INVALID) - { - sysclose (fd); +void File::abort(void) { + IOFD fd = this->fd(); + if (fd != EDM_IOFD_INVALID) { + sysclose(fd); m_flags &= ~InternalAutoClose; - this->fd (EDM_IOFD_INVALID); + this->fd(EDM_IOFD_INVALID); } } diff --git a/Utilities/StorageFactory/src/IOChannel.cc b/Utilities/StorageFactory/src/IOChannel.cc index b1b5a4c18a223..d88beed02f27e 100644 --- a/Utilities/StorageFactory/src/IOChannel.cc +++ b/Utilities/StorageFactory/src/IOChannel.cc @@ -58,26 +58,19 @@ ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -IOChannel::IOChannel (IOFD fd /* = EDM_IOFD_INVALID */) - : m_fd (fd) -{} +IOChannel::IOChannel(IOFD fd /* = EDM_IOFD_INVALID */) : m_fd(fd) {} -IOChannel::~IOChannel (void) -{} +IOChannel::~IOChannel(void) {} ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// /** Get the system file descriptor of the channel. */ -IOFD -IOChannel::fd (void) const -{ return m_fd; } +IOFD IOChannel::fd(void) const { return m_fd; } /** Set the system file descriptor of the channel. (FIXME: This is dangerous. How to deal with WIN32 flags and state object?) */ -void -IOChannel::fd (IOFD value) -{ +void IOChannel::fd(IOFD value) { // FIXME: close old one? // FIXME: reset state? m_fd = value; @@ -88,16 +81,13 @@ IOChannel::fd (IOFD value) ////////////////////////////////////////////////////////////////////// /** Close the channel. By default closes the underlying operating system file descriptor. */ -void -IOChannel::close (void) -{ - if (fd () == EDM_IOFD_INVALID) +void IOChannel::close(void) { + if (fd() == EDM_IOFD_INVALID) return; int error = 0; - if (! sysclose (fd (), &error)) - throwStorageError ("FileCloseError", "Calling IOChannel::close()", - "sysclose()", error); + if (!sysclose(fd(), &error)) + throwStorageError("FileCloseError", "Calling IOChannel::close()", "sysclose()", error); - fd (EDM_IOFD_INVALID); + fd(EDM_IOFD_INVALID); } diff --git a/Utilities/StorageFactory/src/IOInput.cc b/Utilities/StorageFactory/src/IOInput.cc index 51fbf74cf6f57..058c69f031f36 100644 --- a/Utilities/StorageFactory/src/IOInput.cc +++ b/Utilities/StorageFactory/src/IOInput.cc @@ -3,8 +3,7 @@ #include /// Destruct the stream. A no-op. -IOInput::~IOInput (void) -{} +IOInput::~IOInput(void) {} ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// @@ -50,11 +49,9 @@ IOInput::~IOInput (void) includes the situation where the input stream is in non-blocking mode and no input is currently available (FIXME: make this simpler; clarify which exception). */ -int -IOInput::read (void) -{ +int IOInput::read(void) { unsigned char byte; - IOSize n = read (&byte, 1); + IOSize n = read(&byte, 1); return n == 0 ? -1 : byte; } @@ -80,9 +77,7 @@ IOInput::read (void) includes the situation where the input stream is in non-blocking mode and no input is currently available (FIXME: make this simpler; clarify which exception). */ -IOSize -IOInput::read (IOBuffer into) -{ return read (into.data (), into.size ()); } +IOSize IOInput::read(IOBuffer into) { return read(into.data(), into.size()); } /** Read from the input stream into multiple scattered buffers. There are @a buffers to fill in an array starting at @a into; @@ -113,24 +108,19 @@ IOInput::read (IOBuffer into) persistent errors will occur anyway on the next read and sporadic errors like stream becoming unvailable can be ignored. Use #xread() if a different policy is desirable. */ -IOSize -IOInput::readv (IOBuffer *into, IOSize buffers) -{ - assert (! buffers || into); +IOSize IOInput::readv(IOBuffer *into, IOSize buffers) { + assert(!buffers || into); // Keep reading as long as possible; ignore errors if we have read // something, otherwise pass it on. IOSize status; IOSize done = 0; - try - { + try { for (IOSize i = 0; i < buffers; done += status, ++i) - if ((status = read (into [i])) == 0) - break; - } - catch (cms::Exception &) - { - if (! done) + if ((status = read(into[i])) == 0) + break; + } catch (cms::Exception &) { + if (!done) throw; } @@ -162,9 +152,7 @@ IOInput::readv (IOBuffer *into, IOSize buffers) @throws All exceptions from #read() are passed through unhandled. Therefore it is possible that an exception is thrown when this function has already read some data. */ -IOSize -IOInput::xread (IOBuffer into) -{ return xread (into.data (), into.size ()); } +IOSize IOInput::xread(IOBuffer into) { return xread(into.data(), into.size()); } /** Like the corresponding #read() method but reads until the requested number of bytes are read or end of file is reached. @@ -187,16 +175,14 @@ IOInput::xread (IOBuffer into) @throws All exceptions from #read() are passed through unhandled. Therefore it is possible that an exception is thrown when this function has already read some data. */ -IOSize -IOInput::xread (void *into, IOSize n) -{ - assert (into); +IOSize IOInput::xread(void *into, IOSize n) { + assert(into); // Keep reading as long as possible. Let system errors fly over // us, they are a hard error. IOSize x; IOSize done = 0; - while (done < n && (x = read ((char *) into + done, n - done))) + while (done < n && (x = read((char *)into + done, n - done))) done += x; return done; @@ -225,23 +211,20 @@ IOInput::xread (void *into, IOSize n) @throws All exceptions from #read() are passed through unhandled. Therefore it is possible that an exception is thrown when this function has already read some data. */ -IOSize -IOInput::xreadv (IOBuffer *into, IOSize buffers) -{ +IOSize IOInput::xreadv(IOBuffer *into, IOSize buffers) { // FIXME: Use read(into, buffers) and then sort out in case of // failure, the readv probably succeed directly with much less // overhead. - assert (! buffers || into); + assert(!buffers || into); // Keep reading as long as possible. Let system errors fly // over us, they are a hard error. IOSize x; IOSize done = 0; - for (IOSize i = 0; i < buffers; ++i) - { - done += (x = xread (into [i])); - if (x < into [i].size ()) + for (IOSize i = 0; i < buffers; ++i) { + done += (x = xread(into[i])); + if (x < into[i].size()) break; } return done; diff --git a/Utilities/StorageFactory/src/IOOutput.cc b/Utilities/StorageFactory/src/IOOutput.cc index b116a5f25b77e..4bd4457e5db39 100644 --- a/Utilities/StorageFactory/src/IOOutput.cc +++ b/Utilities/StorageFactory/src/IOOutput.cc @@ -3,8 +3,7 @@ #include /// Destruct the stream. A no-op. -IOOutput::~IOOutput (void) -{} +IOOutput::~IOOutput(void) {} ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// @@ -37,9 +36,7 @@ IOOutput::~IOOutput (void) @throws In case of error, an exception is thrown. However if the stream is in non-blocking mode and cannot accept output, it will @em not throw an exception -- zero will be returned. */ -IOSize -IOOutput::write (unsigned char byte) -{ return write (&byte, 1); } +IOSize IOOutput::write(unsigned char byte) { return write(&byte, 1); } /** Write to the output stream the buffer @a from. This method is simply redirected to #write(const void *, IOSize). @@ -59,9 +56,7 @@ IOOutput::write (unsigned char byte) stream is in non-blocking mode and cannot accept output, it will @em not throw an exception -- the return value will be less than requested. */ -IOSize -IOOutput::write (IOBuffer from) -{ return write (from.data (), from.size ()); } +IOSize IOOutput::write(IOBuffer from) { return write(from.data(), from.size()); } /** Write to the output stream from multiple buffers. There are @a buffers to fill in an array starting at @a from. The buffers are @@ -86,27 +81,21 @@ IOOutput::write (IOBuffer from) stream is in non-blocking mode and cannot accept output, it will @em not throw an exception -- the return value will be less than requested. */ -IOSize -IOOutput::writev (const IOBuffer *from, IOSize buffers) -{ - assert (! buffers || from); +IOSize IOOutput::writev(const IOBuffer *from, IOSize buffers) { + assert(!buffers || from); // Keep writing as long as possible; ignore errors if we have // written something, otherwise pass it on. IOSize x; IOSize done = 0; - try - { - for (IOSize i = 0; i < buffers; ++i) - { - done += (x = write (from [i].data (), from [i].size ())); - if (x < from [i].size ()) - break; + try { + for (IOSize i = 0; i < buffers; ++i) { + done += (x = write(from[i].data(), from[i].size())); + if (x < from[i].size()) + break; } - } - catch (cms::Exception &) - { - if (! done) + } catch (cms::Exception &) { + if (!done) throw; } @@ -142,9 +131,7 @@ IOOutput::writev (const IOBuffer *from, IOSize buffers) @throws All exceptions from #write() are passed through unhandled. Therefore it is possible that an exception is thrown when this function has already written some data. */ -IOSize -IOOutput::xwrite (IOBuffer from) -{ return xwrite (from.data (), from.size ()); } +IOSize IOOutput::xwrite(IOBuffer from) { return xwrite(from.data(), from.size()); } /** Like the corresponding #write() method but writes until the requested number of bytes are written. Writes data from the @@ -172,13 +159,11 @@ IOOutput::xwrite (IOBuffer from) @throws All exceptions from #write() are passed through unhandled. Therefore it is possible that an exception is thrown when this function has already written some data. */ -IOSize -IOOutput::xwrite (const void *from, IOSize n) -{ +IOSize IOOutput::xwrite(const void *from, IOSize n) { // Keep writing until we've written it all. Let errors fly over. IOSize done = 0; while (done < n) - done += write ((const char *) from + done, n - done); + done += write((const char *)from + done, n - done); return done; } @@ -211,18 +196,16 @@ IOOutput::xwrite (const void *from, IOSize n) @throws All exceptions from #write() are passed through unhandled. Therefore it is possible that an exception is thrown when this function has already written some data. */ -IOSize -IOOutput::xwritev (const IOBuffer *from, IOSize buffers) -{ +IOSize IOOutput::xwritev(const IOBuffer *from, IOSize buffers) { // Keep writing until we've written it all. Let errors fly over. // FIXME: Use writev(from, buffers) and then sort out in case of // failure, the writev probably succeed directly with much less // overhead. - assert (! buffers || from); + assert(!buffers || from); IOSize done = 0; for (IOSize i = 0; i < buffers; ++i) - done += xwrite (from [i].data (), from [i].size ()); + done += xwrite(from[i].data(), from[i].size()); return done; } diff --git a/Utilities/StorageFactory/src/LocalCacheFile.cc b/Utilities/StorageFactory/src/LocalCacheFile.cc index 2d7e7310e53f4..35cf296db8ca5 100644 --- a/Utilities/StorageFactory/src/LocalCacheFile.cc +++ b/Utilities/StorageFactory/src/LocalCacheFile.cc @@ -9,26 +9,22 @@ #include #include -static const IOOffset CHUNK_SIZE = 128*1024*1024; +static const IOOffset CHUNK_SIZE = 128 * 1024 * 1024; -static void -nowrite(const std::string &why) -{ +static void nowrite(const std::string &why) { cms::Exception ex("LocalCacheFile"); ex << "Cannot change file but operation '" << why << "' was called"; ex.addContext("LocalCacheFile::" + why + "()"); throw ex; } - LocalCacheFile::LocalCacheFile(std::unique_ptr base, const std::string &tmpdir /* = "" */) - : image_(base->size()), - file_(), - storage_(std::move(base)), - closedFile_(false), - cacheCount_(0), - cacheTotal_((image_ + CHUNK_SIZE - 1) / CHUNK_SIZE) -{ + : image_(base->size()), + file_(), + storage_(std::move(base)), + closedFile_(false), + cacheCount_(0), + cacheTotal_((image_ + CHUNK_SIZE - 1) / CHUNK_SIZE) { present_.resize(cacheTotal_, 0); std::string pattern(tmpdir); @@ -39,13 +35,11 @@ LocalCacheFile::LocalCacheFile(std::unique_ptr base, const std::string pattern = "/tmp"; pattern += "/cmssw-shadow-XXXXXX"; - std::vector temp(pattern.c_str(), pattern.c_str()+pattern.size()+1); + std::vector temp(pattern.c_str(), pattern.c_str() + pattern.size() + 1); int fd = mkstemp(&temp[0]); - if (fd == -1) - { + if (fd == -1) { edm::Exception ex(edm::errors::FileOpenError); - ex << "Cannot create temporary file '" << pattern << "': " - << strerror(errno) << " (error " << errno << ")"; + ex << "Cannot create temporary file '" << pattern << "': " << strerror(errno) << " (error " << errno << ")"; ex.addContext("LocalCacheFile::LocalCacheFile"); } @@ -54,42 +48,31 @@ LocalCacheFile::LocalCacheFile(std::unique_ptr base, const std::string file_->resize(image_); } -LocalCacheFile::~LocalCacheFile(void) -{ -} +LocalCacheFile::~LocalCacheFile(void) {} -void -LocalCacheFile::cache(IOOffset start, IOOffset end) -{ +void LocalCacheFile::cache(IOOffset start, IOOffset end) { start = (start / CHUNK_SIZE) * CHUNK_SIZE; end = std::min(end, image_); IOSize nread = 0; IOSize index = start / CHUNK_SIZE; - while (start < end) - { + while (start < end) { IOSize len = std::min(image_ - start, CHUNK_SIZE); - if (! present_[index]) - { + if (!present_[index]) { void *window = mmap(nullptr, len, PROT_READ | PROT_WRITE, MAP_SHARED, file_->fd(), start); - if (window == MAP_FAILED) - { + if (window == MAP_FAILED) { edm::Exception ex(edm::errors::FileReadError); - ex << "Unable to map a window of local cache file: " - << strerror(errno) << " (error " << errno << ")"; + ex << "Unable to map a window of local cache file: " << strerror(errno) << " (error " << errno << ")"; ex.addContext("LocalCacheFile::cache()"); throw ex; } - try - { + try { nread = storage_->read(window, len, start); - } - catch (cms::Exception &e) - { + } catch (cms::Exception &e) { munmap(window, len); - std::ostringstream ost; + std::ostringstream ost; ost << "Unable to cache " << len << " byte file segment at " << start << ": "; edm::Exception ex(edm::errors::FileReadError, ost.str(), e); ex.addContext("LocalCacheFile::cache()"); @@ -98,19 +81,16 @@ LocalCacheFile::cache(IOOffset start, IOOffset end) munmap(window, len); - if (nread != len) - { + if (nread != len) { edm::Exception ex(edm::errors::FileReadError); - ex << "Unable to cache " << len << " byte file segment at " << start - << ": got only " << nread << " bytes back"; + ex << "Unable to cache " << len << " byte file segment at " << start << ": got only " << nread << " bytes back"; ex.addContext("LocalCacheFile::cache()"); throw ex; } present_[index] = 1; ++cacheCount_; - if (cacheCount_ == cacheTotal_) - { + if (cacheCount_ == cacheTotal_) { storage_->close(); closedFile_ = true; } @@ -121,25 +101,19 @@ LocalCacheFile::cache(IOOffset start, IOOffset end) } } -IOSize -LocalCacheFile::read(void *into, IOSize n) -{ +IOSize LocalCacheFile::read(void *into, IOSize n) { IOOffset here = file_->position(); cache(here, here + n); return file_->read(into, n); } -IOSize -LocalCacheFile::read(void *into, IOSize n, IOOffset pos) -{ +IOSize LocalCacheFile::read(void *into, IOSize n, IOOffset pos) { cache(pos, pos + n); return file_->read(into, n, pos); } -IOSize -LocalCacheFile::readv(IOBuffer *into, IOSize n) -{ +IOSize LocalCacheFile::readv(IOBuffer *into, IOSize n) { IOOffset start = file_->position(); IOOffset end = start; for (IOSize i = 0; i < n; ++i) @@ -149,11 +123,8 @@ LocalCacheFile::readv(IOBuffer *into, IOSize n) return file_->readv(into, n); } -IOSize -LocalCacheFile::readv(IOPosBuffer *into, IOSize n) -{ - for (IOSize i = 0; i < n; ++i) - { +IOSize LocalCacheFile::readv(IOPosBuffer *into, IOSize n) { + for (IOSize i = 0; i < n; ++i) { IOOffset start = into[i].offset(); IOOffset end = start + into[i].size(); cache(start, end); @@ -162,53 +133,45 @@ LocalCacheFile::readv(IOPosBuffer *into, IOSize n) return storage_->readv(into, n); } -IOSize -LocalCacheFile::write(const void */*from*/, IOSize) -{ nowrite("write"); return 0; } +IOSize LocalCacheFile::write(const void * /*from*/, IOSize) { + nowrite("write"); + return 0; +} -IOSize -LocalCacheFile::write(const void */*from*/, IOSize, IOOffset /*pos*/) -{ nowrite("write"); return 0; } +IOSize LocalCacheFile::write(const void * /*from*/, IOSize, IOOffset /*pos*/) { + nowrite("write"); + return 0; +} -IOSize -LocalCacheFile::writev(const IOBuffer */*from*/, IOSize) -{ nowrite("writev"); return 0; } +IOSize LocalCacheFile::writev(const IOBuffer * /*from*/, IOSize) { + nowrite("writev"); + return 0; +} -IOSize -LocalCacheFile::writev(const IOPosBuffer */*from*/, IOSize) -{ nowrite("writev"); return 0; } +IOSize LocalCacheFile::writev(const IOPosBuffer * /*from*/, IOSize) { + nowrite("writev"); + return 0; +} -IOOffset -LocalCacheFile::position(IOOffset offset, Relative whence) -{ return file_->position(offset, whence); } +IOOffset LocalCacheFile::position(IOOffset offset, Relative whence) { return file_->position(offset, whence); } -void -LocalCacheFile::resize(IOOffset /*size*/) -{ nowrite("resize"); } +void LocalCacheFile::resize(IOOffset /*size*/) { nowrite("resize"); } -void -LocalCacheFile::flush(void) -{ nowrite("flush"); } +void LocalCacheFile::flush(void) { nowrite("flush"); } -void -LocalCacheFile::close(void) -{ - if (!closedFile_) - { +void LocalCacheFile::close(void) { + if (!closedFile_) { storage_->close(); } file_->close(); } -bool -LocalCacheFile::prefetch(const IOPosBuffer *what, IOSize n) -{ - for (IOSize i = 0; i < n; ++i) - { +bool LocalCacheFile::prefetch(const IOPosBuffer *what, IOSize n) { + for (IOSize i = 0; i < n; ++i) { IOOffset start = what[i].offset(); IOOffset end = start + what[i].size(); cache(start, end); } - + return file_->prefetch(what, n); } diff --git a/Utilities/StorageFactory/src/LocalFileSystem.cc b/Utilities/StorageFactory/src/LocalFileSystem.cc index 1b351941310f4..fa3fea917dd27 100644 --- a/Utilities/StorageFactory/src/LocalFileSystem.cc +++ b/Utilities/StorageFactory/src/LocalFileSystem.cc @@ -9,33 +9,32 @@ #include #include #if BSD -# include -# include -# include +#include +#include +#include #else -# include -# include +#include +#include #endif #include #include #include #include -#pragma GCC diagnostic ignored "-Wformat" // shut warning on '%z' +#pragma GCC diagnostic ignored "-Wformat" // shut warning on '%z' /// Information about file systems on this node. -struct LocalFileSystem::FSInfo -{ - char *fsname; //< file system name - char *type; //< file system type - char *dir; //< mount point directory - char *origin = nullptr; //< mount origin - dev_t dev; //< device id - long fstype; //< file system id - double freespc; //< free space in megabytes - unsigned local : 1; //< flag for local device - unsigned bind : 1; //< flag for bind mounts - std::atomic checked ; //< flag for valid dev, fstype +struct LocalFileSystem::FSInfo { + char *fsname; //< file system name + char *type; //< file system type + char *dir; //< mount point directory + char *origin = nullptr; //< mount origin + dev_t dev; //< device id + long fstype; //< file system id + double freespc; //< free space in megabytes + unsigned local : 1; //< flag for local device + unsigned bind : 1; //< flag for bind mounts + std::atomic checked; //< flag for valid dev, fstype }; /** Read /proc/filesystems to determine which filesystems are local, @@ -57,76 +56,63 @@ struct LocalFileSystem::FSInfo The exceptions to /proc/filesystems list: lustre and fuse file systems are forced to remote status. Everything else like NFS, AFS, GPFS and various cluster-based systems are already remote. */ -int -LocalFileSystem::readFSTypes(void) -{ +int LocalFileSystem::readFSTypes(void) { int ret = 0; #if __linux__ static const char procfs[] = "/proc/filesystems"; FILE *fs = fopen(procfs, "r"); - if (! fs) - { + if (!fs) { int nerr = errno; edm::LogWarning("LocalFileSystem::readFSTypes()") - << "Cannot read '" << procfs << "': " - << strerror(nerr) << " (error " << nerr << ")"; + << "Cannot read '" << procfs << "': " << strerror(nerr) << " (error " << nerr << ")"; return -1; } ssize_t nread; int line = 0; - while (! feof(fs)) - { + while (!feof(fs)) { char *type = nullptr; char *fstype = nullptr; size_t len = 0; ++line; - if ((nread = getdelim(&type, &len, '\t', fs)) == -1 && ! feof(fs)) - { - fprintf(stderr, "%s:%d: %s (%zd; 1)\n", - procfs, line, strerror(errno), nread); + if ((nread = getdelim(&type, &len, '\t', fs)) == -1 && !feof(fs)) { + fprintf(stderr, "%s:%d: %s (%zd; 1)\n", procfs, line, strerror(errno), nread); free(type); ret = -1; break; } - if ((nread = getdelim(&fstype, &len, '\n', fs)) == -1 && ! feof(fs)) - { - fprintf(stderr, "%s:%d: %s (%zd; 2)\n", - procfs, line, strerror(errno), nread); + if ((nread = getdelim(&fstype, &len, '\n', fs)) == -1 && !feof(fs)) { + fprintf(stderr, "%s:%d: %s (%zd; 2)\n", procfs, line, strerror(errno), nread); free(type); free(fstype); ret = -1; break; } - if (feof (fs)) - { + if (feof(fs)) { free(type); free(fstype); break; } - - if (! strcmp(type, "nodev\t") - || ! strcmp(fstype, "lustre\n") - || ! strncmp(fstype, "fuse", 4)) - { + + if (!strcmp(type, "nodev\t") || !strcmp(fstype, "lustre\n") || !strncmp(fstype, "fuse", 4)) { free(type); free(fstype); continue; } assert(nread >= 1); - fstype[nread-1] = 0; + fstype[nread - 1] = 0; fstypes_.push_back(fstype); free(fstype); free(type); } fclose(fs); -#endif // __linux__ +#endif // __linux__ return ret; } @@ -139,9 +125,7 @@ LocalFileSystem::readFSTypes(void) not yet valid; call statFSInfo() to fill those in. This avoids touching irrelevant filesystems unnecessarily; the file system may not be fully functional, or partially offline, or just very slow. */ -LocalFileSystem::FSInfo * -LocalFileSystem::initFSInfo(void *arg) -{ +LocalFileSystem::FSInfo *LocalFileSystem::initFSInfo(void *arg) { #if BSD struct statfs *m = static_cast(arg); size_t infolen = sizeof(struct FSInfo); @@ -149,8 +133,8 @@ LocalFileSystem::initFSInfo(void *arg) size_t dirlen = strlen(m->f_mntonname) + 1; size_t typelen = strlen(m->f_fstypename) + 1; size_t totlen = infolen + fslen + dirlen + typelen; - FSInfo *i = (FSInfo *) malloc(totlen); - char *p = (char *) i; + FSInfo *i = (FSInfo *)malloc(totlen); + char *p = (char *)i; i->fsname = strncpy(p += infolen, m->f_mntfromname, fslen); i->type = strncpy(p += fslen, m->f_fstypename, typelen); i->dir = strncpy(p += typelen, m->f_mntonname, dirlen); @@ -159,12 +143,11 @@ LocalFileSystem::initFSInfo(void *arg) i->freespc = 0; i->bind = 0; i->origin = nullptr; - if (m->f_bsize > 0) - { + if (m->f_bsize > 0) { i->freespc = m->f_bavail; i->freespc *= m->f_bsize; i->freespc /= 1024. * 1024. * 1024.; - } + } /* FIXME: This incorrectly says that mounted disk images are local, even if it was mounted from a network server. The alternative is to walk up the device tree using either a) process IORegistry to @@ -175,7 +158,7 @@ LocalFileSystem::initFSInfo(void *arg) i->checked = 1; return i; -#else // ! BSD +#else // ! BSD mntent *m = static_cast(arg); size_t infolen = sizeof(struct FSInfo); size_t fslen = strlen(m->mnt_fsname) + 1; @@ -183,12 +166,12 @@ LocalFileSystem::initFSInfo(void *arg) size_t typelen = strlen(m->mnt_type) + 1; size_t originlen = strlen(m->mnt_fsname) + 1; size_t totlen = infolen + fslen + dirlen + typelen + originlen; - FSInfo *i = (FSInfo *) malloc(totlen); - char *p = (char *) i; - i->fsname = static_cast(memcpy(p += infolen, m->mnt_fsname, fslen)); - i->type = static_cast(memcpy(p += fslen, m->mnt_type, typelen)); - i->dir = static_cast(memcpy(p += typelen, m->mnt_dir, dirlen)); - i->origin = static_cast(memcpy(p += dirlen, m->mnt_fsname, originlen)); + FSInfo *i = (FSInfo *)malloc(totlen); + char *p = (char *)i; + i->fsname = static_cast(memcpy(p += infolen, m->mnt_fsname, fslen)); + i->type = static_cast(memcpy(p += fslen, m->mnt_type, typelen)); + i->dir = static_cast(memcpy(p += typelen, m->mnt_dir, dirlen)); + i->origin = static_cast(memcpy(p += dirlen, m->mnt_fsname, originlen)); i->dev = -1; i->fstype = -1; i->freespc = 0; @@ -196,10 +179,10 @@ LocalFileSystem::initFSInfo(void *arg) i->checked = false; i->bind = strstr(m->mnt_opts, "bind") != nullptr; - for (size_t j = 0; j < fstypes_.size() && ! i->local; ++j) + for (size_t j = 0; j < fstypes_.size() && !i->local; ++j) if (fstypes_[j] == i->type) i->local = 1; -#endif // BSD +#endif // BSD return i; } @@ -210,18 +193,14 @@ LocalFileSystem::initFSInfo(void *arg) file systems, and initialises FSInfo structure for them. It does not yet call statFSInfo() on them, so the device and file type ids are not yet complete. */ -int -LocalFileSystem::initFSList(void) -{ +int LocalFileSystem::initFSList(void) { #if BSD int rc; struct statfs *mtab = 0; - if ((rc = getmntinfo(&mtab, MNT_NOWAIT)) < 0) - { + if ((rc = getmntinfo(&mtab, MNT_NOWAIT)) < 0) { int nerr = errno; edm::LogWarning("LocalFileSystem::initFSList()") - << "getmntinfo() failed: " << strerror(nerr) - << " (error " << nerr << ")"; + << "getmntinfo() failed: " << strerror(nerr) << " (error " << nerr << ")"; return -1; } @@ -231,15 +210,13 @@ LocalFileSystem::initFSList(void) free(mtab); #else - const char * const _PATH_MOUNTED_LINUX = "/proc/self/mounts"; + const char *const _PATH_MOUNTED_LINUX = "/proc/self/mounts"; struct mntent *m; FILE *mtab = setmntent(_PATH_MOUNTED_LINUX, "r"); - if (! mtab) - { + if (!mtab) { int nerr = errno; edm::LogWarning("LocalFileSystem::initFSList()") - << "Cannot read '" << _PATH_MOUNTED_LINUX << "': " - << strerror(nerr) << " (error " << nerr << ")"; + << "Cannot read '" << _PATH_MOUNTED_LINUX << "': " << strerror(nerr) << " (error " << nerr << ")"; return -1; } @@ -261,49 +238,39 @@ LocalFileSystem::initFSList(void) This function can be called any number of times. It only does the file system check the first time the function is called. */ -int -LocalFileSystem::statFSInfo(FSInfo *i) const -{ +int LocalFileSystem::statFSInfo(FSInfo *i) const { int ret = 0; struct stat s; struct statfs sfs; - if (! i->checked) - { - if (lstat(i->dir, &s) < 0) - { + if (!i->checked) { + if (lstat(i->dir, &s) < 0) { i->checked = true; int nerr = errno; if (nerr != ENOENT && nerr != EACCES) edm::LogWarning("LocalFileSystem::statFSInfo()") - << "Cannot lstat('" << i->dir << "'): " - << strerror(nerr) << " (error " << nerr << ")"; + << "Cannot lstat('" << i->dir << "'): " << strerror(nerr) << " (error " << nerr << ")"; return -1; } - if (statfs(i->dir, &sfs) < 0) - { + if (statfs(i->dir, &sfs) < 0) { i->checked = true; int nerr = errno; edm::LogWarning("LocalFileSystem::statFSInfo()") - << "Cannot statfs('" << i->dir << "'): " - << strerror(nerr) << " (error " << nerr << ")"; + << "Cannot statfs('" << i->dir << "'): " << strerror(nerr) << " (error " << nerr << ")"; return -1; } i->dev = s.st_dev; i->fstype = sfs.f_type; - if (sfs.f_bsize > 0) - { + if (sfs.f_bsize > 0) { i->freespc = sfs.f_bavail; i->freespc *= sfs.f_bsize; i->freespc /= 1024. * 1024. * 1024.; } i->checked = true; - } - else if (i->fstype == -1) - { + } else if (i->fstype == -1) { errno = ENOENT; ret = -1; } @@ -324,15 +291,13 @@ LocalFileSystem::statFSInfo(FSInfo *i) const any mounted file system (e.g. /dev or /selinux), or if the file system is unavailable or some other way dysfunctional, such as dead nfs mount or filesystem does not implement statfs(). */ -LocalFileSystem::FSInfo * -LocalFileSystem::findMount(const char *path, struct statfs *sfs, struct stat *s, std::vector &prev_paths) const -{ - for (const auto & old_path : prev_paths) - { - if (!strcmp(old_path.c_str(), path)) - { - edm::LogWarning("LocalFileSystem::findMount()") - << "Found a loop in bind mounts; stopping evaluation."; +LocalFileSystem::FSInfo *LocalFileSystem::findMount(const char *path, + struct statfs *sfs, + struct stat *s, + std::vector &prev_paths) const { + for (const auto &old_path : prev_paths) { + if (!strcmp(old_path.c_str(), path)) { + edm::LogWarning("LocalFileSystem::findMount()") << "Found a loop in bind mounts; stopping evaluation."; return nullptr; } } @@ -340,8 +305,7 @@ LocalFileSystem::findMount(const char *path, struct statfs *sfs, struct stat *s, FSInfo *best = nullptr; size_t bestlen = 0; size_t len = strlen(path); - for (size_t i = 0; i < fs_.size(); ++i) - { + for (size_t i = 0; i < fs_.size(); ++i) { // First match simply against the file system path. We don't // touch the file system until the path prefix matches. // When we have a path prefix match, check the file system if @@ -351,24 +315,21 @@ LocalFileSystem::findMount(const char *path, struct statfs *sfs, struct stat *s, // The final condition handles cases such as '/' that can appear twice // in the file system list, once as 'rootfs' and once as local fs. size_t fslen = strlen(fs_[i]->dir); - if (! strncmp(fs_[i]->dir, path, fslen) - && ((fslen == 1 && fs_[i]->dir[0] == '/') - || len == fslen || path[fslen] == '/') - && (! best || fslen > bestlen || (fslen == bestlen && !best->local))) - { + if (!strncmp(fs_[i]->dir, path, fslen) && + ((fslen == 1 && fs_[i]->dir[0] == '/') || len == fslen || path[fslen] == '/') && + (!best || fslen > bestlen || (fslen == bestlen && !best->local))) { // Get the file system device and file system ids. if (statFSInfo(fs_[i]) < 0) - return nullptr; + return nullptr; // Check the path is on the same device / file system. If this // fails, we found a better prefix match on path, but it's the // wrong device, so reset our idea of the best match: it can't // be the outer mount any more. Not sure this is the right // thing to do with e.g. loop-back or union mounts. - if (fs_[i]->dev != s->st_dev || fs_[i]->fstype != sfs->f_type) - { - best = nullptr; - continue; + if (fs_[i]->dev != s->st_dev || fs_[i]->fstype != sfs->f_type) { + best = nullptr; + continue; } // OK this is better than anything else we found so far. @@ -377,33 +338,26 @@ LocalFileSystem::findMount(const char *path, struct statfs *sfs, struct stat *s, } } // In the case of a bind mount, try looking again at the source directory. - if (best && best->bind && best->origin) - { + if (best && best->bind && best->origin) { struct stat s2; struct statfs sfs2; char *fullpath = realpath(best->origin, nullptr); - if (! fullpath) + if (!fullpath) fullpath = strdup(best->origin); - if (lstat(fullpath, &s2) < 0) - { + if (lstat(fullpath, &s2) < 0) { int nerr = errno; - edm::LogWarning("LocalFileSystem::findMount()") - << "Cannot lstat('" << fullpath << "' alias '" - << path << "'): " << strerror(nerr) << " (error " - << nerr << ")"; + edm::LogWarning("LocalFileSystem::findMount()") << "Cannot lstat('" << fullpath << "' alias '" << path + << "'): " << strerror(nerr) << " (error " << nerr << ")"; free(fullpath); return best; } - if (statfs(fullpath, &sfs2) < 0) - { + if (statfs(fullpath, &sfs2) < 0) { int nerr = errno; - edm::LogWarning("LocalFileSystem::findMount()") - << "Cannot statfs('" << fullpath << "' alias '" - << path << "'): " << strerror(nerr) << " (error " - << nerr << ")"; + edm::LogWarning("LocalFileSystem::findMount()") << "Cannot statfs('" << fullpath << "' alias '" << path + << "'): " << strerror(nerr) << " (error " << nerr << ")"; free(fullpath); return best; } @@ -425,34 +379,26 @@ LocalFileSystem::findMount(const char *path, struct statfs *sfs, struct stat *s, Does not throw exceptions. If any errors occur, the errors are reported as message logger warnings but the actual error is swallowed and the function simply returns @c false. */ -bool -LocalFileSystem::isLocalPath(const std::string &path) const -{ +bool LocalFileSystem::isLocalPath(const std::string &path) const { struct stat s; struct statfs sfs; char *fullpath = realpath(path.c_str(), nullptr); - if (! fullpath) + if (!fullpath) fullpath = strdup(path.c_str()); - if (lstat(fullpath, &s) < 0) - { + if (lstat(fullpath, &s) < 0) { int nerr = errno; edm::LogWarning("LocalFileSystem::isLocalPath()") - << "Cannot lstat('" << fullpath << "' alias '" - << path << "'): " << strerror(nerr) << " (error " - << nerr << ")"; + << "Cannot lstat('" << fullpath << "' alias '" << path << "'): " << strerror(nerr) << " (error " << nerr << ")"; free(fullpath); return false; } - - if (statfs(fullpath, &sfs) < 0) - { + + if (statfs(fullpath, &sfs) < 0) { int nerr = errno; - edm::LogWarning("LocalFileSystem::isLocalPath()") - << "Cannot statfs('" << fullpath << "' alias '" - << path << "'): " << strerror(nerr) << " (error " - << nerr << ")"; + edm::LogWarning("LocalFileSystem::isLocalPath()") << "Cannot statfs('" << fullpath << "' alias '" << path + << "'): " << strerror(nerr) << " (error " << nerr << ")"; free(fullpath); return false; } @@ -482,31 +428,27 @@ LocalFileSystem::isLocalPath(const std::string &path) const are reported as message logger warnings but the actual error is swallowed and the directory concerned is skipped. Non-existent and inaccessible directories are silently ignored without warning. */ -std::pair -LocalFileSystem::findCachePath(const std::vector &paths, - double minFreeSpace) const -{ +std::pair LocalFileSystem::findCachePath(const std::vector &paths, + double minFreeSpace) const { struct stat s; struct statfs sfs; std::ostringstream warningst; warningst << "Cannot use lazy-download because:\n"; - for (size_t i = 0, e = paths.size(); i < e; ++i) - { + for (size_t i = 0, e = paths.size(); i < e; ++i) { char *fullpath; const char *inpath = paths[i].c_str(); const char *path = inpath; - if (*path == '$') - { - char *p = getenv(path+1); + if (*path == '$') { + char *p = getenv(path + 1); if (p && *p) - path = p; - else if (! strcmp(path, "$TMPDIR")) - path = "/tmp"; + path = p; + else if (!strcmp(path, "$TMPDIR")) + path = "/tmp"; } - if (! (fullpath = realpath(path, nullptr))) + if (!(fullpath = realpath(path, nullptr))) fullpath = strdup(path); #if 0 @@ -516,25 +458,19 @@ LocalFileSystem::findCachePath(const std::vector &paths, << minFreeSpace << " free space" << std::endl; #endif - if (lstat(fullpath, &s) < 0) - { + if (lstat(fullpath, &s) < 0) { int nerr = errno; if (nerr != ENOENT && nerr != EACCES) - edm::LogWarning("LocalFileSystem::findCachePath()") - << "Cannot lstat('" << fullpath << "', from '" - << inpath << "'): " << strerror(nerr) << " (error " - << nerr << ")"; + edm::LogWarning("LocalFileSystem::findCachePath()") << "Cannot lstat('" << fullpath << "', from '" << inpath + << "'): " << strerror(nerr) << " (error " << nerr << ")"; free(fullpath); continue; } - - if (statfs(fullpath, &sfs) < 0) - { + + if (statfs(fullpath, &sfs) < 0) { int nerr = errno; - edm::LogWarning("LocalFileSystem::findCachePath()") - << "Cannot statfs('" << fullpath << "', from '" - << inpath << "'): " << strerror(nerr) << " (error " - << nerr << ")"; + edm::LogWarning("LocalFileSystem::findCachePath()") << "Cannot statfs('" << fullpath << "', from '" << inpath + << "'): " << strerror(nerr) << " (error " << nerr << ")"; free(fullpath); continue; } @@ -551,27 +487,17 @@ LocalFileSystem::findCachePath(const std::vector &paths, << std::endl; #endif - if (m - && m->local - && m->freespc >= minFreeSpace - && access(fullpath, W_OK) == 0) - { + if (m && m->local && m->freespc >= minFreeSpace && access(fullpath, W_OK) == 0) { std::string result(fullpath); free(fullpath); - return std::make_pair(result,std::string()); - } - else if (m) - { - if (!m->local) - { + return std::make_pair(result, std::string()); + } else if (m) { + if (!m->local) { warningst << "- The mount " << fullpath << " is not local.\n"; - } - else if (m->freespc < minFreeSpace) - { - warningst << " - The mount at " << fullpath << " has only " << m->freespc << " GB free; a minumum of " << minFreeSpace << " GB is required.\n"; - } - else if (access(fullpath, W_OK)) - { + } else if (m->freespc < minFreeSpace) { + warningst << " - The mount at " << fullpath << " has only " << m->freespc << " GB free; a minumum of " + << minFreeSpace << " GB is required.\n"; + } else if (access(fullpath, W_OK)) { warningst << " - The process has no permission to write into " << fullpath << "\n"; } } @@ -580,17 +506,15 @@ LocalFileSystem::findCachePath(const std::vector &paths, } std::string warning_str = warningst.str(); - if (!warning_str.empty()) - { - warning_str = warning_str.substr(0, warning_str.size()-2); + if (!warning_str.empty()) { + warning_str = warning_str.substr(0, warning_str.size() - 2); } - + return std::make_pair(std::string(), std::move(warning_str)); } /** Initialise local file system status. */ -LocalFileSystem::LocalFileSystem(void) -{ +LocalFileSystem::LocalFileSystem(void) { if (readFSTypes() < 0) return; @@ -599,8 +523,7 @@ LocalFileSystem::LocalFileSystem(void) } /** Free local file system status resources. */ -LocalFileSystem::~LocalFileSystem(void) -{ +LocalFileSystem::~LocalFileSystem(void) { for (size_t i = 0, e = fs_.size(); i < e; ++i) free(fs_[i]); } diff --git a/Utilities/StorageFactory/src/RemoteFile.cc b/Utilities/StorageFactory/src/RemoteFile.cc index dc6fe122d500b..797c4ed2cab70 100644 --- a/Utilities/StorageFactory/src/RemoteFile.cc +++ b/Utilities/StorageFactory/src/RemoteFile.cc @@ -10,22 +10,19 @@ #include #include #if __APPLE__ -# include -# define environ (*_NSGetEnviron()) +#include +#define environ (*_NSGetEnviron()) #endif -static std::string -join (char **cmd) -{ +static std::string join(char **cmd) { size_t size = 0; for (char **p = cmd; p && p[0]; ++p) size += 1 + strlen(*p); std::string result; - result.reserve (size); + result.reserve(size); - for (char **p = cmd; p && p[0]; ++p) - { + for (char **p = cmd; p && p[0]; ++p) { if (p != cmd) result += ' '; result += *p; @@ -34,104 +31,89 @@ join (char **cmd) return result; } -RemoteFile::RemoteFile (IOFD fd, const std::string &name) - : File (fd), - name_ (name) -{} +RemoteFile::RemoteFile(IOFD fd, const std::string &name) : File(fd), name_(name) {} -void -RemoteFile::remove (void) -{ unlink (name_.c_str()); } +void RemoteFile::remove(void) { unlink(name_.c_str()); } -void -RemoteFile::close (void) -{ remove(); File::close (); } +void RemoteFile::close(void) { + remove(); + File::close(); +} -void -RemoteFile::abort (void) -{ remove(); File::abort (); } +void RemoteFile::abort(void) { + remove(); + File::abort(); +} -int -RemoteFile::local (const std::string &tmpdir, std::string &temp) -{ +int RemoteFile::local(const std::string &tmpdir, std::string &temp) { // Download temporary files to the current directory by default. // This is better for grid jobs as the current directory is // likely to have more space, and is more optimised for // large files, and is cleaned up after the job. - if (tmpdir.empty () || tmpdir == ".") - { - size_t len = pathconf (".", _PC_PATH_MAX); - char *buf = (char *) malloc (len); - getcwd (buf, len); + if (tmpdir.empty() || tmpdir == ".") { + size_t len = pathconf(".", _PC_PATH_MAX); + char *buf = (char *)malloc(len); + getcwd(buf, len); - temp.reserve (len + 32); + temp.reserve(len + 32); temp = buf; - free (buf); - } - else - { - temp.reserve (tmpdir.size() + 32); + free(buf); + } else { + temp.reserve(tmpdir.size() + 32); temp = tmpdir; } - if (temp[temp.size()-1] != '/') + if (temp[temp.size() - 1] != '/') temp += '/'; temp += "storage-factory-local-XXXXXX"; - temp.c_str(); // null terminate for mkstemp + temp.c_str(); // null terminate for mkstemp - int fd = mkstemp (&temp[0]); + int fd = mkstemp(&temp[0]); if (fd == -1) throwStorageError("RemoteFile", "Calling RemoteFile::local()", "mkstemp()", errno); return fd; } -std::unique_ptr -RemoteFile::get (int localfd, const std::string &name, char **cmd, int mode) -{ +std::unique_ptr RemoteFile::get(int localfd, const std::string &name, char **cmd, int mode) { // FIXME: On write, create a temporary local file open for write; // on close, trigger transfer to destination. If opening existing // file for write, may need to first download. - assert (! (mode & (IOFlags::OpenWrite | IOFlags::OpenCreate))); + assert(!(mode & (IOFlags::OpenWrite | IOFlags::OpenCreate))); - pid_t pid = -1; - int rc = posix_spawnp (&pid, cmd[0], nullptr, nullptr, cmd, environ); + pid_t pid = -1; + int rc = posix_spawnp(&pid, cmd[0], nullptr, nullptr, cmd, environ); - if (rc == -1) - { + if (rc == -1) { int errsave = errno; - ::close (localfd); - unlink (name.c_str()); - throwStorageError ("RemoteFile", "Calling RemoteFile::get()", "posix_spawnp()", errsave); + ::close(localfd); + unlink(name.c_str()); + throwStorageError("RemoteFile", "Calling RemoteFile::get()", "posix_spawnp()", errsave); } pid_t rcpid; do rcpid = waitpid(pid, &rc, 0); - while (rcpid == (pid_t) -1 && errno == EINTR); + while (rcpid == (pid_t)-1 && errno == EINTR); - if (rcpid == (pid_t) -1) - { + if (rcpid == (pid_t)-1) { int errsave = errno; - ::close (localfd); - unlink (name.c_str()); - throwStorageError ("RemoteFile", "Calling RemoteFile::get()", "waitpid()", errsave); + ::close(localfd); + unlink(name.c_str()); + throwStorageError("RemoteFile", "Calling RemoteFile::get()", "waitpid()", errsave); } if (WIFEXITED(rc) && WEXITSTATUS(rc) == 0) - return std::unique_ptr ( static_cast( new RemoteFile(localfd, name) ) ); - else - { - ::close (localfd); - unlink (name.c_str()); + return std::unique_ptr(static_cast(new RemoteFile(localfd, name))); + else { + ::close(localfd); + unlink(name.c_str()); cms::Exception ex("RemoteFile"); ex << "'" << join(cmd) << "'" - << (WIFEXITED(rc) ? " exited with exit code " - : WIFSIGNALED(rc) ? " died from signal " - : " died for an obscure unknown reason with exit status ") - << (WIFEXITED(rc) ? WEXITSTATUS(rc) - : WIFSIGNALED(rc) ? WTERMSIG(rc) - : rc); + << (WIFEXITED(rc) + ? " exited with exit code " + : WIFSIGNALED(rc) ? " died from signal " : " died for an obscure unknown reason with exit status ") + << (WIFEXITED(rc) ? WEXITSTATUS(rc) : WIFSIGNALED(rc) ? WTERMSIG(rc) : rc); ex.addContext("Calling RemoteFile::get()"); throw ex; } diff --git a/Utilities/StorageFactory/src/StatisticsSenderService.cc b/Utilities/StorageFactory/src/StatisticsSenderService.cc index 2c1f8f4f86f75..d69164bd8b01d 100644 --- a/Utilities/StorageFactory/src/StatisticsSenderService.cc +++ b/Utilities/StorageFactory/src/StatisticsSenderService.cc @@ -15,12 +15,11 @@ #include #include -#define UPDATE_STATISTIC(x) \ - m_ ## x = x; +#define UPDATE_STATISTIC(x) m_##x = x; -#define UPDATE_AND_OUTPUT_STATISTIC(x) \ - os << "\"" #x "\":" << (x-m_ ## x) << ", "; \ - UPDATE_STATISTIC(x) +#define UPDATE_AND_OUTPUT_STATISTIC(x) \ + os << "\"" #x "\":" << (x - m_##x) << ", "; \ + UPDATE_STATISTIC(x) // Simple hack to define HOST_NAME_MAX on Mac. // Allows arrays to be statically allocated @@ -33,20 +32,18 @@ using namespace edm::storage; -StatisticsSenderService::FileStatistics::FileStatistics() : - m_read_single_operations(0), - m_read_single_bytes(0), - m_read_single_square(0), - m_read_vector_operations(0), - m_read_vector_bytes(0), - m_read_vector_square(0), - m_read_vector_count_sum(0), - m_read_vector_count_square(0), - m_start_time(time(nullptr)) -{} - -void -StatisticsSenderService::FileStatistics::fillUDP(std::ostringstream &os) { +StatisticsSenderService::FileStatistics::FileStatistics() + : m_read_single_operations(0), + m_read_single_bytes(0), + m_read_single_square(0), + m_read_vector_operations(0), + m_read_vector_bytes(0), + m_read_vector_square(0), + m_read_vector_count_sum(0), + m_read_vector_count_square(0), + m_start_time(time(nullptr)) {} + +void StatisticsSenderService::FileStatistics::fillUDP(std::ostringstream &os) { const StorageAccount::StorageStats &stats = StorageAccount::summary(); ssize_t read_single_operations = 0; ssize_t read_single_bytes = 0; @@ -57,7 +54,7 @@ StatisticsSenderService::FileStatistics::fillUDP(std::ostringstream &os) { ssize_t read_vector_count_sum = 0; ssize_t read_vector_count_square = 0; auto token = StorageAccount::tokenForStorageClassName("tstoragefile"); - for (StorageAccount::StorageStats::const_iterator i = stats.begin (); i != stats.end(); ++i) { + for (StorageAccount::StorageStats::const_iterator i = stats.begin(); i != stats.end(); ++i) { if (i->first == token.value()) { continue; } @@ -77,27 +74,43 @@ StatisticsSenderService::FileStatistics::fillUDP(std::ostringstream &os) { } int64_t single_op_count = read_single_operations - m_read_single_operations; if (single_op_count > 0) { - double single_sum = read_single_bytes-m_read_single_bytes; - double single_average = single_sum/static_cast(single_op_count); - os << "\"read_single_sigma\":" << sqrt((static_cast(read_single_square-m_read_single_square) - single_average*single_average*single_op_count)/static_cast(single_op_count)) << ", "; + double single_sum = read_single_bytes - m_read_single_bytes; + double single_average = single_sum / static_cast(single_op_count); + os << "\"read_single_sigma\":" + << sqrt((static_cast(read_single_square - m_read_single_square) - + single_average * single_average * single_op_count) / + static_cast(single_op_count)) + << ", "; os << "\"read_single_average\":" << single_average << ", "; } m_read_single_square = read_single_square; int64_t vector_op_count = read_vector_operations - m_read_vector_operations; if (vector_op_count > 0) { - double vector_average = static_cast(read_vector_bytes-m_read_vector_bytes)/static_cast(vector_op_count); + double vector_average = + static_cast(read_vector_bytes - m_read_vector_bytes) / static_cast(vector_op_count); os << "\"read_vector_average\":" << vector_average << ", "; - os << "\"read_vector_sigma\":" << sqrt((static_cast(read_vector_square-m_read_vector_square) - vector_average*vector_average*vector_op_count)/static_cast(vector_op_count)) << ", "; - double vector_count_average = static_cast(read_vector_count_sum-m_read_vector_count_sum)/static_cast(vector_op_count); + os << "\"read_vector_sigma\":" + << sqrt((static_cast(read_vector_square - m_read_vector_square) - + vector_average * vector_average * vector_op_count) / + static_cast(vector_op_count)) + << ", "; + double vector_count_average = + static_cast(read_vector_count_sum - m_read_vector_count_sum) / static_cast(vector_op_count); os << "\"read_vector_count_average\":" << vector_count_average << ", "; - os << "\"read_vector_count_sigma\":" << sqrt((static_cast(read_vector_count_square-m_read_vector_count_square) - vector_count_average*vector_count_average*vector_op_count)/static_cast(vector_op_count)) << ", "; + os << "\"read_vector_count_sigma\":" + << sqrt((static_cast(read_vector_count_square - m_read_vector_count_square) - + vector_count_average * vector_count_average * vector_op_count) / + static_cast(vector_op_count)) + << ", "; } m_read_vector_square = read_vector_square; m_read_vector_count_square = read_vector_count_square; m_read_vector_count_sum = read_vector_count_sum; - os << "\"read_bytes\":" << (read_vector_bytes + read_single_bytes - m_read_vector_bytes - m_read_single_bytes) << ", "; - os << "\"read_bytes_at_close\":" << (read_vector_bytes + read_single_bytes - m_read_vector_bytes - m_read_single_bytes) << ", "; + os << "\"read_bytes\":" << (read_vector_bytes + read_single_bytes - m_read_vector_bytes - m_read_single_bytes) + << ", "; + os << "\"read_bytes_at_close\":" + << (read_vector_bytes + read_single_bytes - m_read_vector_bytes - m_read_single_bytes) << ", "; // See top of file for macros; not complex, just avoiding copy/paste UPDATE_AND_OUTPUT_STATISTIC(read_single_operations) @@ -111,18 +124,17 @@ StatisticsSenderService::FileStatistics::fillUDP(std::ostringstream &os) { os << "\"end_time\":" << m_start_time; } -StatisticsSenderService::StatisticsSenderService(edm::ParameterSet const& /*pset*/, edm::ActivityRegistry& ar) : - m_clienthost("unknown"), - m_clientdomain("unknown"), - m_serverhost("unknown"), - m_serverdomain("unknown"), - m_filelfn("unknown"), - m_filestats(), - m_guid(Guid().toString()), - m_counter(0), - m_size(-1), - m_userdn("unknown") -{ +StatisticsSenderService::StatisticsSenderService(edm::ParameterSet const & /*pset*/, edm::ActivityRegistry &ar) + : m_clienthost("unknown"), + m_clientdomain("unknown"), + m_serverhost("unknown"), + m_serverdomain("unknown"), + m_filelfn("unknown"), + m_filestats(), + m_guid(Guid().toString()), + m_counter(0), + m_size(-1), + m_userdn("unknown") { determineHostnames(); ar.watchPreCloseFile(this, &StatisticsSenderService::filePreCloseEvent); if (!getX509Subject(m_userdn)) { @@ -130,15 +142,13 @@ StatisticsSenderService::StatisticsSenderService(edm::ParameterSet const& /*pset } } -const char * -StatisticsSenderService::getJobID() { - const char * id = getenv(JOB_UNIQUE_ID_ENV); +const char *StatisticsSenderService::getJobID() { + const char *id = getenv(JOB_UNIQUE_ID_ENV); // Dashboard developers requested that we migrate to this environment variable. return id ? id : getenv(JOB_UNIQUE_ID_ENV_V2); } -void -StatisticsSenderService::setCurrentServer(const std::string &servername) { +void StatisticsSenderService::setCurrentServer(const std::string &servername) { size_t dot_pos = servername.find("."); std::string serverhost; std::string serverdomain; @@ -147,7 +157,7 @@ StatisticsSenderService::setCurrentServer(const std::string &servername) { serverdomain = "unknown"; } else { serverhost = servername.substr(0, dot_pos); - serverdomain = servername.substr(dot_pos+1, servername.find(":")-dot_pos-1); + serverdomain = servername.substr(dot_pos + 1, servername.find(":") - dot_pos - 1); if (serverdomain.empty()) { serverdomain = "unknown"; } @@ -159,13 +169,9 @@ StatisticsSenderService::setCurrentServer(const std::string &servername) { } } -void -StatisticsSenderService::setSize(size_t size) { - m_size = size; -} +void StatisticsSenderService::setSize(size_t size) { m_size = size; } -void -StatisticsSenderService::filePreCloseEvent(std::string const& lfn, bool usedFallback) { +void StatisticsSenderService::filePreCloseEvent(std::string const &lfn, bool usedFallback) { m_filelfn = lfn; edm::Service pSLC; @@ -173,17 +179,14 @@ StatisticsSenderService::filePreCloseEvent(std::string const& lfn, bool usedFall return; } - const struct addrinfo * addresses = pSLC->statisticsDestination(); + const struct addrinfo *addresses = pSLC->statisticsDestination(); if (!addresses) { return; } - std::set const * info = pSLC->statisticsInfo(); - if (info && !info->empty() && (m_userdn != "unknown") && ( - (info->find("dn") == info->end()) || - (info->find("nodn") != info->end())) - ) - { + std::set const *info = pSLC->statisticsInfo(); + if (info && !info->empty() && (m_userdn != "unknown") && + ((info->find("dn") == info->end()) || (info->find("nodn") != info->end()))) { m_userdn = "not reported"; } @@ -195,18 +198,17 @@ StatisticsSenderService::filePreCloseEvent(std::string const& lfn, bool usedFall if (sock < 0) { continue; } - auto close_del = [](int* iSocket) { close(*iSocket); }; - std::unique_ptr guard(&sock, close_del); + auto close_del = [](int *iSocket) { close(*iSocket); }; + std::unique_ptr guard(&sock, close_del); if (sendto(sock, results.c_str(), results.size(), 0, address->ai_addr, address->ai_addrlen) >= 0) { - break; + break; } } m_counter++; } -void -StatisticsSenderService::determineHostnames(void) { +void StatisticsSenderService::determineHostnames(void) { char tmpName[HOST_NAME_MAX]; if (gethostname(tmpName, HOST_NAME_MAX) != 0) { // Sigh, no way to log errors from here. @@ -218,13 +220,12 @@ StatisticsSenderService::determineHostnames(void) { if (dot_pos == std::string::npos) { m_clientdomain = "unknown"; } else { - m_clientdomain = m_clienthost.substr(dot_pos+1, m_clienthost.size()-dot_pos-1); + m_clientdomain = m_clienthost.substr(dot_pos + 1, m_clienthost.size() - dot_pos - 1); m_clienthost = m_clienthost.substr(0, dot_pos); } } -void -StatisticsSenderService::fillUDP(const std::string& siteName, bool usedFallback, std::string &udpinfo) { +void StatisticsSenderService::fillUDP(const std::string &siteName, bool usedFallback, std::string &udpinfo) { std::ostringstream os; // Header - same for all IO accesses @@ -242,7 +243,7 @@ StatisticsSenderService::fillUDP(const std::string& siteName, bool usedFallback, serverhost = m_serverhost; serverdomain = m_serverdomain; } - + os << "\"user_dn\":\"" << m_userdn << "\", "; os << "\"client_host\":\"" << m_clienthost << "\", "; os << "\"client_domain\":\"" << m_clientdomain << "\", "; @@ -252,7 +253,7 @@ StatisticsSenderService::fillUDP(const std::string& siteName, bool usedFallback, os << "\"file_lfn\":\"" << m_filelfn << "\", "; // Dashboard devs requested that we send out no app_info if a job ID // is not present in the environment. - const char * jobId = getJobID(); + const char *jobId = getJobID(); if (jobId) { os << "\"app_info\":\"" << jobId << "\", "; } @@ -283,17 +284,17 @@ StatisticsSenderService::fillUDP(const std::string& siteName, bool usedFallback, * THIS DOES NOT VERIFY THE RESULTS, and is a best-effort GUESS. * Again, DO NOT REUSE THIS CODE THINKING IT VERIFIES THE CHAIN! */ -static X509 * findEEC(STACK_OF(X509) * certstack) { +static X509 *findEEC(STACK_OF(X509) * certstack) { int depth = sk_X509_num(certstack); if (depth == 0) { return nullptr; } - int idx = depth-1; + int idx = depth - 1; char *priorsubject = nullptr; char *subject = nullptr; X509 *x509cert = sk_X509_value(certstack, idx); - for (; x509cert && idx>0; idx--) { - subject = X509_NAME_oneline(X509_get_subject_name(x509cert),nullptr,0); + for (; x509cert && idx > 0; idx--) { + subject = X509_NAME_oneline(X509_get_subject_name(x509cert), nullptr, 0); if (subject && priorsubject && (strncmp(subject, priorsubject, strlen(subject)) != 0)) { break; } @@ -310,8 +311,7 @@ static X509 * findEEC(STACK_OF(X509) * certstack) { return x509cert; } -static bool -getX509SubjectFromFile(const std::string &filename, std::string &result) { +static bool getX509SubjectFromFile(const std::string &filename, std::string &result) { BIO *biof = nullptr; STACK_OF(X509) *certs = nullptr; char *subject = nullptr; @@ -320,34 +320,42 @@ getX509SubjectFromFile(const std::string &filename, std::string &result) { char *name = nullptr; long len = 0U; - if((biof = BIO_new_file(filename.c_str(), "r"))) { - + if ((biof = BIO_new_file(filename.c_str(), "r"))) { certs = sk_X509_new_null(); bool encountered_error = false; while ((!encountered_error) && (!BIO_eof(biof)) && PEM_read_bio(biof, &name, &header, &data, &len)) { if (strcmp(name, PEM_STRING_X509) == 0 || strcmp(name, PEM_STRING_X509_OLD) == 0) { - X509 * tmp_cert = nullptr; + X509 *tmp_cert = nullptr; // See WARNINGS section in http://www.openssl.org/docs/crypto/d2i_X509.html // Without this cmsRun crashes on a mac with a valid grid proxy. const unsigned char *p; - p=data; + p = data; tmp_cert = d2i_X509(&tmp_cert, &p, len); if (tmp_cert) { sk_X509_push(certs, tmp_cert); } else { encountered_error = true; } - } // Note we ignore any proxy key in the file. - if (data) { OPENSSL_free(data); data = nullptr;} - if (header) { OPENSSL_free(header); header = nullptr;} - if (name) { OPENSSL_free(name); name = nullptr;} + } // Note we ignore any proxy key in the file. + if (data) { + OPENSSL_free(data); + data = nullptr; + } + if (header) { + OPENSSL_free(header); + header = nullptr; + } + if (name) { + OPENSSL_free(name); + name = nullptr; + } } X509 *x509cert = nullptr; if (!encountered_error && sk_X509_num(certs)) { x509cert = findEEC(certs); } if (x509cert) { - subject = X509_NAME_oneline(X509_get_subject_name(x509cert),nullptr,0); + subject = X509_NAME_oneline(X509_get_subject_name(x509cert), nullptr, 0); } // Note we do not free x509cert directly, as it's still owned by the certs stack. if (certs) { @@ -358,14 +366,13 @@ getX509SubjectFromFile(const std::string &filename, std::string &result) { if (subject) { result = subject; OPENSSL_free(subject); - return true; + return true; } } return false; } -bool -StatisticsSenderService::getX509Subject(std::string &result) { +bool StatisticsSenderService::getX509Subject(std::string &result) { char *filename = getenv("X509_USER_PROXY"); if (filename && getX509SubjectFromFile(filename, result)) { return true; diff --git a/Utilities/StorageFactory/src/Storage.cc b/Utilities/StorageFactory/src/Storage.cc index 04a16aad1585f..3e250a5e30867 100644 --- a/Utilities/StorageFactory/src/Storage.cc +++ b/Utilities/StorageFactory/src/Storage.cc @@ -2,47 +2,35 @@ #include "FWCore/Utilities/interface/Exception.h" #include -Storage::Storage (void) -{} +Storage::Storage(void) {} -Storage::~Storage (void) -{} +Storage::~Storage(void) {} ////////////////////////////////////////////////////////////////////// -IOSize -Storage::read (IOBuffer into, IOOffset pos) -{ return read (into.data (), into.size (), pos); } +IOSize Storage::read(IOBuffer into, IOOffset pos) { return read(into.data(), into.size(), pos); } -IOSize -Storage::read (void *into, IOSize n, IOOffset pos) -{ +IOSize Storage::read(void *into, IOSize n, IOOffset pos) { // FIXME: this is not thread safe! split into separate interface // that a particular storage can choose to support or not? make // sure that throw semantics are correct here! // FIXME: use saveposition object in case exceptions are thrown? - IOOffset here = position (); - position (pos); - n = read (into, n); - position (here); + IOOffset here = position(); + position(pos); + n = read(into, n); + position(here); return n; } -IOSize -Storage::readv (IOPosBuffer *into, IOSize n) -{ +IOSize Storage::readv(IOPosBuffer *into, IOSize n) { IOOffset here = position(); IOSize total = 0; - for (IOSize i = 0; i < n; ++i) - { - try - { + for (IOSize i = 0; i < n; ++i) { + try { position(into[i].offset()); total += read(into[i].data(), into[i].size()); - } - catch (cms::Exception &) - { - if (! total) - throw; + } catch (cms::Exception &) { + if (!total) + throw; break; } } @@ -51,39 +39,29 @@ Storage::readv (IOPosBuffer *into, IOSize n) } ////////////////////////////////////////////////////////////////////// -IOSize -Storage::write (IOBuffer from, IOOffset pos) -{ return write (from.data (), from.size (), pos); } +IOSize Storage::write(IOBuffer from, IOOffset pos) { return write(from.data(), from.size(), pos); } -IOSize -Storage::write (const void *from, IOSize n, IOOffset pos) -{ +IOSize Storage::write(const void *from, IOSize n, IOOffset pos) { // FIXME: this is not thread safe! split into separate interface // that a particular storage can choose to support or not? make // sure that throw semantics are correct here! // FIXME: use saveposition object in case exceptions are thrown? - IOOffset here = position (); - position (pos); - n = write (from, n); - position (here); + IOOffset here = position(); + position(pos); + n = write(from, n); + position(here); return n; } -IOSize -Storage::writev (const IOPosBuffer *from, IOSize n) -{ +IOSize Storage::writev(const IOPosBuffer *from, IOSize n) { IOSize total = 0; - for (IOSize i = 0; i < n; ++i) - { - try - { + for (IOSize i = 0; i < n; ++i) { + try { total += write(from[i].data(), from[i].size(), from[i].offset()); - } - catch (cms::Exception &) - { - if (! total) - throw; + } catch (cms::Exception &) { + if (!total) + throw; break; } } @@ -91,44 +69,30 @@ Storage::writev (const IOPosBuffer *from, IOSize n) } ////////////////////////////////////////////////////////////////////// -IOOffset -Storage::position (void) const -{ - Storage *self = const_cast (this); - return self->position (0, CURRENT); +IOOffset Storage::position(void) const { + Storage *self = const_cast(this); + return self->position(0, CURRENT); } -IOOffset -Storage::size (void) const -{ +IOOffset Storage::size(void) const { // FIXME: use saveposition object in case exceptions are thrown? - Storage *self = const_cast (this); - IOOffset here = position (); - self->position (0, END); - IOOffset size = position (); - self->position (here); // FIXME: VERIFY()? + Storage *self = const_cast(this); + IOOffset here = position(); + self->position(0, END); + IOOffset size = position(); + self->position(here); // FIXME: VERIFY()? return size; } -void -Storage::rewind (void) -{ position(0); } +void Storage::rewind(void) { position(0); } ////////////////////////////////////////////////////////////////////// -bool -Storage::prefetch (const IOPosBuffer * /* what */, IOSize /* n */) -{ return false; } +bool Storage::prefetch(const IOPosBuffer * /* what */, IOSize /* n */) { return false; } ////////////////////////////////////////////////////////////////////// -void -Storage::flush (void) -{} +void Storage::flush(void) {} -void -Storage::close (void) -{} +void Storage::close(void) {} ////////////////////////////////////////////////////////////////////// -bool -Storage::eof (void) const -{ return position () == size (); } +bool Storage::eof(void) const { return position() == size(); } diff --git a/Utilities/StorageFactory/src/StorageAccount.cc b/Utilities/StorageFactory/src/StorageAccount.cc index 3c3eaebb42161..cdafbb2236016 100644 --- a/Utilities/StorageFactory/src/StorageAccount.cc +++ b/Utilities/StorageFactory/src/StorageAccount.cc @@ -6,35 +6,16 @@ #include namespace { - char const * const kOperationNames[] = { - "check", - "close", - "construct", - "destruct", - "flush", - "open", - "position", - "prefetch", - "read", - "readActual", - "readAsync", - "readPrefetchToCache", - "readViaCache", - "readv", - "resize", - "seek", - "stagein", - "stat", - "write", - "writeActual", - "writeViaCache", - "writev" - }; - + char const* const kOperationNames[] = { + "check", "close", "construct", "destruct", "flush", "open", + "position", "prefetch", "read", "readActual", "readAsync", "readPrefetchToCache", + "readViaCache", "readv", "resize", "seek", "stagein", "stat", + "write", "writeActual", "writeViaCache", "writev"}; + //Storage class names to the value of the token to which they are assigned tbb::concurrent_unordered_map s_nameToToken; std::atomic s_nextTokenValue{0}; -} +} // namespace StorageAccount::StorageStats StorageAccount::m_stats; @@ -54,20 +35,20 @@ inline char const* StorageAccount::operationName(Operation operation) { return kOperationNames[static_cast(operation)]; } -StorageAccount::StorageClassToken StorageAccount::tokenForStorageClassName( std::string const & iName) { +StorageAccount::StorageClassToken StorageAccount::tokenForStorageClassName(std::string const& iName) { auto itFound = s_nameToToken.find(iName); - if( itFound != s_nameToToken.end()) { + if (itFound != s_nameToToken.end()) { return StorageClassToken(itFound->second); } int value = s_nextTokenValue++; - + s_nameToToken.insert(std::make_pair(iName, value)); - + return StorageClassToken(value); } -const std::string& StorageAccount::nameForToken( StorageClassToken iToken) { - for( auto it = s_nameToToken.begin(), itEnd = s_nameToToken.end(); it != itEnd; ++it) { +const std::string& StorageAccount::nameForToken(StorageClassToken iToken) { + for (auto it = s_nameToToken.begin(), itEnd = s_nameToToken.end(); it != itEnd; ++it) { if (it->second == iToken.value()) { return it->first; } @@ -75,41 +56,35 @@ const std::string& StorageAccount::nameForToken( StorageClassToken iToken) { assert(false); } - -std::string -StorageAccount::summaryText (bool banner /*=false*/) { +std::string StorageAccount::summaryText(bool banner /*=false*/) { bool first = true; std::ostringstream os; if (banner) os << "stats: class/operation/attempts/successes/amount/time-total/time-min/time-max\n"; - for (auto i = s_nameToToken.begin (); i != s_nameToToken.end(); ++i) { + for (auto i = s_nameToToken.begin(); i != s_nameToToken.end(); ++i) { auto const& opStats = m_stats[i->second]; - for (auto j = opStats.begin (); j != opStats.end (); ++j, first = false) - os << (first ? "" : "; ") - << (i->first) << '/' - << kOperationNames[j->first] << '=' - << j->second.attempts << '/' - << j->second.successes << '/' - << (static_cast(j->second.amount) / 1024 / 1024) << "MB/" + for (auto j = opStats.begin(); j != opStats.end(); ++j, first = false) + os << (first ? "" : "; ") << (i->first) << '/' << kOperationNames[j->first] << '=' << j->second.attempts << '/' + << j->second.successes << '/' << (static_cast(j->second.amount) / 1024 / 1024) << "MB/" << (static_cast(j->second.timeTotal) / 1000 / 1000) << "ms/" << (static_cast(j->second.timeMin) / 1000 / 1000) << "ms/" << (static_cast(j->second.timeMax) / 1000 / 1000) << "ms"; } - return os.str (); + return os.str(); } -void -StorageAccount::fillSummary(std::map& summary) { +void StorageAccount::fillSummary(std::map& summary) { int const oneM = 1000 * 1000; int const oneMeg = 1024 * 1024; - for (auto i = s_nameToToken.begin (); i != s_nameToToken.end(); ++i) { + for (auto i = s_nameToToken.begin(); i != s_nameToToken.end(); ++i) { auto const& opStats = m_stats[i->second]; for (auto j = opStats.begin(); j != opStats.end(); ++j) { std::ostringstream os; os << "Timing-" << i->first << "-" << kOperationNames[j->first] << "-"; summary.insert(std::make_pair(os.str() + "numOperations", i2str(j->second.attempts))); summary.insert(std::make_pair(os.str() + "numSuccessfulOperations", i2str(j->second.successes))); - summary.insert(std::make_pair(os.str() + "totalMegabytes", d2str(static_cast(j->second.amount) / oneMeg))); + summary.insert( + std::make_pair(os.str() + "totalMegabytes", d2str(static_cast(j->second.amount) / oneMeg))); summary.insert(std::make_pair(os.str() + "totalMsecs", d2str(static_cast(j->second.timeTotal) / oneM))); summary.insert(std::make_pair(os.str() + "minMsecs", d2str(static_cast(j->second.timeMin) / oneM))); summary.insert(std::make_pair(os.str() + "maxMsecs", d2str(static_cast(j->second.timeMax) / oneM))); @@ -117,35 +92,28 @@ StorageAccount::fillSummary(std::map& summary) { } } -const StorageAccount::StorageStats& -StorageAccount::summary (void) -{ return m_stats; } +const StorageAccount::StorageStats& StorageAccount::summary(void) { return m_stats; } -StorageAccount::Counter& -StorageAccount::counter (StorageClassToken token, Operation operation) { - auto &opstats = m_stats [token.value()]; +StorageAccount::Counter& StorageAccount::counter(StorageClassToken token, Operation operation) { + auto& opstats = m_stats[token.value()]; return opstats[static_cast(operation)]; } -StorageAccount::Stamp::Stamp (Counter &counter) - : m_counter (counter), - m_start (std::chrono::high_resolution_clock::now()) -{ +StorageAccount::Stamp::Stamp(Counter& counter) + : m_counter(counter), m_start(std::chrono::high_resolution_clock::now()) { m_counter.attempts++; } -void -StorageAccount::Stamp::tick (uint64_t amount, int64_t count) const -{ +void StorageAccount::Stamp::tick(uint64_t amount, int64_t count) const { std::chrono::nanoseconds elapsed_ns = std::chrono::high_resolution_clock::now() - m_start; uint64_t elapsed = elapsed_ns.count(); m_counter.successes++; m_counter.vector_count += count; - m_counter.vector_square += count*count; + m_counter.vector_square += count * count; m_counter.amount += amount; - Counter::addTo(m_counter.amount_square, amount*amount); + Counter::addTo(m_counter.amount_square, amount * amount); Counter::addTo(m_counter.timeTotal, elapsed); if (elapsed < m_counter.timeMin || m_counter.successes == 1) diff --git a/Utilities/StorageFactory/src/StorageAccountProxy.cc b/Utilities/StorageFactory/src/StorageAccountProxy.cc index 5c632284d1815..51be35fe4e613 100644 --- a/Utilities/StorageFactory/src/StorageAccountProxy.cc +++ b/Utilities/StorageFactory/src/StorageAccountProxy.cc @@ -1,143 +1,113 @@ #include "Utilities/StorageFactory/interface/StorageAccountProxy.h" -StorageAccountProxy::StorageAccountProxy (const std::string &storageClass, - std::unique_ptr baseStorage) - : m_baseStorage (std::move(baseStorage)), - m_token(StorageAccount::tokenForStorageClassName(storageClass)), - m_statsRead (StorageAccount::counter (m_token, StorageAccount::Operation::read)), - m_statsReadV (StorageAccount::counter (m_token, StorageAccount::Operation::readv)), - m_statsWrite (StorageAccount::counter (m_token, StorageAccount::Operation::write)), - m_statsWriteV (StorageAccount::counter (m_token, StorageAccount::Operation::writev)), - m_statsPosition (StorageAccount::counter (m_token, StorageAccount::Operation::position)), - m_statsPrefetch (StorageAccount::counter (m_token, StorageAccount::Operation::prefetch)) -{ - StorageAccount::Stamp stats (StorageAccount::counter (m_token, StorageAccount::Operation::construct)); - stats.tick (); +StorageAccountProxy::StorageAccountProxy(const std::string &storageClass, std::unique_ptr baseStorage) + : m_baseStorage(std::move(baseStorage)), + m_token(StorageAccount::tokenForStorageClassName(storageClass)), + m_statsRead(StorageAccount::counter(m_token, StorageAccount::Operation::read)), + m_statsReadV(StorageAccount::counter(m_token, StorageAccount::Operation::readv)), + m_statsWrite(StorageAccount::counter(m_token, StorageAccount::Operation::write)), + m_statsWriteV(StorageAccount::counter(m_token, StorageAccount::Operation::writev)), + m_statsPosition(StorageAccount::counter(m_token, StorageAccount::Operation::position)), + m_statsPrefetch(StorageAccount::counter(m_token, StorageAccount::Operation::prefetch)) { + StorageAccount::Stamp stats(StorageAccount::counter(m_token, StorageAccount::Operation::construct)); + stats.tick(); } -StorageAccountProxy::~StorageAccountProxy (void) -{ - StorageAccount::Stamp stats (StorageAccount::counter (m_token, StorageAccount::Operation::destruct)); +StorageAccountProxy::~StorageAccountProxy(void) { + StorageAccount::Stamp stats(StorageAccount::counter(m_token, StorageAccount::Operation::destruct)); releaseStorage(); - stats.tick (); + stats.tick(); } -IOSize -StorageAccountProxy::read (void *into, IOSize n) -{ - StorageAccount::Stamp stats (m_statsRead); - IOSize result = m_baseStorage->read (into, n); - stats.tick (result); +IOSize StorageAccountProxy::read(void *into, IOSize n) { + StorageAccount::Stamp stats(m_statsRead); + IOSize result = m_baseStorage->read(into, n); + stats.tick(result); return result; } -IOSize -StorageAccountProxy::read (void *into, IOSize n, IOOffset pos) -{ - StorageAccount::Stamp stats (m_statsRead); - IOSize result = m_baseStorage->read (into, n, pos); - stats.tick (result); +IOSize StorageAccountProxy::read(void *into, IOSize n, IOOffset pos) { + StorageAccount::Stamp stats(m_statsRead); + IOSize result = m_baseStorage->read(into, n, pos); + stats.tick(result); return result; } -IOSize -StorageAccountProxy::readv (IOBuffer *into, IOSize n) -{ - StorageAccount::Stamp stats (m_statsReadV); - IOSize result = m_baseStorage->readv (into, n); - stats.tick (result, n); +IOSize StorageAccountProxy::readv(IOBuffer *into, IOSize n) { + StorageAccount::Stamp stats(m_statsReadV); + IOSize result = m_baseStorage->readv(into, n); + stats.tick(result, n); return result; } -IOSize -StorageAccountProxy::readv (IOPosBuffer *into, IOSize n) -{ - StorageAccount::Stamp stats (m_statsReadV); - IOSize result = m_baseStorage->readv (into, n); - stats.tick (result, n); +IOSize StorageAccountProxy::readv(IOPosBuffer *into, IOSize n) { + StorageAccount::Stamp stats(m_statsReadV); + IOSize result = m_baseStorage->readv(into, n); + stats.tick(result, n); return result; } -IOSize -StorageAccountProxy::write (const void *from, IOSize n) -{ - StorageAccount::Stamp stats (m_statsWrite); - IOSize result = m_baseStorage->write (from, n); - stats.tick (result); +IOSize StorageAccountProxy::write(const void *from, IOSize n) { + StorageAccount::Stamp stats(m_statsWrite); + IOSize result = m_baseStorage->write(from, n); + stats.tick(result); return result; } -IOSize -StorageAccountProxy::write (const void *from, IOSize n, IOOffset pos) -{ - StorageAccount::Stamp stats (m_statsWrite); - IOSize result = m_baseStorage->write (from, n, pos); - stats.tick (result); +IOSize StorageAccountProxy::write(const void *from, IOSize n, IOOffset pos) { + StorageAccount::Stamp stats(m_statsWrite); + IOSize result = m_baseStorage->write(from, n, pos); + stats.tick(result); return result; } -IOSize -StorageAccountProxy::writev (const IOBuffer *from, IOSize n) -{ - StorageAccount::Stamp stats (m_statsWriteV); - IOSize result = m_baseStorage->writev (from, n); - stats.tick (result, n); +IOSize StorageAccountProxy::writev(const IOBuffer *from, IOSize n) { + StorageAccount::Stamp stats(m_statsWriteV); + IOSize result = m_baseStorage->writev(from, n); + stats.tick(result, n); return result; } -IOSize -StorageAccountProxy::writev (const IOPosBuffer *from, IOSize n) -{ - StorageAccount::Stamp stats (m_statsWriteV); - IOSize result = m_baseStorage->writev (from, n); - stats.tick (result, n); +IOSize StorageAccountProxy::writev(const IOPosBuffer *from, IOSize n) { + StorageAccount::Stamp stats(m_statsWriteV); + IOSize result = m_baseStorage->writev(from, n); + stats.tick(result, n); return result; } -IOOffset -StorageAccountProxy::position (IOOffset offset, Relative whence) -{ - StorageAccount::Stamp stats (m_statsPosition); - IOOffset result = m_baseStorage->position (offset, whence); - stats.tick (); +IOOffset StorageAccountProxy::position(IOOffset offset, Relative whence) { + StorageAccount::Stamp stats(m_statsPosition); + IOOffset result = m_baseStorage->position(offset, whence); + stats.tick(); return result; } -void -StorageAccountProxy::resize (IOOffset size) -{ - StorageAccount::Stamp stats (StorageAccount::counter (m_token, StorageAccount::Operation::resize)); - m_baseStorage->resize (size); - stats.tick (); +void StorageAccountProxy::resize(IOOffset size) { + StorageAccount::Stamp stats(StorageAccount::counter(m_token, StorageAccount::Operation::resize)); + m_baseStorage->resize(size); + stats.tick(); } -void -StorageAccountProxy::flush (void) -{ - StorageAccount::Stamp stats (StorageAccount::counter (m_token, StorageAccount::Operation::flush)); - m_baseStorage->flush (); - stats.tick (); +void StorageAccountProxy::flush(void) { + StorageAccount::Stamp stats(StorageAccount::counter(m_token, StorageAccount::Operation::flush)); + m_baseStorage->flush(); + stats.tick(); } -void -StorageAccountProxy::close (void) -{ - StorageAccount::Stamp stats (StorageAccount::counter (m_token, StorageAccount::Operation::close)); - m_baseStorage->close (); - stats.tick (); +void StorageAccountProxy::close(void) { + StorageAccount::Stamp stats(StorageAccount::counter(m_token, StorageAccount::Operation::close)); + m_baseStorage->close(); + stats.tick(); } -bool -StorageAccountProxy::prefetch (const IOPosBuffer *what, IOSize n) -{ - StorageAccount::Stamp stats (m_statsPrefetch); +bool StorageAccountProxy::prefetch(const IOPosBuffer *what, IOSize n) { + StorageAccount::Stamp stats(m_statsPrefetch); bool value = m_baseStorage->prefetch(what, n); - if (value) - { + if (value) { IOSize total = 0; for (IOSize i = 0; i < n; ++i) total += what[i].size(); - stats.tick (total); + stats.tick(total); } return value; } diff --git a/Utilities/StorageFactory/src/StorageFactory.cc b/Utilities/StorageFactory/src/StorageFactory.cc index 64f8e6e8324d6..cf5985bad3403 100644 --- a/Utilities/StorageFactory/src/StorageFactory.cc +++ b/Utilities/StorageFactory/src/StorageFactory.cc @@ -10,77 +10,48 @@ StorageFactory StorageFactory::s_instance; -StorageFactory::StorageFactory (void) - : m_cacheHint(CACHE_HINT_AUTO_DETECT), - m_readHint(READ_HINT_AUTO), - m_accounting (false), - m_tempfree (4.), // GB - m_temppath (".:$TMPDIR"), - m_timeout(0U), - m_debugLevel(0U) -{ +StorageFactory::StorageFactory(void) + : m_cacheHint(CACHE_HINT_AUTO_DETECT), + m_readHint(READ_HINT_AUTO), + m_accounting(false), + m_tempfree(4.), // GB + m_temppath(".:$TMPDIR"), + m_timeout(0U), + m_debugLevel(0U) { setTempDir(m_temppath, m_tempfree); } -StorageFactory::~StorageFactory (void) -{ -} +StorageFactory::~StorageFactory(void) {} -const StorageFactory * -StorageFactory::get (void) -{ return &s_instance; } +const StorageFactory *StorageFactory::get(void) { return &s_instance; } -StorageFactory * -StorageFactory::getToModify (void) -{ return &s_instance; } +StorageFactory *StorageFactory::getToModify(void) { return &s_instance; } -bool -StorageFactory::enableAccounting (bool enabled) -{ +bool StorageFactory::enableAccounting(bool enabled) { bool old = m_accounting; m_accounting = enabled; return old; } -bool -StorageFactory::accounting(void) const -{ return m_accounting; } +bool StorageFactory::accounting(void) const { return m_accounting; } -void -StorageFactory::setCacheHint(CacheHint value) -{ m_cacheHint = value; } +void StorageFactory::setCacheHint(CacheHint value) { m_cacheHint = value; } -StorageFactory::CacheHint -StorageFactory::cacheHint(void) const -{ return m_cacheHint; } +StorageFactory::CacheHint StorageFactory::cacheHint(void) const { return m_cacheHint; } -void -StorageFactory::setReadHint(ReadHint value) -{ m_readHint = value; } +void StorageFactory::setReadHint(ReadHint value) { m_readHint = value; } -StorageFactory::ReadHint -StorageFactory::readHint(void) const -{ return m_readHint; } +StorageFactory::ReadHint StorageFactory::readHint(void) const { return m_readHint; } -void -StorageFactory::setTimeout(unsigned int timeout) -{ m_timeout = timeout; } +void StorageFactory::setTimeout(unsigned int timeout) { m_timeout = timeout; } -unsigned int -StorageFactory::timeout(void) const -{ return m_timeout; } +unsigned int StorageFactory::timeout(void) const { return m_timeout; } -void -StorageFactory::setDebugLevel(unsigned int level) -{ m_debugLevel = level; } +void StorageFactory::setDebugLevel(unsigned int level) { m_debugLevel = level; } -unsigned int -StorageFactory::debugLevel(void) const -{ return m_debugLevel; } +unsigned int StorageFactory::debugLevel(void) const { return m_debugLevel; } -void -StorageFactory::setTempDir(const std::string &s, double minFreeSpace) -{ +void StorageFactory::setTempDir(const std::string &s, double minFreeSpace) { #if 0 std::cerr /* edm::LogInfo("StorageFactory") */ << "Considering path '" << s @@ -92,18 +63,14 @@ StorageFactory::setTempDir(const std::string &s, double minFreeSpace) std::vector dirs; dirs.reserve(std::count(s.begin(), s.end(), ':') + 1); - while (true) - { + while (true) { size_t end = s.find(':', begin); - if (end == std::string::npos) - { + if (end == std::string::npos) { dirs.push_back(s.substr(begin, end)); break; - } - else - { + } else { dirs.push_back(s.substr(begin, end - begin)); - begin = end+1; + begin = end + 1; } } @@ -118,183 +85,141 @@ StorageFactory::setTempDir(const std::string &s, double minFreeSpace) #endif } -std::string -StorageFactory::tempDir(void) const -{ return m_tempdir; } +std::string StorageFactory::tempDir(void) const { return m_tempdir; } -std::string -StorageFactory::tempPath(void) const -{ return m_temppath; } +std::string StorageFactory::tempPath(void) const { return m_temppath; } -double -StorageFactory::tempMinFree(void) const -{ return m_tempfree; } +double StorageFactory::tempMinFree(void) const { return m_tempfree; } -StorageMaker * -StorageFactory::getMaker (const std::string &proto) const -{ +StorageMaker *StorageFactory::getMaker(const std::string &proto) const { auto itFound = m_makers.find(proto); - if(itFound != m_makers.end()) { - return itFound->second.get(); + if (itFound != m_makers.end()) { + return itFound->second.get(); } - if (! edmplugin::PluginManager::isAvailable()) { + if (!edmplugin::PluginManager::isAvailable()) { edmplugin::PluginManager::configure(edmplugin::standard::config()); } - std::shared_ptr instance{ StorageMakerFactory::get()->tryToCreate(proto)}; - auto insertResult = m_makers.insert(MakerTable::value_type(proto,instance)); + std::shared_ptr instance{StorageMakerFactory::get()->tryToCreate(proto)}; + auto insertResult = m_makers.insert(MakerTable::value_type(proto, instance)); //Can't use instance since it is possible that another thread beat // us to the insertion so the map contains a different instance. return insertResult.first->second.get(); } -StorageMaker * -StorageFactory::getMaker (const std::string &url, - std::string &protocol, - std::string &rest) const -{ +StorageMaker *StorageFactory::getMaker(const std::string &url, std::string &protocol, std::string &rest) const { size_t p = url.find(':'); - if (p != std::string::npos) - { - protocol = url.substr(0,p); - rest = url.substr(p+1); - } - else - { - protocol = "file"; + if (p != std::string::npos) { + protocol = url.substr(0, p); + rest = url.substr(p + 1); + } else { + protocol = "file"; rest = url; } - return getMaker (protocol); + return getMaker(protocol); } - -std::unique_ptr -StorageFactory::open (const std::string &url, int mode /* = IOFlags::OpenRead */) const -{ + +std::unique_ptr StorageFactory::open(const std::string &url, int mode /* = IOFlags::OpenRead */) const { std::string protocol; std::string rest; std::unique_ptr ret; std::unique_ptr stats; - if (StorageMaker *maker = getMaker (url, protocol, rest)) - { + if (StorageMaker *maker = getMaker(url, protocol, rest)) { if (m_accounting) { - auto token = StorageAccount::tokenForStorageClassName(protocol); - stats.reset(new StorageAccount::Stamp(StorageAccount::counter (token, StorageAccount::Operation::open))); + auto token = StorageAccount::tokenForStorageClassName(protocol); + stats.reset(new StorageAccount::Stamp(StorageAccount::counter(token, StorageAccount::Operation::open))); } - try - { - if (auto storage = maker->open (protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout))) - { - if (dynamic_cast(storage.get())) - protocol = "local-cache"; - - if (m_accounting) - ret = std::make_unique(protocol, std::move(storage)); - else - ret = std::move(storage); + try { + if (auto storage = maker->open( + protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout))) { + if (dynamic_cast(storage.get())) + protocol = "local-cache"; + + if (m_accounting) + ret = std::make_unique(protocol, std::move(storage)); + else + ret = std::move(storage); if (stats) - stats->tick(); + stats->tick(); } - } - catch (cms::Exception &err) - { + } catch (cms::Exception &err) { err.addContext("Calling StorageFactory::open()"); err.addAdditionalInfo(err.message()); err.clearMessage(); err << "Failed to open the file '" << url << "'"; throw; } - } + } return ret; } -void -StorageFactory::stagein (const std::string &url) const -{ +void StorageFactory::stagein(const std::string &url) const { std::string protocol; std::string rest; std::unique_ptr stats; - if (StorageMaker *maker = getMaker (url, protocol, rest)) - { + if (StorageMaker *maker = getMaker(url, protocol, rest)) { if (m_accounting) { auto token = StorageAccount::tokenForStorageClassName(protocol); - stats.reset(new StorageAccount::Stamp(StorageAccount::counter (token, StorageAccount::Operation::stagein))); - } - try - { - maker->stagein (protocol, rest,StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout)); - if (stats) stats->tick(); + stats.reset(new StorageAccount::Stamp(StorageAccount::counter(token, StorageAccount::Operation::stagein))); } - catch (cms::Exception &err) - { - edm::LogWarning("StorageFactory::stagein()") - << "Failed to stage in file '" << url << "' because:\n" - << err.explainSelf(); + try { + maker->stagein(protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout)); + if (stats) + stats->tick(); + } catch (cms::Exception &err) { + edm::LogWarning("StorageFactory::stagein()") << "Failed to stage in file '" << url << "' because:\n" + << err.explainSelf(); } } } -bool -StorageFactory::check (const std::string &url, IOOffset *size /* = 0 */) const -{ +bool StorageFactory::check(const std::string &url, IOOffset *size /* = 0 */) const { std::string protocol; std::string rest; bool ret = false; std::unique_ptr stats; - if (StorageMaker *maker = getMaker (url, protocol, rest)) - { + if (StorageMaker *maker = getMaker(url, protocol, rest)) { if (m_accounting) { auto token = StorageAccount::tokenForStorageClassName(protocol); - stats.reset(new StorageAccount::Stamp(StorageAccount::counter (token, StorageAccount::Operation::check))); + stats.reset(new StorageAccount::Stamp(StorageAccount::counter(token, StorageAccount::Operation::check))); } - try - { - ret = maker->check (protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size); - if (stats) stats->tick(); - } - catch (cms::Exception &err) - { + try { + ret = maker->check( + protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size); + if (stats) + stats->tick(); + } catch (cms::Exception &err) { edm::LogWarning("StorageFactory::check()") - << "Existence or size check for the file '" << url << "' failed because:\n" - << err.explainSelf(); + << "Existence or size check for the file '" << url << "' failed because:\n" + << err.explainSelf(); } } - + return ret; } -std::unique_ptr -StorageFactory::wrapNonLocalFile (std::unique_ptr s, - const std::string &proto, - const std::string &path, - int mode) const -{ +std::unique_ptr StorageFactory::wrapNonLocalFile(std::unique_ptr s, + const std::string &proto, + const std::string &path, + int mode) const { StorageFactory::CacheHint hint = cacheHint(); - if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) - { - if (mode & IOFlags::OpenWrite) - { - // For now, issue no warning - otherwise, we'd always warn on output files. - } - else if (m_tempdir.empty()) - { - edm::LogWarning("StorageFactory") << m_unusableDirWarnings; - } - else if ( (not path.empty()) and m_lfs.isLocalPath(path)) - { - // For now, issue no warning - otherwise, we'd always warn on local input files. - } - else - { - if (accounting()) {s = std::make_unique(proto, std::move(s));} - s = std::make_unique(std::move(s), m_tempdir); + if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) { + if (mode & IOFlags::OpenWrite) { + // For now, issue no warning - otherwise, we'd always warn on output files. + } else if (m_tempdir.empty()) { + edm::LogWarning("StorageFactory") << m_unusableDirWarnings; + } else if ((not path.empty()) and m_lfs.isLocalPath(path)) { + // For now, issue no warning - otherwise, we'd always warn on local input files. + } else { + if (accounting()) { + s = std::make_unique(proto, std::move(s)); } + s = std::make_unique(std::move(s), m_tempdir); + } } return s; } - - - diff --git a/Utilities/StorageFactory/src/StorageMaker.cc b/Utilities/StorageFactory/src/StorageMaker.cc index 4eeabcd6a2a2a..e8a08ef4c29cc 100644 --- a/Utilities/StorageFactory/src/StorageMaker.cc +++ b/Utilities/StorageFactory/src/StorageMaker.cc @@ -3,18 +3,12 @@ #include "Utilities/StorageFactory/interface/IOFlags.h" #include -void -StorageMaker::stagein (const std::string &/*proto*/, - const std::string &/*path*/, - const AuxSettings& ) const -{} +void StorageMaker::stagein(const std::string & /*proto*/, const std::string & /*path*/, const AuxSettings &) const {} -bool -StorageMaker::check (const std::string &proto, - const std::string &path, - const AuxSettings& aux, - IOOffset *size /* = 0 */) const -{ +bool StorageMaker::check(const std::string &proto, + const std::string &path, + const AuxSettings &aux, + IOOffset *size /* = 0 */) const { // Fallback method is to open the file and check its // size. Because grid jobs run in a directory where // there is usually more space than in /tmp, and that @@ -24,12 +18,11 @@ StorageMaker::check (const std::string &proto, // destructor or close method. bool found = false; int mode = IOFlags::OpenRead | IOFlags::OpenUnbuffered; - if (auto s = open (proto, path, mode, aux)) - { + if (auto s = open(proto, path, mode, aux)) { if (size) - *size = s->size (); + *size = s->size(); - s->close (); + s->close(); found = true; } diff --git a/Utilities/StorageFactory/src/SysFile.h b/Utilities/StorageFactory/src/SysFile.h index 6f56de80e1d33..acb8784ad20f9 100644 --- a/Utilities/StorageFactory/src/SysFile.h +++ b/Utilities/StorageFactory/src/SysFile.h @@ -1,24 +1,24 @@ #ifndef STORAGE_FACTORY_SYS_FILE_H -# define STORAGE_FACTORY_SYS_FILE_H +#define STORAGE_FACTORY_SYS_FILE_H -# include -# include -# include -# include -# include -# include -# include +#include +#include +#include +#include +#include +#include +#include -# if !defined O_SYNC && defined O_SYNCIO -# define O_SYNC O_SYNCIO -# endif +#if !defined O_SYNC && defined O_SYNCIO +#define O_SYNC O_SYNCIO +#endif -# if !defined O_NONBLOCK && defined O_NDELAY -# define O_NONBLOCK O_NDELAY -# endif +#if !defined O_NONBLOCK && defined O_NDELAY +#define O_NONBLOCK O_NDELAY +#endif -# ifndef O_NONBLOCK -# define O_NONBLOCK 0 -# endif +#ifndef O_NONBLOCK +#define O_NONBLOCK 0 +#endif -#endif // STORAGE_FACTORY_SYS_FILE_H +#endif // STORAGE_FACTORY_SYS_FILE_H diff --git a/Utilities/StorageFactory/src/SysIOChannel.h b/Utilities/StorageFactory/src/SysIOChannel.h index 0580c12c97e53..041a4d0043f74 100644 --- a/Utilities/StorageFactory/src/SysIOChannel.h +++ b/Utilities/StorageFactory/src/SysIOChannel.h @@ -1,10 +1,10 @@ #ifndef STORAGE_FACTORY_SYS_IO_CHANNEL_H -# define STORAGE_FACTORY_SYS_IO_CHANNEL_H +#define STORAGE_FACTORY_SYS_IO_CHANNEL_H -# include -# include -# include -# include -# include +#include +#include +#include +#include +#include -#endif // STORAGE_FACTORY_SYS_IO_CHANNEL_H +#endif // STORAGE_FACTORY_SYS_IO_CHANNEL_H diff --git a/Utilities/StorageFactory/src/Throw.cc b/Utilities/StorageFactory/src/Throw.cc index 6e6d32933d9a1..30db5dbf4d914 100644 --- a/Utilities/StorageFactory/src/Throw.cc +++ b/Utilities/StorageFactory/src/Throw.cc @@ -3,26 +3,16 @@ #include #include -void -throwStorageError (const char* category, - const char *context, - const char *call, int error) -{ +void throwStorageError(const char *category, const char *context, const char *call, int error) { cms::Exception ex(category); - ex << call << " failed with system error '" - << strerror (error) << "' (error code " << error << ")"; + ex << call << " failed with system error '" << strerror(error) << "' (error code " << error << ")"; ex.addContext(context); throw ex; } -void -throwStorageError (edm::errors::ErrorCodes category, - const char *context, - const char *call, int error) -{ +void throwStorageError(edm::errors::ErrorCodes category, const char *context, const char *call, int error) { edm::Exception ex(category); - ex << call << " failed with system error '" - << strerror (error) << "' (error code " << error << ")"; + ex << call << " failed with system error '" << strerror(error) << "' (error code " << error << ")"; ex.addContext(context); throw ex; } diff --git a/Utilities/StorageFactory/src/Throw.h b/Utilities/StorageFactory/src/Throw.h index 8b4795520a987..4bbbcd73c9be2 100644 --- a/Utilities/StorageFactory/src/Throw.h +++ b/Utilities/StorageFactory/src/Throw.h @@ -3,14 +3,8 @@ #include "FWCore/Utilities/interface/EDMException.h" -void throwStorageError (const char* category, - const char *context, - const char *call, - int error); +void throwStorageError(const char *category, const char *context, const char *call, int error); -void throwStorageError (edm::errors::ErrorCodes category, - const char *context, - const char *call, - int error); +void throwStorageError(edm::errors::ErrorCodes category, const char *context, const char *call, int error); -#endif // STORAGE_FACTORY_THROW_H +#endif // STORAGE_FACTORY_THROW_H diff --git a/Utilities/StorageFactory/src/UnixFile.cc b/Utilities/StorageFactory/src/UnixFile.cc index 9f8ed966d6a4e..b724973191975 100644 --- a/Utilities/StorageFactory/src/UnixFile.cc +++ b/Utilities/StorageFactory/src/UnixFile.cc @@ -6,20 +6,15 @@ using namespace IOFlags; -IOFD -File::sysduplicate (IOFD fd) -{ +IOFD File::sysduplicate(IOFD fd) { IOFD copyfd; - if ((copyfd = ::dup (fd)) == EDM_IOFD_INVALID) - throwStorageError ("FileDuplicateError", "Calling File::sysduplicate()", "dup()", errno); + if ((copyfd = ::dup(fd)) == EDM_IOFD_INVALID) + throwStorageError("FileDuplicateError", "Calling File::sysduplicate()", "dup()", errno); return copyfd; } -void -File::sysopen (const char *name, int flags, int perms, - IOFD &newfd, unsigned int& /*newflags*/) -{ +void File::sysopen(const char *name, int flags, int perms, IOFD &newfd, unsigned int & /*newflags*/) { // Translate our flags to system flags. int openflags = 0; @@ -56,18 +51,16 @@ File::sysopen (const char *name, int flags, int perms, if (flags & OpenNotCTTY) openflags |= O_NOCTTY; - if ((newfd = ::open (name, openflags, perms)) == -1) - throwStorageError (edm::errors::FileOpenError, "Calling File::sysopen()", "open()", errno); + if ((newfd = ::open(name, openflags, perms)) == -1) + throwStorageError(edm::errors::FileOpenError, "Calling File::sysopen()", "open()", errno); } -IOSize -File::read (void *into, IOSize n, IOOffset pos) -{ - assert (pos >= 0); +IOSize File::read(void *into, IOSize n, IOOffset pos) { + assert(pos >= 0); ssize_t s; do - s = ::pread (fd (), into, n, pos); + s = ::pread(fd(), into, n, pos); while (s == -1 && errno == EINTR); if (s == -1) @@ -76,14 +69,12 @@ File::read (void *into, IOSize n, IOOffset pos) return s; } -IOSize -File::write (const void *from, IOSize n, IOOffset pos) -{ - assert (pos >= 0); +IOSize File::write(const void *from, IOSize n, IOOffset pos) { + assert(pos >= 0); ssize_t s; do - s = ::pwrite (fd (), from, n, pos); + s = ::pwrite(fd(), from, n, pos); while (s == -1 && errno == EINTR); if (s == -1) @@ -91,70 +82,59 @@ File::write (const void *from, IOSize n, IOOffset pos) if (m_flags & OpenUnbuffered) // FIXME: Exception handling? - flush (); + flush(); return s; } -IOOffset -File::size (void) const -{ - IOFD fd = this->fd (); - assert (fd != EDM_IOFD_INVALID); +IOOffset File::size(void) const { + IOFD fd = this->fd(); + assert(fd != EDM_IOFD_INVALID); struct stat info; - if (fstat (fd, &info) == -1) + if (fstat(fd, &info) == -1) throwStorageError("FileSizeError", "Calling File::size()", "fstat()", errno); return info.st_size; } -IOOffset -File::position (IOOffset offset, Relative whence /* = SET */) -{ - IOFD fd = this->fd (); - assert (fd != EDM_IOFD_INVALID); - assert (whence == CURRENT || whence == SET || whence == END); +IOOffset File::position(IOOffset offset, Relative whence /* = SET */) { + IOFD fd = this->fd(); + assert(fd != EDM_IOFD_INVALID); + assert(whence == CURRENT || whence == SET || whence == END); IOOffset result; - int mywhence = (whence == SET ? SEEK_SET - : whence == CURRENT ? SEEK_CUR - : SEEK_END); - if ((result = ::lseek (fd, offset, mywhence)) == -1) + int mywhence = (whence == SET ? SEEK_SET : whence == CURRENT ? SEEK_CUR : SEEK_END); + if ((result = ::lseek(fd, offset, mywhence)) == -1) throwStorageError("FilePositionError", "Calling File::position()", "lseek()", errno); return result; } -void -File::resize (IOOffset size) -{ - IOFD fd = this->fd (); - assert (fd != EDM_IOFD_INVALID); +void File::resize(IOOffset size) { + IOFD fd = this->fd(); + assert(fd != EDM_IOFD_INVALID); - if (ftruncate (fd, size) == -1) + if (ftruncate(fd, size) == -1) throwStorageError("FileResizeError", "Calling File::resize()", "ftruncate()", errno); } -void -File::flush (void) -{ - IOFD fd = this->fd (); - assert (fd != EDM_IOFD_INVALID); +void File::flush(void) { + IOFD fd = this->fd(); + assert(fd != EDM_IOFD_INVALID); #if _POSIX_SYNCHRONIZED_IO > 0 - if (fdatasync (fd) == -1) + if (fdatasync(fd) == -1) throwStorageError("FileFlushError", "Calling File::flush()", "fdatasync()", errno); #elif _POSIX_FSYNC > 0 - if (fsync (fd) == -1) + if (fsync(fd) == -1) throwStorageError("FileFlushError", "Calling File::flush()", "fsync()", errno); #endif } -bool -File::sysclose (IOFD fd, int *error /* = 0 */) -{ - int ret = ::close (fd); - if (error) *error = errno; +bool File::sysclose(IOFD fd, int *error /* = 0 */) { + int ret = ::close(fd); + if (error) + *error = errno; return ret != -1; } diff --git a/Utilities/StorageFactory/src/UnixIOChannel.cc b/Utilities/StorageFactory/src/UnixIOChannel.cc index 1222c73012651..afe690d857518 100644 --- a/Utilities/StorageFactory/src/UnixIOChannel.cc +++ b/Utilities/StorageFactory/src/UnixIOChannel.cc @@ -6,47 +6,42 @@ #include #include -IOSize -IOChannel::read (void *into, IOSize n) -{ +IOSize IOChannel::read(void *into, IOSize n) { ssize_t s; do - s = ::read (fd (), into, n); + s = ::read(fd(), into, n); while (s == -1 && errno == EINTR); if (s == -1) - throwStorageError (edm::errors::FileReadError, "Calling IOChannel::read()", "read()", errno); + throwStorageError(edm::errors::FileReadError, "Calling IOChannel::read()", "read()", errno); return s; } -IOSize -IOChannel::readv (IOBuffer *into, IOSize buffers) -{ - assert (! buffers || into); +IOSize IOChannel::readv(IOBuffer *into, IOSize buffers) { + assert(!buffers || into); // readv may not support zero buffers. - if (! buffers) + if (!buffers) return 0; ssize_t n = 0; // Convert the buffers to system format. - std::vector bufs (buffers); - for (IOSize i = 0; i < buffers; ++i) - { - bufs [i].iov_len = into [i].size (); - bufs [i].iov_base = (caddr_t) into [i].data (); + std::vector bufs(buffers); + for (IOSize i = 0; i < buffers; ++i) { + bufs[i].iov_len = into[i].size(); + bufs[i].iov_base = (caddr_t)into[i].data(); } // Read as long as signals cancel the read before doing anything. do - n = ::readv (fd (), &bufs [0], buffers); + n = ::readv(fd(), &bufs[0], buffers); while (n == -1 && errno == EINTR); // If it was serious error, throw it. if (n == -1) - throwStorageError (edm::errors::FileReadError, "Calling IOChannel::readv", "readv()", errno); + throwStorageError(edm::errors::FileReadError, "Calling IOChannel::readv", "readv()", errno); // Return the number of bytes actually read. return n; @@ -55,47 +50,42 @@ IOChannel::readv (IOBuffer *into, IOSize buffers) ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -IOSize -IOChannel::write (const void *from, IOSize n) -{ +IOSize IOChannel::write(const void *from, IOSize n) { ssize_t s; do - s = ::write (fd (), from, n); + s = ::write(fd(), from, n); while (s == -1 && errno == EINTR); if (s == -1 && errno != EWOULDBLOCK) - throwStorageError (edm::errors::FileWriteError, "Calling IOChannel::write()", "write()", errno); + throwStorageError(edm::errors::FileWriteError, "Calling IOChannel::write()", "write()", errno); return s >= 0 ? s : 0; } -IOSize -IOChannel::writev (const IOBuffer *from, IOSize buffers) -{ - assert (! buffers || from); +IOSize IOChannel::writev(const IOBuffer *from, IOSize buffers) { + assert(!buffers || from); // writev may not support zero buffers. - if (! buffers) + if (!buffers) return 0; ssize_t n = 0; // Convert the buffers to system format. - std::vector bufs (buffers); - for (IOSize i = 0; i < buffers; ++i) - { - bufs [i].iov_len = from [i].size (); - bufs [i].iov_base = (caddr_t) from [i].data (); + std::vector bufs(buffers); + for (IOSize i = 0; i < buffers; ++i) { + bufs[i].iov_len = from[i].size(); + bufs[i].iov_base = (caddr_t)from[i].data(); } // Read as long as signals cancel the read before doing anything. do - n = ::writev (fd (), &bufs [0], buffers); + n = ::writev(fd(), &bufs[0], buffers); while (n == -1 && errno == EINTR); // If it was serious error, throw it. if (n == -1) - throwStorageError (edm::errors::FileWriteError, "Calling IOChannel::writev()", "writev()", errno); + throwStorageError(edm::errors::FileWriteError, "Calling IOChannel::writev()", "writev()", errno); // Return the number of bytes actually written. return n; @@ -104,45 +94,39 @@ IOChannel::writev (const IOBuffer *from, IOSize buffers) ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -void -IOChannel::setBlocking (bool value) -{ +void IOChannel::setBlocking(bool value) { #ifdef O_NONBLOCK int mode; int off = value ? ~0 : ~(O_NDELAY | O_NONBLOCK); - int on = value ? O_NONBLOCK : 0; + int on = value ? O_NONBLOCK : 0; - if ((mode = fcntl (fd (), F_GETFL, 0)) == -1 - || fcntl (fd (), F_SETFL, (mode & off) | on) == -1) - throwStorageError ("FileSetBlockingError", "Calling IOChannel::setBlocking()", "fcntl()", errno); + if ((mode = fcntl(fd(), F_GETFL, 0)) == -1 || fcntl(fd(), F_SETFL, (mode & off) | on) == -1) + throwStorageError("FileSetBlockingError", "Calling IOChannel::setBlocking()", "fcntl()", errno); #elif defined FIONBIO int mode = value; - if (ioctl (fd (), FIONBIO, &value) == -1) - throwStorageError ("FileSetBlockingError", "Calling IOChannel::setBlocking()", "ioctl()", errno); + if (ioctl(fd(), FIONBIO, &value) == -1) + throwStorageError("FileSetBlockingError", "Calling IOChannel::setBlocking()", "ioctl()", errno); #endif } -bool -IOChannel::isBlocking (void) const -{ +bool IOChannel::isBlocking(void) const { #ifdef O_NONBLOCK int mode; - if ((mode = fcntl (fd (), F_GETFL, 0)) == -1) - throwStorageError ("FileIsBlockingError", "Calling IOChannel::isBlocking()", "fcntl()", errno); + if ((mode = fcntl(fd(), F_GETFL, 0)) == -1) + throwStorageError("FileIsBlockingError", "Calling IOChannel::isBlocking()", "fcntl()", errno); return mode & (O_NDELAY | O_NONBLOCK); -#else // ! O_NONBLOCK +#else // ! O_NONBLOCK return true; -#endif // O_NONBLOCK +#endif // O_NONBLOCK } ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -bool -IOChannel::sysclose (IOFD fd, int *error /* = 0 */) -{ - int ret = ::close (fd); - if (error) *error = errno; +bool IOChannel::sysclose(IOFD fd, int *error /* = 0 */) { + int ret = ::close(fd); + if (error) + *error = errno; return ret != -1; } diff --git a/Utilities/StorageFactory/test/Test.h b/Utilities/StorageFactory/test/Test.h index e387654351234..b968be4be6776 100644 --- a/Utilities/StorageFactory/test/Test.h +++ b/Utilities/StorageFactory/test/Test.h @@ -10,35 +10,30 @@ #include static std::shared_ptr gobbleUpTheGoop; -static void initTest(void) -{ +static void initTest(void) { // Initialise the plug-in manager. edmplugin::PluginManager::configure(edmplugin::standard::config()); // Enable storage accounting - StorageFactory::getToModify ()->enableAccounting (true); + StorageFactory::getToModify()->enableAccounting(true); // This interface sucks but does the job, which is to silence // the message logger chatter about "info" or "debug" messages, // and prevent the tests from hanging forever due to lack of a // logger. - try - { - gobbleUpTheGoop = std::shared_ptr - (edm::PresenceFactory::get()->makePresence("SingleThreadMSPresence").release()); - } - catch (cms::Exception &e) - { + try { + gobbleUpTheGoop = + std::shared_ptr(edm::PresenceFactory::get()->makePresence("SingleThreadMSPresence").release()); + } catch (cms::Exception &e) { std::cerr << e.explainSelf() << std::endl; } const char *pset = - ");default=-P()>)>"; - edm::MessageLoggerQ::MLqMOD (new std::string); - edm::MessageLoggerQ::MLqCFG (new edm::ParameterSet (pset)); - edm::LogInfo("AvoidExitCrash") - << "Message logger, please don't crash at exit if" - << " nothing else worthwhile was printed. Thanks."; + ");default=-P()>)>"; + edm::MessageLoggerQ::MLqMOD(new std::string); + edm::MessageLoggerQ::MLqCFG(new edm::ParameterSet(pset)); + edm::LogInfo("AvoidExitCrash") << "Message logger, please don't crash at exit if" + << " nothing else worthwhile was printed. Thanks."; } diff --git a/Utilities/StorageFactory/test/any.cpp b/Utilities/StorageFactory/test/any.cpp index 061578de4456b..16b6200a24e04 100644 --- a/Utilities/StorageFactory/test/any.cpp +++ b/Utilities/StorageFactory/test/any.cpp @@ -2,39 +2,38 @@ #include "Utilities/StorageFactory/interface/Storage.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int argc, char **argv) try -{ +int main(int argc, char **argv) try { initTest(); - if (argc != 2) - { + if (argc != 2) { std::cerr << "usage: " << argv[0] << " FILE\n"; return EXIT_FAILURE; } - IOOffset size = -1; - bool exists = StorageFactory::get ()->check(argv [1], &size); + IOOffset size = -1; + bool exists = StorageFactory::get()->check(argv[1], &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; - if (! exists) return EXIT_SUCCESS; + if (!exists) + return EXIT_SUCCESS; static const int SIZE = 1048576; - auto s = StorageFactory::get ()->open (argv [1]); - char *buf = (char *) malloc (SIZE); - IOSize n; + auto s = StorageFactory::get()->open(argv[1]); + char *buf = (char *)malloc(SIZE); + IOSize n; - while ((n = s->read (buf, SIZE))) - std::cout.write (buf, n); + while ((n = s->read(buf, SIZE))) + std::cout.write(buf, n); s->close(); - free (buf); + free(buf); std::cerr << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const &e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const &e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/ftp.cpp b/Utilities/StorageFactory/test/ftp.cpp index 9d74bb3e45800..1826f6d9f1326 100644 --- a/Utilities/StorageFactory/test/ftp.cpp +++ b/Utilities/StorageFactory/test/ftp.cpp @@ -1,21 +1,19 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOOffset size = -1; - bool exists = StorageFactory::get ()->check - ("ftp://cmsdoc.cern.ch/WELCOME", &size); + IOOffset size = -1; + bool exists = StorageFactory::get()->check("ftp://cmsdoc.cern.ch/WELCOME", &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/ftp2.cpp b/Utilities/StorageFactory/test/ftp2.cpp index 0d7f3fca15fe6..b4797d8e91996 100644 --- a/Utilities/StorageFactory/test/ftp2.cpp +++ b/Utilities/StorageFactory/test/ftp2.cpp @@ -1,21 +1,19 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOOffset size = -1; - bool exists = StorageFactory::get ()->check - ("ftp://cmsdoc.cern.ch/non-existent", &size); + IOOffset size = -1; + bool exists = StorageFactory::get()->check("ftp://cmsdoc.cern.ch/non-existent", &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; - std::cout << StorageAccount::summaryText (true) << std::endl; + std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/local.cpp b/Utilities/StorageFactory/test/local.cpp index 15da5958f802d..c071da299d263 100644 --- a/Utilities/StorageFactory/test/local.cpp +++ b/Utilities/StorageFactory/test/local.cpp @@ -2,25 +2,24 @@ #include "Utilities/StorageFactory/interface/Storage.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - auto s = StorageFactory::get ()->open ("/etc/passwd"); - char buf [1024]; - IOSize n; + auto s = StorageFactory::get()->open("/etc/passwd"); + char buf[1024]; + IOSize n; - while ((n = s->read (buf, sizeof (buf)))) - std::cout.write (buf, n); + while ((n = s->read(buf, sizeof(buf)))) + std::cout.write(buf, n); s->close(); - std::cout << StorageAccount::summaryText (true) << std::endl; + std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/local2.cpp b/Utilities/StorageFactory/test/local2.cpp index beb987456bb2c..49c62e8425f58 100644 --- a/Utilities/StorageFactory/test/local2.cpp +++ b/Utilities/StorageFactory/test/local2.cpp @@ -1,20 +1,19 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOOffset size; - bool exists = StorageFactory::get ()->check ("/etc/passwd", &size); + IOOffset size; + bool exists = StorageFactory::get()->check("/etc/passwd", &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; - std::cout << "stats:\n" << StorageAccount::summaryText () << std::endl; + std::cout << "stats:\n" << StorageAccount::summaryText() << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/local3.cpp b/Utilities/StorageFactory/test/local3.cpp index 54ce56d3b7aad..07af3ae84477e 100644 --- a/Utilities/StorageFactory/test/local3.cpp +++ b/Utilities/StorageFactory/test/local3.cpp @@ -1,18 +1,17 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - bool exists = StorageFactory::get ()->check ("/etc/passwdx"); + bool exists = StorageFactory::get()->check("/etc/passwdx"); std::cout << "exists = " << exists << "\n"; - std::cout << "stats:\n" << StorageAccount::summaryText () << std::endl; + std::cout << "stats:\n" << StorageAccount::summaryText() << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/mkstemp.cpp b/Utilities/StorageFactory/test/mkstemp.cpp index c28c28ac96708..ec99c3edddc98 100644 --- a/Utilities/StorageFactory/test/mkstemp.cpp +++ b/Utilities/StorageFactory/test/mkstemp.cpp @@ -8,7 +8,7 @@ #include #include -int main (int, char **) try { +int main(int, char**) try { initTest(); char pattern[] = "mkstemp-test-XXXXXX\0"; struct stat status; @@ -17,27 +17,24 @@ int main (int, char **) try { umask(previous_umask); if (fd == -1) { throw cms::Exception("TemporaryFile") - << "Cannot create temporary file '" << pattern << "': " - << strerror(errno) << " (error " << errno << ")"; + << "Cannot create temporary file '" << pattern << "': " << strerror(errno) << " (error " << errno << ")"; } int ret = fstat(fd, &status); unlink(pattern); - if(ret != 0) { + if (ret != 0) { throw cms::Exception("TemporaryFile") - << "Cannot fstat temporary file '" << pattern << "': " - << strerror(errno) << " (error " << errno << ")"; + << "Cannot fstat temporary file '" << pattern << "': " << strerror(errno) << " (error " << errno << ")"; } mode_t mode = status.st_mode & 0777; - if(mode != 0600) { - throw cms::Exception("TemporaryFile") - << "Temporary file '" << pattern << "': " - << "created with mode " << std::oct << mode << " rather than 0600"; + if (mode != 0600) { + throw cms::Exception("TemporaryFile") << "Temporary file '" << pattern << "': " + << "created with mode " << std::oct << mode << " rather than 0600"; } return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/randomread.cpp b/Utilities/StorageFactory/test/randomread.cpp index ec46218220b3f..333171cad29cc 100644 --- a/Utilities/StorageFactory/test/randomread.cpp +++ b/Utilities/StorageFactory/test/randomread.cpp @@ -5,74 +5,65 @@ #include #include -int main (int argc, char **argv) try -{ +int main(int argc, char** argv) try { initTest(); - if (argc != 2) - { + if (argc != 2) { std::cerr << "usage: " << argv[0] << " FILE...\n"; return EXIT_FAILURE; } - StorageFactory::getToModify ()->enableAccounting(true); + StorageFactory::getToModify()->enableAccounting(true); std::vector> storages; - std::vector sizes; - for (int i = 1; i < argc; ++i) - { - IOOffset size = -1; - bool exists = StorageFactory::get ()->check(argv [i], &size); + std::vector sizes; + for (int i = 1; i < argc; ++i) { + IOOffset size = -1; + bool exists = StorageFactory::get()->check(argv[i], &size); std::cout << argv[i] << " exists = " << exists << ", size = " << size << std::endl; - if (exists) - { - storages.push_back(StorageFactory::get ()->open - (argv [i],IOFlags::OpenRead|IOFlags::OpenUnbuffered)); - sizes.push_back(size); + if (exists) { + storages.push_back(StorageFactory::get()->open(argv[i], IOFlags::OpenRead | IOFlags::OpenUnbuffered)); + sizes.push_back(size); } } if (sizes.empty()) return EXIT_SUCCESS; - std::cout << "stats:\n" << StorageAccount::summaryText (true) << std::endl; + std::cout << "stats:\n" << StorageAccount::summaryText(true) << std::endl; - IOSize n; - char buf [10000]; - IOSize maxBufSize = sizeof(buf); - int niter = 100000; + IOSize n; + char buf[10000]; + IOSize maxBufSize = sizeof(buf); + int niter = 100000; - while (niter--) - for (size_t i = 0; i < sizes.size(); ++i) - { + while (niter--) + for (size_t i = 0; i < sizes.size(); ++i) { double bfract = double(rand()) / double(std::numeric_limits::max()); double pfract = double(rand()) / double(std::numeric_limits::max()); IOSize bufSize = static_cast(maxBufSize * bfract); - IOOffset pos = static_cast((sizes[i] - bufSize)*pfract); + IOOffset pos = static_cast((sizes[i] - bufSize) * pfract); // std::cout << "read " << bufSize << " at " << pos << std::endl; - storages[i]->position(pos); - n = storages[i]->read (buf, bufSize); - if (n != bufSize) - { - std::cerr << "error for " << i << " (" << argv[i+1] - << "): tried to read " << bufSize << " bytes at " << pos - << "; got " << n << " bytes\n"; - break; + storages[i]->position(pos); + n = storages[i]->read(buf, bufSize); + if (n != bufSize) { + std::cerr << "error for " << i << " (" << argv[i + 1] << "): tried to read " << bufSize << " bytes at " << pos + << "; got " << n << " bytes\n"; + break; } } - for (size_t i = 0; i < sizes.size(); ++i) - { + for (size_t i = 0; i < sizes.size(); ++i) { storages[i]->close(); } std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/rfio2.cpp b/Utilities/StorageFactory/test/rfio2.cpp index c0e283a0080c8..514c54062bd6f 100644 --- a/Utilities/StorageFactory/test/rfio2.cpp +++ b/Utilities/StorageFactory/test/rfio2.cpp @@ -1,22 +1,22 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - IOOffset size; - bool exists = StorageFactory::get ()->check - ("rfio:/castor/cern.ch/cms/reconstruction/datafiles/" - "ORCA_7_5_2/PoolFileCatalog.xml", &size); + IOOffset size; + bool exists = StorageFactory::get()->check( + "rfio:/castor/cern.ch/cms/reconstruction/datafiles/" + "ORCA_7_5_2/PoolFileCatalog.xml", + &size); std::cout << "exists = " << exists << ", size = " << size << "\n"; std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/rfio3.cpp b/Utilities/StorageFactory/test/rfio3.cpp index 3015a0a554158..a0fb3d26f0538 100644 --- a/Utilities/StorageFactory/test/rfio3.cpp +++ b/Utilities/StorageFactory/test/rfio3.cpp @@ -1,21 +1,20 @@ #include "Utilities/StorageFactory/test/Test.h" #include "FWCore/Utilities/interface/Exception.h" -int main (int, char **/*argv*/) try -{ +int main(int, char** /*argv*/) try { initTest(); - bool exists = StorageFactory::get ()->check - ("rfio:/castor/cern.ch/cms/reconstruction/datafiles/" - "ORCA_7_5_2/PoolFileCatalog.xmlx"); + bool exists = StorageFactory::get()->check( + "rfio:/castor/cern.ch/cms/reconstruction/datafiles/" + "ORCA_7_5_2/PoolFileCatalog.xmlx"); std::cout << "exists = " << exists << "\n"; std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/t0Repack.cpp b/Utilities/StorageFactory/test/t0Repack.cpp index 0ce7c86ac4fb3..2776e5ced8db8 100644 --- a/Utilities/StorageFactory/test/t0Repack.cpp +++ b/Utilities/StorageFactory/test/t0Repack.cpp @@ -9,83 +9,65 @@ #include #include -int main (int argc, char **argv) try -{ +int main(int argc, char **argv) try { initTest(); - if (argc < 4) - { + if (argc < 4) { std::cerr << "usage: " << argv[0] << " NUM-DATASETS OUTPUT-FILE INDEX-FILE...\n"; return EXIT_FAILURE; } - int datasetN = ::atoi(argv [1]); - std::string outputURL = argv[2]; + int datasetN = ::atoi(argv[1]); + std::string outputURL = argv[2]; std::unique_ptr outputFile = 0; - IOSize totSize = 0; - std::vector> indexFiles; - std::vector indexSizes; + IOSize totSize = 0; + std::vector> indexFiles; + std::vector indexSizes; - StorageFactory::getToModify ()->enableAccounting(true); + StorageFactory::getToModify()->enableAccounting(true); std::cerr << "write to file " << outputURL << " " << datasetN << " datasets\n"; - for (int i = 3; i < argc; ++i) - { + for (int i = 3; i < argc; ++i) { IOOffset size = -1; - if (StorageFactory::get ()->check(argv [i], &size)) - { - indexFiles.push_back(StorageFactory::get ()->open (argv [i],IOFlags::OpenRead)); + if (StorageFactory::get()->check(argv[i], &size)) { + indexFiles.push_back(StorageFactory::get()->open(argv[i], IOFlags::OpenRead)); indexSizes.push_back(size); - } - else - { - std::cerr << "index file " << argv [i] << " does not exist\n"; + } else { + std::cerr << "index file " << argv[i] << " does not exist\n"; return EXIT_FAILURE; } } // open output file - try - { - outputFile = StorageFactory::get ()->open - (outputURL, IOFlags::OpenWrite|IOFlags::OpenCreate|IOFlags::OpenTruncate); - } - catch (cms::Exception &e) - { - std::cerr << "error in opening output file " << outputURL - << ": " << e.explainSelf () << std::endl; + try { + outputFile = + StorageFactory::get()->open(outputURL, IOFlags::OpenWrite | IOFlags::OpenCreate | IOFlags::OpenTruncate); + } catch (cms::Exception &e) { + std::cerr << "error in opening output file " << outputURL << ": " << e.explainSelf() << std::endl; return EXIT_FAILURE; } // parse index file, read buffer, select and copy to output file - for (size_t i = 0; i < indexFiles.size(); ++i) - { + for (size_t i = 0; i < indexFiles.size(); ++i) { // suck in the index file. - std::cout << "reading from index file " << argv[i+3] << std::endl; + std::cout << "reading from index file " << argv[i + 3] << std::endl; std::istringstream in; - try - { - std::vector lbuf(indexSizes[i]+1, '\0'); - IOSize nn = indexFiles[i]->read(&lbuf[0], indexSizes[i]); - - if (indexSizes[i] < 0 || static_cast(nn) != indexSizes[i]) - { - std::cerr << "error in reading from index file " << argv[i+3] << std::endl; - std::cerr << "asked for " << indexSizes[i] << " bytes, got " << nn << " bytes\n"; + try { + std::vector lbuf(indexSizes[i] + 1, '\0'); + IOSize nn = indexFiles[i]->read(&lbuf[0], indexSizes[i]); + + if (indexSizes[i] < 0 || static_cast(nn) != indexSizes[i]) { + std::cerr << "error in reading from index file " << argv[i + 3] << std::endl; + std::cerr << "asked for " << indexSizes[i] << " bytes, got " << nn << " bytes\n"; return EXIT_FAILURE; } in.str(&lbuf[0]); - } - catch (cms::Exception &e) - { - std::cerr << "error in reading from index file " << argv [i+3] << std::endl - << e.explainSelf() << std::endl; + } catch (cms::Exception &e) { + std::cerr << "error in reading from index file " << argv[i + 3] << std::endl << e.explainSelf() << std::endl; return EXIT_FAILURE; - } - catch (...) - { - std::cerr << "error in reading from index file " << argv [i+3] << std::endl; + } catch (...) { + std::cerr << "error in reading from index file " << argv[i + 3] << std::endl; return EXIT_FAILURE; } @@ -94,93 +76,71 @@ int main (int argc, char **argv) try std::cout << "first line is '" << line1 << "'\n"; std::string::size_type pos = line1.find('='); if (pos != std::string::npos) - pos = line1.find_first_not_of(' ',pos+1); - if (pos == std::string::npos) - { - std::cerr << "badly formed index file " << argv [i+3] << std::endl; + pos = line1.find_first_not_of(' ', pos + 1); + if (pos == std::string::npos) { + std::cerr << "badly formed index file " << argv[i + 3] << std::endl; std::cerr << "first line is:\n" << line1 << std::endl; return EXIT_FAILURE; } - line1.erase(0,pos); + line1.erase(0, pos); std::unique_ptr s; - IOOffset size = 0; - try - { + IOOffset size = 0; + try { std::cout << "input event file " << i << " is " << line1 << std::endl; - if (StorageFactory::get ()->check(line1, &size)) - s = StorageFactory::get ()->open (line1, IOFlags::OpenRead); - else - { + if (StorageFactory::get()->check(line1, &size)) + s = StorageFactory::get()->open(line1, IOFlags::OpenRead); + else { std::cerr << "input file " << line1 << " does not exist\n"; return EXIT_FAILURE; } - } - catch (cms::Exception &e) - { - std::cerr << "error in opening input file " << line1 << std::endl - << e.explainSelf() << std::endl; + } catch (cms::Exception &e) { + std::cerr << "error in opening input file " << line1 << std::endl << e.explainSelf() << std::endl; return EXIT_FAILURE; - } - catch (...) - { + } catch (...) { std::cerr << "error in opening input file " << line1 << std::endl; return EXIT_FAILURE; } std::vector vbuf; - while (in) - { - int dataset = -1; - IOOffset bufLoc = -1; - IOSize bufSize = 0; + while (in) { + int dataset = -1; + IOOffset bufLoc = -1; + IOSize bufSize = 0; in >> dataset >> bufLoc >> bufSize; if (dataset != datasetN) - continue; + continue; std::cout << "copy buf at " << bufLoc << " of size " << bufSize << std::endl; if (bufSize > vbuf.size()) - vbuf.resize(bufSize); + vbuf.resize(bufSize); - char * buf = &vbuf[0]; - try - { + char *buf = &vbuf[0]; + try { s->position(bufLoc); - IOSize n = s->read (buf, bufSize); + IOSize n = s->read(buf, bufSize); totSize += n; - if (n != bufSize) - { + if (n != bufSize) { std::cerr << "error in reading from input file " << line1 << std::endl; - std::cerr << "asked for " << bufSize << " bytes, got " << n << " bytes\n"; + std::cerr << "asked for " << bufSize << " bytes, got " << n << " bytes\n"; return EXIT_FAILURE; } - } - catch (cms::Exception &e) - { - std::cerr << "error in reading input file " << line1 << std::endl - << e.explainSelf() << std::endl; + } catch (cms::Exception &e) { + std::cerr << "error in reading input file " << line1 << std::endl << e.explainSelf() << std::endl; return EXIT_FAILURE; - } - catch (...) - { + } catch (...) { std::cerr << "error in reading input file " << line1 << std::endl; return EXIT_FAILURE; } - try - { - outputFile->write(buf,bufSize); - } - catch (cms::Exception &e) - { - std::cerr << "error in writing output file " << outputURL << std::endl - << e.explainSelf() << std::endl; + try { + outputFile->write(buf, bufSize); + } catch (cms::Exception &e) { + std::cerr << "error in writing output file " << outputURL << std::endl << e.explainSelf() << std::endl; return EXIT_FAILURE; - } - catch (...) - { + } catch (...) { std::cerr << "error in writing output file " << outputURL << std::endl; return EXIT_FAILURE; } @@ -192,12 +152,12 @@ int main (int argc, char **argv) try outputFile->close(); std::cout << "copied a total of " << totSize << " bytes" << std::endl; - std::cout << StorageAccount::summaryText (true) << std::endl; + std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const &e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const &e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/StorageFactory/test/write.cpp b/Utilities/StorageFactory/test/write.cpp index 310d791e87e16..1bc4a121b35ef 100644 --- a/Utilities/StorageFactory/test/write.cpp +++ b/Utilities/StorageFactory/test/write.cpp @@ -4,34 +4,32 @@ #include "Utilities/StorageFactory/interface/Storage.h" #include -int main (int argc, char *argv[]) try -{ +int main(int argc, char* argv[]) try { initTest(); - std::string path ("rfio:/castor/cern.ch/cms/test/IBTestFiles/rfiotestwrite"); - if (argc > 1) { - path = std::string ("rfio:") + argv[1]; - } - std::cout << "copying /etc/profile to " << path << "\n"; + std::string path("rfio:/castor/cern.ch/cms/test/IBTestFiles/rfiotestwrite"); + if (argc > 1) { + path = std::string("rfio:") + argv[1]; + } + std::cout << "copying /etc/profile to " << path << "\n"; - IOSize bytes; - unsigned char buf [4096]; - File input ("/etc/profile"); - auto s = StorageFactory::get ()->open - (path.c_str (), IOFlags::OpenWrite|IOFlags::OpenCreate|IOFlags::OpenTruncate); + IOSize bytes; + unsigned char buf[4096]; + File input("/etc/profile"); + auto s = StorageFactory::get()->open(path.c_str(), IOFlags::OpenWrite | IOFlags::OpenCreate | IOFlags::OpenTruncate); - while ((bytes = input.read (buf, sizeof (buf)))) - s->write (buf, bytes); + while ((bytes = input.read(buf, sizeof(buf)))) + s->write(buf, bytes); - input.close (); - s->close (); + input.close(); + s->close(); std::cout << StorageAccount::summaryText(true) << std::endl; return EXIT_SUCCESS; -} catch(cms::Exception const& e) { +} catch (cms::Exception const& e) { std::cerr << e.explainSelf() << std::endl; return EXIT_FAILURE; -} catch(std::exception const& e) { +} catch (std::exception const& e) { std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/Utilities/Testing/interface/CppUnit_testdriver.icpp b/Utilities/Testing/interface/CppUnit_testdriver.icpp index 07c02e6dac5fa..cf30c81acfd6c 100644 --- a/Utilities/Testing/interface/CppUnit_testdriver.icpp +++ b/Utilities/Testing/interface/CppUnit_testdriver.icpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include /** Main body for all the CppUnit test classes * @@ -14,44 +14,37 @@ * */ +int main(int argc, char* argv[]) { + std::string testPath = (argc > 1) ? std::string(argv[1]) : ""; - int - main( int argc, char* argv[] ) - { - std::string testPath = (argc > 1) ? std::string(argv[1]) : ""; - - // Create the event manager and test controller - CppUnit::TestResult controller; - - // Add a listener that colllects test result - CppUnit::TestResultCollector result; - controller.addListener( &result ); - - // Add a listener that print dots as test run. - CppUnit::TextTestProgressListener progress; - controller.addListener( &progress ); - - // Add the top suite to the test runner - CppUnit::TestRunner runner; - runner.addTest( CppUnit::TestFactoryRegistry::getRegistry().makeTest() ); - try - { - std::cout << "Running " << testPath; - runner.run( controller, testPath ); - - std::cerr << std::endl; - - // Print test in a compiler compatible format. - CppUnit::CompilerOutputter outputter( &result, std::cerr ); - outputter.write(); - } - catch ( std::invalid_argument &e ) // Test path not resolved - { - std::cerr << std::endl - << "ERROR: " << e.what() - << std::endl; - return 0; - } - - return result.wasSuccessful() ? 0 : 1; - } + // Create the event manager and test controller + CppUnit::TestResult controller; + + // Add a listener that colllects test result + CppUnit::TestResultCollector result; + controller.addListener(&result); + + // Add a listener that print dots as test run. + CppUnit::TextTestProgressListener progress; + controller.addListener(&progress); + + // Add the top suite to the test runner + CppUnit::TestRunner runner; + runner.addTest(CppUnit::TestFactoryRegistry::getRegistry().makeTest()); + try { + std::cout << "Running " << testPath; + runner.run(controller, testPath); + + std::cerr << std::endl; + + // Print test in a compiler compatible format. + CppUnit::CompilerOutputter outputter(&result, std::cerr); + outputter.write(); + } catch (std::invalid_argument& e) // Test path not resolved + { + std::cerr << std::endl << "ERROR: " << e.what() << std::endl; + return 0; + } + + return result.wasSuccessful() ? 0 : 1; +} diff --git a/Utilities/Xerces/interface/XercesStrUtils.h b/Utilities/Xerces/interface/XercesStrUtils.h index 78214b9dbdc23..109d0e51008e1 100644 --- a/Utilities/Xerces/interface/XercesStrUtils.h +++ b/Utilities/Xerces/interface/XercesStrUtils.h @@ -13,33 +13,25 @@ XERCES_CPP_NAMESPACE_USE namespace cms { namespace xerces { inline void dispose(XMLCh* ptr) { XMLString::release(&ptr); } - inline void dispose(char* ptr) { XMLString::release(&ptr); } + inline void dispose(char* ptr) { XMLString::release(&ptr); } - template< class CharType > - class ZStr // Zero-terminated string. + template + class ZStr // Zero-terminated string. { public: - ZStr(CharType const* str) - : m_array(const_cast(str), &dispose) - {} - + ZStr(CharType const* str) : m_array(const_cast(str), &dispose) {} + CharType const* ptr() const { return m_array.get(); } private: - std::unique_ptr< CharType, void (*)(CharType*) > m_array; + std::unique_ptr m_array; }; - - inline ZStr uStr(char const* str) { - return ZStr(XMLString::transcode(str)); - } - - inline ZStr cStr(XMLCh const* str) { - return ZStr(XMLString::transcode(str)); - } - - inline std::string toString(XMLCh const* toTranscode) { - return std::string(cStr(toTranscode).ptr()); - } + + inline ZStr uStr(char const* str) { return ZStr(XMLString::transcode(str)); } + + inline ZStr cStr(XMLCh const* str) { return ZStr(XMLString::transcode(str)); } + + inline std::string toString(XMLCh const* toTranscode) { return std::string(cStr(toTranscode).ptr()); } inline unsigned int toUInt(XMLCh const* toTranscode) { std::istringstream iss(toString(toTranscode)); @@ -51,17 +43,17 @@ namespace cms { inline bool toBool(XMLCh const* toTranscode) { std::string value = toString(toTranscode); if ((value == "true") || (value == "1")) - return true; + return true; return false; } - + inline double toDouble(XMLCh const* toTranscode) { std::istringstream iss(toString(toTranscode)); double returnValue; iss >> returnValue; return returnValue; } - } -} + } // namespace xerces +} // namespace cms #endif diff --git a/Utilities/XrdAdaptor/plugins/XrdStorageMaker.cc b/Utilities/XrdAdaptor/plugins/XrdStorageMaker.cc index 0c976603de217..81e6407691623 100644 --- a/Utilities/XrdAdaptor/plugins/XrdStorageMaker.cc +++ b/Utilities/XrdAdaptor/plugins/XrdStorageMaker.cc @@ -16,35 +16,28 @@ #include #include -class MakerResponseHandler : public XrdCl::ResponseHandler -{ +class MakerResponseHandler : public XrdCl::ResponseHandler { public: - void HandleResponse( XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response ) override - { - // Note: Prepare call has a response object. - delete response; - delete status; - } - + void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override { + // Note: Prepare call has a response object. + delete response; + delete status; + } }; -class XrdStorageMaker final : public StorageMaker -{ +class XrdStorageMaker final : public StorageMaker { public: - static const unsigned int XRD_DEFAULT_TIMEOUT = 3*60; + static const unsigned int XRD_DEFAULT_TIMEOUT = 3 * 60; - XrdStorageMaker(): - m_lastDebugLevel(1),//so that 0 will trigger change - m_lastTimeout(0) - { + XrdStorageMaker() + : m_lastDebugLevel(1), //so that 0 will trigger change + m_lastTimeout(0) { // When CMSSW loads, both XrdCl and XrdClient end up being loaded // (ROOT loads XrdClient). XrdClient forces IPv4-only. Accordingly, // we must explicitly set the default network stack in XrdCl to // whatever is available on the node (IPv4 or IPv6). XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); - if (env) - { + if (env) { env->PutString("NetworkStack", "IPAuto"); } XrdNetUtils::SetAuto(XrdNetUtils::prefAuto); @@ -54,52 +47,47 @@ class XrdStorageMaker final : public StorageMaker /** Open a storage object for the given URL (protocol + path), using the @a mode bits. No temporary files are downloaded. */ - std::unique_ptr open (const std::string &proto, - const std::string &path, - int mode, - const AuxSettings& aux) const override - { + std::unique_ptr open(const std::string &proto, + const std::string &path, + int mode, + const AuxSettings &aux) const override { setDebugLevel(aux.debugLevel); setTimeout(aux.timeout); - + const StorageFactory *f = StorageFactory::get(); StorageFactory::ReadHint readHint = f->readHint(); StorageFactory::CacheHint cacheHint = f->cacheHint(); - if (readHint != StorageFactory::READ_HINT_UNBUFFERED - || cacheHint == StorageFactory::CACHE_HINT_STORAGE) + if (readHint != StorageFactory::READ_HINT_UNBUFFERED || cacheHint == StorageFactory::CACHE_HINT_STORAGE) mode &= ~IOFlags::OpenUnbuffered; else - mode |= IOFlags::OpenUnbuffered; + mode |= IOFlags::OpenUnbuffered; std::string fullpath(proto + ":" + path); auto file = std::make_unique(fullpath, mode); return f->wrapNonLocalFile(std::move(file), proto, std::string(), mode); } - void stagein (const std::string &proto, const std::string &path, - const AuxSettings& aux) const override - { + void stagein(const std::string &proto, const std::string &path, const AuxSettings &aux) const override { setDebugLevel(aux.debugLevel); setTimeout(aux.timeout); std::string fullpath(proto + ":" + path); XrdCl::URL url(fullpath); XrdCl::FileSystem fs(url); - std::vector fileList; fileList.push_back(url.GetPath()); + std::vector fileList; + fileList.push_back(url.GetPath()); auto status = fs.Prepare(fileList, XrdCl::PrepareFlags::Stage, 0, &m_null_handler); - if (!status.IsOK()) - { - edm::LogWarning("StageInError") << "XrdCl::FileSystem::Prepare failed with error '" - << status.ToStr() << "' (errNo = " << status.errNo << ")"; + if (!status.IsOK()) { + edm::LogWarning("StageInError") << "XrdCl::FileSystem::Prepare failed with error '" << status.ToStr() + << "' (errNo = " << status.errNo << ")"; } } - bool check (const std::string &proto, - const std::string &path, - const AuxSettings& aux, - IOOffset *size = nullptr) const override - { + bool check(const std::string &proto, + const std::string &path, + const AuxSettings &aux, + IOOffset *size = nullptr) const override { setDebugLevel(aux.debugLevel); setTimeout(aux.timeout); @@ -108,31 +96,29 @@ class XrdStorageMaker final : public StorageMaker XrdCl::FileSystem fs(url); XrdCl::StatInfo *stat; - if (!(fs.Stat(url.GetPath(), stat)).IsOK() || (stat == nullptr)) - { - return false; + if (!(fs.Stat(url.GetPath(), stat)).IsOK() || (stat == nullptr)) { + return false; } - if (size) *size = stat->GetSize(); + if (size) + *size = stat->GetSize(); return true; } - void setDebugLevel (unsigned int level) const - { + void setDebugLevel(unsigned int level) const { auto oldLevel = m_lastDebugLevel.load(); - if(level == oldLevel) { + if (level == oldLevel) { return; } std::lock_guard guard(m_envMutex); - if(oldLevel != m_lastDebugLevel) { + if (oldLevel != m_lastDebugLevel) { //another thread just changed this value return; } - + // 'Error' is way too low of debug level - we have interest // in warning in the default - switch (level) - { + switch (level) { case 0: XrdCl::DefaultEnv::SetLogLevel("Warning"); break; @@ -157,24 +143,22 @@ class XrdStorageMaker final : public StorageMaker m_lastDebugLevel = level; } - void setTimeout(unsigned int timeout) const - { + void setTimeout(unsigned int timeout) const { timeout = timeout ? timeout : XRD_DEFAULT_TIMEOUT; auto oldTimeout = m_lastTimeout.load(); if (oldTimeout == timeout) { return; } - + std::lock_guard guard(m_envMutex); if (oldTimeout != m_lastTimeout) { //Another thread beat us to changing the value return; } - + XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); - if (env) - { + if (env) { env->PutInt("StreamTimeout", timeout); env->PutInt("RequestTimeout", timeout); env->PutInt("ConnectionWindow", timeout); @@ -182,7 +166,7 @@ class XrdStorageMaker final : public StorageMaker // Crank down some of the connection defaults. We have more // aggressive error recovery than the default client so we // can error out sooner. - env->PutInt("ConnectionWindow", timeout/6+1); + env->PutInt("ConnectionWindow", timeout / 6 + 1); env->PutInt("ConnectionRetry", 2); } m_lastTimeout = timeout; @@ -195,6 +179,5 @@ class XrdStorageMaker final : public StorageMaker mutable std::atomic m_lastTimeout; }; -DEFINE_EDM_PLUGIN (StorageMakerFactory, XrdStorageMaker, "root"); -DEFINE_FWK_SERVICE (XrdAdaptor::XrdStatisticsService); - +DEFINE_EDM_PLUGIN(StorageMakerFactory, XrdStorageMaker, "root"); +DEFINE_FWK_SERVICE(XrdAdaptor::XrdStatisticsService); diff --git a/Utilities/XrdAdaptor/src/QualityMetric.cc b/Utilities/XrdAdaptor/src/QualityMetric.cc index 58dc82c956ab5..8265aaeabcccf 100644 --- a/Utilities/XrdAdaptor/src/QualityMetric.cc +++ b/Utilities/XrdAdaptor/src/QualityMetric.cc @@ -9,71 +9,62 @@ #ifdef __MACH__ #include #include -#define GET_CLOCK_MONOTONIC(ts) \ -{ \ - clock_serv_t cclock; \ - mach_timespec_t mts; \ - host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); \ - clock_get_time(cclock, &mts); \ - mach_port_deallocate(mach_task_self(), cclock); \ - ts.tv_sec = mts.tv_sec; \ - ts.tv_nsec = mts.tv_nsec; \ -} +#define GET_CLOCK_MONOTONIC(ts) \ + { \ + clock_serv_t cclock; \ + mach_timespec_t mts; \ + host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); \ + clock_get_time(cclock, &mts); \ + mach_port_deallocate(mach_task_self(), cclock); \ + ts.tv_sec = mts.tv_sec; \ + ts.tv_nsec = mts.tv_nsec; \ + } #else -#define GET_CLOCK_MONOTONIC(ts) \ - clock_gettime(CLOCK_MONOTONIC, &ts); +#define GET_CLOCK_MONOTONIC(ts) clock_gettime(CLOCK_MONOTONIC, &ts); #endif - using namespace XrdAdaptor; QualityMetricWatch::QualityMetricWatch(QualityMetric *parent1, QualityMetric *parent2) - : m_parent1(parent1), m_parent2(parent2) -{ - // TODO: just assuming success. - GET_CLOCK_MONOTONIC(m_start); + : m_parent1(parent1), m_parent2(parent2) { + // TODO: just assuming success. + GET_CLOCK_MONOTONIC(m_start); } -QualityMetricWatch::~QualityMetricWatch() -{ - if (m_parent1 && m_parent2) - { - timespec stop; - GET_CLOCK_MONOTONIC(stop); - - int ms = 1000*(stop.tv_sec - m_start.tv_sec) + (stop.tv_nsec - m_start.tv_nsec)/1e6; - edm::LogVerbatim("XrdAdaptorInternal") << "Finished timer after " << ms << std::endl; - m_parent1->finishWatch(stop, ms); - m_parent2->finishWatch(stop, ms); - } -} +QualityMetricWatch::~QualityMetricWatch() { + if (m_parent1 && m_parent2) { + timespec stop; + GET_CLOCK_MONOTONIC(stop); -QualityMetricWatch::QualityMetricWatch(QualityMetricWatch &&that) -{ - m_parent1 = that.m_parent1; - m_parent2 = that.m_parent2; - m_start = that.m_start; - that.m_parent1 = nullptr; - that.m_parent2 = nullptr; - that.m_start = {0, 0}; + int ms = 1000 * (stop.tv_sec - m_start.tv_sec) + (stop.tv_nsec - m_start.tv_nsec) / 1e6; + edm::LogVerbatim("XrdAdaptorInternal") << "Finished timer after " << ms << std::endl; + m_parent1->finishWatch(stop, ms); + m_parent2->finishWatch(stop, ms); + } } -void -QualityMetricWatch::swap(QualityMetricWatch &that) -{ - QualityMetric *tmp; - tmp = that.m_parent1; - that.m_parent1 = m_parent1; - m_parent1 = tmp; - tmp = that.m_parent2; - that.m_parent2 = m_parent2; - m_parent2 = tmp; - timespec tmp2; - tmp2 = that.m_start; - that.m_start = m_start; - m_start = tmp2; +QualityMetricWatch::QualityMetricWatch(QualityMetricWatch &&that) { + m_parent1 = that.m_parent1; + m_parent2 = that.m_parent2; + m_start = that.m_start; + that.m_parent1 = nullptr; + that.m_parent2 = nullptr; + that.m_start = {0, 0}; } +void QualityMetricWatch::swap(QualityMetricWatch &that) { + QualityMetric *tmp; + tmp = that.m_parent1; + that.m_parent1 = m_parent1; + m_parent1 = tmp; + tmp = that.m_parent2; + that.m_parent2 = m_parent2; + m_parent2 = tmp; + timespec tmp2; + tmp2 = that.m_start; + that.m_start = m_start; + m_start = tmp2; +} QualityMetric::QualityMetric(timespec now, int default_value) : m_value(default_value), @@ -83,119 +74,88 @@ QualityMetric::QualityMetric(timespec now, int default_value) m_interval1_val(-1), m_interval2_val(-1), m_interval3_val(-1), - m_interval4_val(-1) -{ + m_interval4_val(-1) {} + +void QualityMetric::finishWatch(timespec stop, int ms) { + std::unique_lock sentry(m_mutex); + + m_value = -1; + if (stop.tv_sec > m_interval0_start + interval_length) { + m_interval4_val = m_interval3_val; + m_interval3_val = m_interval2_val; + m_interval2_val = m_interval1_val; + m_interval1_val = m_interval0_val; + m_interval0_n = 1; + m_interval0_val = ms; + m_interval0_start = stop.tv_sec; + } else { + int num = m_interval0_val * m_interval0_n + ms; + m_interval0_n++; + m_interval0_val = num / m_interval0_n; + } } -void -QualityMetric::finishWatch(timespec stop, int ms) -{ - std::unique_lock sentry(m_mutex); - - m_value = -1; - if (stop.tv_sec > m_interval0_start+interval_length) - { - m_interval4_val = m_interval3_val; - m_interval3_val = m_interval2_val; - m_interval2_val = m_interval1_val; - m_interval1_val = m_interval0_val; - m_interval0_n = 1; - m_interval0_val = ms; - m_interval0_start = stop.tv_sec; +unsigned QualityMetric::get() { + std::unique_lock sentry(m_mutex); + + if (m_value == -1) { + unsigned den = 0; + m_value = 0; + if (m_interval0_val >= 0) { + den += 16; + m_value = 16 * m_interval0_val; } - else - { - int num = m_interval0_val * m_interval0_n + ms; - m_interval0_n++; - m_interval0_val = num / m_interval0_n; + if (m_interval1_val >= 0) { + den += 8; + m_value += 8 * m_interval1_val; } -} - -unsigned -QualityMetric::get() -{ - std::unique_lock sentry(m_mutex); - - if (m_value == -1) - { - unsigned den = 0; - m_value = 0; - if (m_interval0_val >= 0) - { - den += 16; - m_value = 16*m_interval0_val; - } - if (m_interval1_val >= 0) - { - den += 8; - m_value += 8*m_interval1_val; - } - if (m_interval2_val >= 0) - { - den += 4; - m_value += 4*m_interval2_val; - } - if (m_interval3_val >= 0) - { - den += 2; - m_value += 2*m_interval3_val; - } - if (m_interval4_val >= 0) - { - den += 1; - m_value += m_interval4_val; - } - if (den) - m_value /= den; - else - m_value = 260; + if (m_interval2_val >= 0) { + den += 4; + m_value += 4 * m_interval2_val; } - return m_value; + if (m_interval3_val >= 0) { + den += 2; + m_value += 2 * m_interval3_val; + } + if (m_interval4_val >= 0) { + den += 1; + m_value += m_interval4_val; + } + if (den) + m_value /= den; + else + m_value = 260; + } + return m_value; } - CMS_THREAD_SAFE QualityMetricFactory QualityMetricFactory::m_instance; - -std::unique_ptr -QualityMetricFactory::get(timespec now, const std::string &id) -{ - auto itFound = m_instance.m_sources.find(id); - if (itFound == m_instance.m_sources.end()) - { - // try to make a new one - std::unique_ptr source(new QualityMetricUniqueSource(now)); - auto insertResult = m_instance.m_sources.insert(std::make_pair(id, source.get())); - itFound = insertResult.first; - if (insertResult.second) - { // Insert was successful; release our reference. - source.release(); - } // Otherwise, we raced with a different thread and they won; we will delete our new QM source. - } - return itFound->second->newSource(now); +std::unique_ptr QualityMetricFactory::get(timespec now, const std::string &id) { + auto itFound = m_instance.m_sources.find(id); + if (itFound == m_instance.m_sources.end()) { + // try to make a new one + std::unique_ptr source(new QualityMetricUniqueSource(now)); + auto insertResult = m_instance.m_sources.insert(std::make_pair(id, source.get())); + itFound = insertResult.first; + if (insertResult.second) { // Insert was successful; release our reference. + source.release(); + } // Otherwise, we raced with a different thread and they won; we will delete our new QM source. + } + return itFound->second->newSource(now); } - QualityMetricSource::QualityMetricSource(QualityMetricUniqueSource &parent, timespec now, int default_value) - : QualityMetric(now, default_value), - m_parent(parent) -{} - -void -QualityMetricSource::startWatch(QualityMetricWatch & watch) -{ - QualityMetricWatch tmp(&m_parent, this); - watch.swap(tmp); + : QualityMetric(now, default_value), m_parent(parent) {} + +void QualityMetricSource::startWatch(QualityMetricWatch &watch) { + QualityMetricWatch tmp(&m_parent, this); + watch.swap(tmp); } -QualityMetricUniqueSource::QualityMetricUniqueSource(timespec now) - : QualityMetric(now) -{} +QualityMetricUniqueSource::QualityMetricUniqueSource(timespec now) : QualityMetric(now) {} -std::unique_ptr -QualityMetricUniqueSource::newSource(timespec now) -{ - std::unique_ptr child(new QualityMetricSource(*this, now, get())); - return child; +std::unique_ptr QualityMetricUniqueSource::newSource(timespec now) { + std::unique_ptr child(new QualityMetricSource(*this, now, get())); + return child; } - diff --git a/Utilities/XrdAdaptor/src/QualityMetric.h b/Utilities/XrdAdaptor/src/QualityMetric.h index 51243fcbcc72e..d5460c35c75b2 100644 --- a/Utilities/XrdAdaptor/src/QualityMetric.h +++ b/Utilities/XrdAdaptor/src/QualityMetric.h @@ -14,35 +14,35 @@ namespace XrdAdaptor { -class QualityMetric; -class QualityMetricSource; -class QualityMetricUniqueSource; + class QualityMetric; + class QualityMetricSource; + class QualityMetricUniqueSource; -class QualityMetricWatch : boost::noncopyable { -friend class QualityMetricSource; + class QualityMetricWatch : boost::noncopyable { + friend class QualityMetricSource; -public: + public: QualityMetricWatch() : m_parent1(nullptr), m_parent2(nullptr) {} QualityMetricWatch(QualityMetricWatch &&); ~QualityMetricWatch(); void swap(QualityMetricWatch &); -private: + private: QualityMetricWatch(QualityMetric *parent1, QualityMetric *parent2); timespec m_start; - edm::propagate_const m_parent1; - edm::propagate_const m_parent2; -}; + edm::propagate_const m_parent1; + edm::propagate_const m_parent2; + }; -class QualityMetric : boost::noncopyable { -friend class QualityMetricWatch; + class QualityMetric : boost::noncopyable { + friend class QualityMetricWatch; -public: - QualityMetric(timespec now, int default_value=260); + public: + QualityMetric(timespec now, int default_value = 260); unsigned get(); -private: + private: void finishWatch(timespec now, int ms); static const unsigned interval_length = 60; @@ -57,52 +57,47 @@ friend class QualityMetricWatch; int m_interval4_val; std::mutex m_mutex; -}; + }; -class QualityMetricFactory { + class QualityMetricFactory { + friend class Source; -friend class Source; - -private: - static - std::unique_ptr get(timespec now, const std::string &id); + private: + static std::unique_ptr get(timespec now, const std::string &id); CMS_THREAD_SAFE static QualityMetricFactory m_instance; - typedef tbb::concurrent_unordered_map MetricMap; + typedef tbb::concurrent_unordered_map MetricMap; MetricMap m_sources; -}; + }; -/** + /** * This QM implementation is meant to be held by each XrdAdaptor::Source * instance */ -class QualityMetricSource final : public QualityMetric { - -friend class QualityMetricUniqueSource; + class QualityMetricSource final : public QualityMetric { + friend class QualityMetricUniqueSource; -public: + public: void startWatch(QualityMetricWatch &); -private: + private: QualityMetricSource(QualityMetricUniqueSource &parent, timespec now, int default_value); QualityMetricUniqueSource &m_parent; -}; + }; -/* + /* * This quality metric tracks all accesses to a given source ID. */ -class QualityMetricUniqueSource final : public QualityMetric { + class QualityMetricUniqueSource final : public QualityMetric { + friend class QualityMetricFactory; -friend class QualityMetricFactory; - -private: + private: QualityMetricUniqueSource(timespec now); std::unique_ptr newSource(timespec now); -}; - -} + }; -#endif // Utilities_XrdAdaptor_QualityMetric_h +} // namespace XrdAdaptor +#endif // Utilities_XrdAdaptor_QualityMetric_h diff --git a/Utilities/XrdAdaptor/src/XrdFile.cc b/Utilities/XrdAdaptor/src/XrdFile.cc index 9cd7892aca5a0..ec4d44e95e279 100644 --- a/Utilities/XrdAdaptor/src/XrdFile.cc +++ b/Utilities/XrdAdaptor/src/XrdFile.cc @@ -14,86 +14,46 @@ using namespace XrdAdaptor; // To be re-enabled when the monitoring interface is back. //static const char *kCrabJobIdEnv = "CRAB_UNIQUE_JOB_ID"; -#define XRD_CL_MAX_CHUNK 512*1024 +#define XRD_CL_MAX_CHUNK 512 * 1024 #define XRD_CL_MAX_SIZE 1024 -#define XRD_CL_MAX_READ_SIZE (8*1024*1024) +#define XRD_CL_MAX_READ_SIZE (8 * 1024 * 1024) -XrdFile::XrdFile (void) - : m_offset (0), - m_size(-1), - m_close (false), - m_name(), - m_op_count(0) -{ -} +XrdFile::XrdFile(void) : m_offset(0), m_size(-1), m_close(false), m_name(), m_op_count(0) {} -XrdFile::XrdFile (const char *name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) - : m_offset (0), - m_size(-1), - m_close (false), - m_name(), - m_op_count(0) -{ - open (name, flags, perms); +XrdFile::XrdFile(const char *name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) + : m_offset(0), m_size(-1), m_close(false), m_name(), m_op_count(0) { + open(name, flags, perms); } -XrdFile::XrdFile (const std::string &name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) - : m_offset (0), - m_size(-1), - m_close (false), - m_name(), - m_op_count(0) -{ - open (name.c_str (), flags, perms); +XrdFile::XrdFile(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) + : m_offset(0), m_size(-1), m_close(false), m_name(), m_op_count(0) { + open(name.c_str(), flags, perms); } -XrdFile::~XrdFile (void) -{ +XrdFile::~XrdFile(void) { if (m_close) - edm::LogError("XrdFileError") - << "Destructor called on XROOTD file '" << m_name - << "' but the file is still open"; + edm::LogError("XrdFileError") << "Destructor called on XROOTD file '" << m_name << "' but the file is still open"; } ////////////////////////////////////////////////////////////////////// -void -XrdFile::create (const char *name, - bool exclusive /* = false */, - int perms /* = 066 */) -{ - open (name, - (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate - | (exclusive ? IOFlags::OpenExclusive : 0)), - perms); +void XrdFile::create(const char *name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name, + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), + perms); +} + +void XrdFile::create(const std::string &name, bool exclusive /* = false */, int perms /* = 066 */) { + open(name.c_str(), + (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate | (exclusive ? IOFlags::OpenExclusive : 0)), + perms); } -void -XrdFile::create (const std::string &name, - bool exclusive /* = false */, - int perms /* = 066 */) -{ - open (name.c_str (), - (IOFlags::OpenCreate | IOFlags::OpenWrite | IOFlags::OpenTruncate - | (exclusive ? IOFlags::OpenExclusive : 0)), - perms); +void XrdFile::open(const std::string &name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { + open(name.c_str(), flags, perms); } -void -XrdFile::open (const std::string &name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) -{ open (name.c_str (), flags, perms); } - -void -XrdFile::open (const char *name, - int flags /* = IOFlags::OpenRead */, - int perms /* = 066 */) -{ +void XrdFile::open(const char *name, int flags /* = IOFlags::OpenRead */, int perms /* = 066 */) { // Actual open if ((name == nullptr) || (*name == 0)) { edm::Exception ex(edm::errors::FileOpenError); @@ -123,9 +83,8 @@ XrdFile::open (const char *name, throw ex; } - if (flags & IOFlags::OpenCreate) - { - if (! (flags & IOFlags::OpenExclusive)) + if (flags & IOFlags::OpenCreate) { + if (!(flags & IOFlags::OpenExclusive)) openflags |= XrdCl::OpenFlags::Delete; openflags |= XrdCl::OpenFlags::New; openflags |= XrdCl::OpenFlags::MakePath; @@ -153,18 +112,17 @@ XrdFile::open (const char *name, auto file = getActiveFile(); XrdCl::XRootDStatus status; XrdCl::StatInfo *statInfo = nullptr; - if (! (status = file->Stat(false, statInfo)).IsOK()) { + if (!(status = file->Stat(false, statInfo)).IsOK()) { edm::Exception ex(edm::errors::FileOpenError); - ex << "XrdCl::File::Stat(name='" << name - << ") => error '" << status.ToStr() - << "' (errno=" << status.errNo << ", code=" << status.code << ")"; + ex << "XrdCl::File::Stat(name='" << name << ") => error '" << status.ToStr() << "' (errno=" << status.errNo + << ", code=" << status.code << ")"; ex.addContext("Calling XrdFile::open()"); addConnection(ex); throw ex; } assert(statInfo); m_size = statInfo->GetSize(); - delete(statInfo); + delete (statInfo); m_offset = 0; m_close = true; @@ -172,7 +130,7 @@ XrdFile::open (const char *name, // Send the monitoring info, if available. // Note: getenv is not reentrant. // Commenting out until this is available in the new client. -/* + /* char * crabJobId = getenv(kCrabJobIdEnv); if (crabJobId) { kXR_unt32 dictId; @@ -187,24 +145,19 @@ XrdFile::open (const char *name, m_requestmanager->getActiveSourceNames(sources); std::stringstream ss; ss << "Active sources: "; - for (auto const& it : sources) + for (auto const &it : sources) ss << it << ", "; edm::LogInfo("XrdFileInfo") << ss.str(); } -void -XrdFile::close (void) -{ - if (! m_requestmanager.get()) - { - edm::LogError("XrdFileError") - << "XrdFile::close(name='" << m_name - << "') called but the file is not open"; +void XrdFile::close(void) { + if (!m_requestmanager.get()) { + edm::LogError("XrdFileError") << "XrdFile::close(name='" << m_name << "') called but the file is not open"; m_close = false; return; } - m_requestmanager = nullptr; // propagate_const has no reset() function + m_requestmanager = nullptr; // propagate_const has no reset() function m_close = false; m_offset = 0; @@ -212,23 +165,18 @@ XrdFile::close (void) edm::LogInfo("XrdFileInfo") << "Closed " << m_name; } -void -XrdFile::abort (void) -{ - m_requestmanager = nullptr; // propagate_const has no reset() function +void XrdFile::abort(void) { + m_requestmanager = nullptr; // propagate_const has no reset() function m_close = false; m_offset = 0; m_size = -1; } ////////////////////////////////////////////////////////////////////// -IOSize -XrdFile::read (void *into, IOSize n) -{ +IOSize XrdFile::read(void *into, IOSize n) { if (n > 0x7fffffff) { edm::Exception ex(edm::errors::FileReadError); - ex << "XrdFile::read(name='" << m_name << "', n=" << n - << ") too many bytes, limit is 0x7fffffff"; + ex << "XrdFile::read(name='" << m_name << "', n=" << n << ") too many bytes, limit is 0x7fffffff"; ex.addContext("Calling XrdFile::read()"); addConnection(ex); throw ex; @@ -239,13 +187,10 @@ XrdFile::read (void *into, IOSize n) return bytesRead; } -IOSize -XrdFile::read (void *into, IOSize n, IOOffset pos) -{ +IOSize XrdFile::read(void *into, IOSize n, IOOffset pos) { if (n > 0x7fffffff) { edm::Exception ex(edm::errors::FileReadError); - ex << "XrdFile::read(name='" << m_name << "', n=" << n - << ") exceeds read size limit 0x7fffffff"; + ex << "XrdFile::read(name='" << m_name << "', n=" << n << ") exceeds read size limit 0x7fffffff"; ex.addContext("Calling XrdFile::read()"); addConnection(ex); throw ex; @@ -276,7 +221,7 @@ XrdFile::read (void *into, IOSize n, IOOffset pos) addConnection(ex); throw ex; } else if (result != expected) { - readReturnedShort = true; + readReturnedShort = true; } bytesRead += result; }; @@ -294,7 +239,7 @@ XrdFile::read (void *into, IOSize n, IOOffset pos) check_read(prev_future, prev_future_expected); // Update counters. - into = static_cast(into) + chunk; + into = static_cast(into) + chunk; n -= chunk; pos += chunk; } @@ -306,13 +251,11 @@ XrdFile::read (void *into, IOSize n, IOOffset pos) } // This method is rarely used by CMS; hence, it is a small wrapper and not efficient. -IOSize -XrdFile::readv (IOBuffer *into, IOSize n) -{ +IOSize XrdFile::readv(IOBuffer *into, IOSize n) { std::vector new_buf; new_buf.reserve(n); IOOffset off = 0; - for (IOSize i=0; i, IOSize>> readv_futures; - while (idx < n) - { + while (idx < n) { cl->clear(); IOSize size = 0; while (idx < n) { @@ -355,7 +295,7 @@ XrdFile::readv (IOPosBuffer *into, IOSize n) IOOffset offset = into[idx].offset(); IOSize length = into[idx].size(); size += length; - char * buffer = static_cast(into[idx].data()); + char *buffer = static_cast(into[idx].data()); while (length > XRD_CL_MAX_CHUNK) { IOPosBuffer ci; ci.set_size(XRD_CL_MAX_CHUNK); @@ -365,7 +305,7 @@ XrdFile::readv (IOPosBuffer *into, IOSize n) ci.set_data(buffer); buffer += XRD_CL_MAX_CHUNK; cl->emplace_back(ci); - rollback_count ++; + rollback_count++; } IOPosBuffer ci; ci.set_size(length); @@ -373,23 +313,18 @@ XrdFile::readv (IOPosBuffer *into, IOSize n) ci.set_data(buffer); cl->emplace_back(ci); - if (cl->size() > adjust) - { - while (rollback_count--) cl->pop_back(); + if (cl->size() > adjust) { + while (rollback_count--) + cl->pop_back(); size = current_size; break; - } - else - { + } else { idx++; } } - try - { + try { readv_futures.emplace_back(m_requestmanager->handle(cl), size); - } - catch (edm::Exception& ex) - { + } catch (edm::Exception &ex) { ex.addContext("Calling XrdFile::readv()"); throw; } @@ -397,66 +332,49 @@ XrdFile::readv (IOPosBuffer *into, IOSize n) // Assure that we have made some progress. assert(last_idx < idx); last_idx = idx; - } std::chrono::time_point start, end; start = std::chrono::high_resolution_clock::now(); - // If there are multiple readv calls, wait until all return until looking - // at the results of any. This guarantees that all readv's have finished - // by time we call .get() for the first time (in case one of the readv's - // result in an exception). - // - // We cannot have outstanding readv's on function exit as the XrdCl may - // write into the corresponding buffer at the same time as ROOT. - if (readv_futures.size() > 1) - { - for (auto & readv_result : readv_futures) - { - if (readv_result.first.valid()) - { + // If there are multiple readv calls, wait until all return until looking + // at the results of any. This guarantees that all readv's have finished + // by time we call .get() for the first time (in case one of the readv's + // result in an exception). + // + // We cannot have outstanding readv's on function exit as the XrdCl may + // write into the corresponding buffer at the same time as ROOT. + if (readv_futures.size() > 1) { + for (auto &readv_result : readv_futures) { + if (readv_result.first.valid()) { readv_result.first.wait(); } } } - for (auto & readv_result : readv_futures) - { + for (auto &readv_result : readv_futures) { IOSize result = 0; - try - { + try { const int retry_count = 5; - for (int retries=0; retrieshandle(cl).get(); - } - else - { + } else { throw; } } assert(result == readv_result.second); } - } - catch (edm::Exception& ex) - { + } catch (edm::Exception &ex) { ex.addContext("Calling XrdFile::readv()"); throw; - } - catch (std::exception& ex) - { + } catch (std::exception &ex) { edm::Exception newex(edm::errors::StdException); newex << "A std::exception was thrown when processing an xrootd request: " << ex.what(); newex.addContext("Calling XrdFile::readv()"); @@ -466,18 +384,18 @@ XrdFile::readv (IOPosBuffer *into, IOSize n) } end = std::chrono::high_resolution_clock::now(); - edm::LogVerbatim("XrdAdaptorInternal") << "[" << m_op_count.fetch_add(1) << "] Time for readv: " << static_cast(std::chrono::duration_cast(end-start).count()) << " (sub-readv requests: " << readv_futures.size() << ")" << std::endl; + edm::LogVerbatim("XrdAdaptorInternal") + << "[" << m_op_count.fetch_add(1) << "] Time for readv: " + << static_cast(std::chrono::duration_cast(end - start).count()) + << " (sub-readv requests: " << readv_futures.size() << ")" << std::endl; return final_result; } -IOSize -XrdFile::write (const void *from, IOSize n) -{ +IOSize XrdFile::write(const void *from, IOSize n) { if (n > 0x7fffffff) { edm::Exception ex(edm::errors::FileWriteError); - ex << "XrdFile::write(name='" << m_name << "', n=" << n - << ") too many bytes, limit is 0x7fffffff"; + ex << "XrdFile::write(name='" << m_name << "', n=" << n << ") too many bytes, limit is 0x7fffffff"; ex.addContext("Calling XrdFile::write()"); addConnection(ex); throw ex; @@ -487,8 +405,7 @@ XrdFile::write (const void *from, IOSize n) XrdCl::XRootDStatus s = file->Write(m_offset, n, from); if (!s.IsOK()) { edm::Exception ex(edm::errors::FileWriteError); - ex << "XrdFile::write(name='" << m_name << "', n=" << n - << ") failed with error '" << s.ToStr() + ex << "XrdFile::write(name='" << m_name << "', n=" << n << ") failed with error '" << s.ToStr() << "' (errno=" << s.errNo << ", code=" << s.code << ")"; ex.addContext("Calling XrdFile::write()"); addConnection(ex); @@ -502,13 +419,10 @@ XrdFile::write (const void *from, IOSize n) return n; } -IOSize -XrdFile::write (const void *from, IOSize n, IOOffset pos) -{ +IOSize XrdFile::write(const void *from, IOSize n, IOOffset pos) { if (n > 0x7fffffff) { edm::Exception ex(edm::errors::FileWriteError); - ex << "XrdFile::write(name='" << m_name << "', n=" << n - << ") too many bytes, limit is 0x7fffffff"; + ex << "XrdFile::write(name='" << m_name << "', n=" << n << ") too many bytes, limit is 0x7fffffff"; ex.addContext("Calling XrdFile::write()"); addConnection(ex); throw ex; @@ -518,23 +432,20 @@ XrdFile::write (const void *from, IOSize n, IOOffset pos) XrdCl::XRootDStatus s = file->Write(pos, n, from); if (!s.IsOK()) { edm::Exception ex(edm::errors::FileWriteError); - ex << "XrdFile::write(name='" << m_name << "', n=" << n - << ") failed with error '" << s.ToStr() + ex << "XrdFile::write(name='" << m_name << "', n=" << n << ") failed with error '" << s.ToStr() << "' (errno=" << s.errNo << ", code=" << s.code << ")"; ex.addContext("Calling XrdFile::write()"); addConnection(ex); throw ex; } - assert (m_size != -1); + assert(m_size != -1); if (static_cast(pos + n) > m_size) m_size = pos + n; return n; } -bool -XrdFile::prefetch (const IOPosBuffer *what, IOSize n) -{ +bool XrdFile::prefetch(const IOPosBuffer *what, IOSize n) { // The new Xrootd client does not contain any internal buffers. // Hence, prefetching is disabled completely. return false; @@ -543,38 +454,35 @@ XrdFile::prefetch (const IOPosBuffer *what, IOSize n) ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////// -IOOffset -XrdFile::position (IOOffset offset, Relative whence /* = SET */) -{ - if (! m_requestmanager.get()) { +IOOffset XrdFile::position(IOOffset offset, Relative whence /* = SET */) { + if (!m_requestmanager.get()) { cms::Exception ex("FilePositionError"); ex << "XrdFile::position() called on a closed file"; ex.addContext("Calling XrdFile::position()"); addConnection(ex); throw ex; } - switch (whence) - { - case SET: - m_offset = offset; - break; - - case CURRENT: - m_offset += offset; - break; - - // TODO: None of this works with concurrent writers to the file. - case END: - assert(m_size != -1); - m_offset = m_size + offset; - break; - - default: - cms::Exception ex("FilePositionError"); - ex << "XrdFile::position() called with incorrect 'whence' parameter"; - ex.addContext("Calling XrdFile::position()"); - addConnection(ex); - throw ex; + switch (whence) { + case SET: + m_offset = offset; + break; + + case CURRENT: + m_offset += offset; + break; + + // TODO: None of this works with concurrent writers to the file. + case END: + assert(m_size != -1); + m_offset = m_size + offset; + break; + + default: + cms::Exception ex("FilePositionError"); + ex << "XrdFile::position() called with incorrect 'whence' parameter"; + ex.addContext("Calling XrdFile::position()"); + addConnection(ex); + throw ex; } if (m_offset < 0) @@ -586,9 +494,7 @@ XrdFile::position (IOOffset offset, Relative whence /* = SET */) return m_offset; } -void -XrdFile::resize (IOOffset /* size */) -{ +void XrdFile::resize(IOOffset /* size */) { cms::Exception ex("FileResizeError"); ex << "XrdFile::resize(name='" << m_name << "') not implemented"; ex.addContext("Calling XrdFile::resize()"); @@ -596,11 +502,8 @@ XrdFile::resize (IOOffset /* size */) throw ex; } -std::shared_ptr -XrdFile::getActiveFile (void) -{ - if (!m_requestmanager.get()) - { +std::shared_ptr XrdFile::getActiveFile(void) { + if (!m_requestmanager.get()) { cms::Exception ex("XrdFileLogicError"); ex << "Xrd::getActiveFile(name='" << m_name << "') no active request manager"; ex.addContext("Calling XrdFile::getActiveFile()"); @@ -611,12 +514,8 @@ XrdFile::getActiveFile (void) return m_requestmanager->getActiveFile(); } -void -XrdFile::addConnection (cms::Exception &ex) -{ - if (m_requestmanager.get()) - { +void XrdFile::addConnection(cms::Exception &ex) { + if (m_requestmanager.get()) { m_requestmanager->addConnections(ex); } } - diff --git a/Utilities/XrdAdaptor/src/XrdFile.h b/Utilities/XrdAdaptor/src/XrdFile.h index c0b3922ed1ac9..6079f15fbfd92 100644 --- a/Utilities/XrdAdaptor/src/XrdFile.h +++ b/Utilities/XrdAdaptor/src/XrdFile.h @@ -1,77 +1,66 @@ #ifndef Utilities_XrdAdaptor_XrdFile_h #define Utilities_XrdAdaptor_XrdFile_h -# include "Utilities/StorageFactory/interface/Storage.h" -# include "Utilities/StorageFactory/interface/IOFlags.h" -# include "FWCore/Utilities/interface/Exception.h" -# include "FWCore/Utilities/interface/propagate_const.h" -# include "XrdCl/XrdClFile.hh" -# include -# include -# include +#include "Utilities/StorageFactory/interface/Storage.h" +#include "Utilities/StorageFactory/interface/IOFlags.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "XrdCl/XrdClFile.hh" +#include +#include +#include namespace XrdAdaptor { -class RequestManager; + class RequestManager; } -class XrdFile : public Storage -{ +class XrdFile : public Storage { public: - XrdFile (void); - XrdFile (IOFD fd); - XrdFile (const char *name, int flags = IOFlags::OpenRead, int perms = 0666); - XrdFile (const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); - ~XrdFile (void) override; + XrdFile(void); + XrdFile(IOFD fd); + XrdFile(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + XrdFile(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + ~XrdFile(void) override; - virtual void create (const char *name, - bool exclusive = false, - int perms = 0666); - virtual void create (const std::string &name, - bool exclusive = false, - int perms = 0666); - virtual void open (const char *name, - int flags = IOFlags::OpenRead, - int perms = 0666); - virtual void open (const std::string &name, - int flags = IOFlags::OpenRead, - int perms = 0666); + virtual void create(const char *name, bool exclusive = false, int perms = 0666); + virtual void create(const std::string &name, bool exclusive = false, int perms = 0666); + virtual void open(const char *name, int flags = IOFlags::OpenRead, int perms = 0666); + virtual void open(const std::string &name, int flags = IOFlags::OpenRead, int perms = 0666); + using Storage::position; using Storage::read; using Storage::readv; using Storage::write; - using Storage::position; - bool prefetch (const IOPosBuffer *what, IOSize n) override; - IOSize read (void *into, IOSize n) override; - IOSize read (void *into, IOSize n, IOOffset pos) override; - IOSize readv (IOBuffer *into, IOSize n) override; - IOSize readv (IOPosBuffer *into, IOSize n) override; - IOSize write (const void *from, IOSize n) override; - IOSize write (const void *from, IOSize n, IOOffset pos) override; + bool prefetch(const IOPosBuffer *what, IOSize n) override; + IOSize read(void *into, IOSize n) override; + IOSize read(void *into, IOSize n, IOOffset pos) override; + IOSize readv(IOBuffer *into, IOSize n) override; + IOSize readv(IOPosBuffer *into, IOSize n) override; + IOSize write(const void *from, IOSize n) override; + IOSize write(const void *from, IOSize n, IOOffset pos) override; - IOOffset position (IOOffset offset, Relative whence = SET) override; - void resize (IOOffset size) override; + IOOffset position(IOOffset offset, Relative whence = SET) override; + void resize(IOOffset size) override; - void close (void) override; - virtual void abort (void); + void close(void) override; + virtual void abort(void); private: - - void addConnection(cms::Exception &); + void addConnection(cms::Exception &); /** * Returns a file handle from one of the active sources. * Verifies the file is open and throws an exception as necessary. */ - std::shared_ptr getActiveFile(); + std::shared_ptr getActiveFile(); edm::propagate_const> m_requestmanager; - IOOffset m_offset; - IOOffset m_size; - bool m_close; - std::string m_name; - std::atomic m_op_count; - + IOOffset m_offset; + IOOffset m_size; + bool m_close; + std::string m_name; + std::atomic m_op_count; }; -#endif // XRD_ADAPTOR_XRD_FILE_H +#endif // XRD_ADAPTOR_XRD_FILE_H diff --git a/Utilities/XrdAdaptor/src/XrdHostHandler.hh b/Utilities/XrdAdaptor/src/XrdHostHandler.hh index 0628a8cfdaace..f9546a2437ef3 100644 --- a/Utilities/XrdAdaptor/src/XrdHostHandler.hh +++ b/Utilities/XrdAdaptor/src/XrdHostHandler.hh @@ -5,11 +5,11 @@ #include "FWCore/Utilities/interface/get_underlying_safe.h" #if defined(__linux__) - #define HAVE_ATOMICS 1 - #include "XrdSys/XrdSysLinuxSemaphore.hh" - typedef XrdSys::LinuxSemaphore Semaphore; +#define HAVE_ATOMICS 1 +#include "XrdSys/XrdSysLinuxSemaphore.hh" +typedef XrdSys::LinuxSemaphore Semaphore; #else - typedef XrdSysSemaphore Semaphore; +typedef XrdSysSemaphore Semaphore; #endif /** @@ -18,21 +18,13 @@ * utilize. */ -class SyncHostResponseHandler: public XrdCl::ResponseHandler -{ +class SyncHostResponseHandler : public XrdCl::ResponseHandler { public: - - SyncHostResponseHandler(): - sem(0) - { - } + SyncHostResponseHandler() : sem(0) {} ~SyncHostResponseHandler() override = default; - - void HandleResponse(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response) override - { + void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override { // propagate_const has no reset() function pStatus_ = std::unique_ptr(status); pResponse_ = std::unique_ptr(response); @@ -40,9 +32,8 @@ public: } void HandleResponseWithHosts(XrdCl::XRootDStatus *status, - XrdCl::AnyObject *response, - XrdCl::HostList *hostList) override - { + XrdCl::AnyObject *response, + XrdCl::HostList *hostList) override { // propagate_const has no reset() function pStatus_ = std::unique_ptr(status); pResponse_ = std::unique_ptr(response); @@ -50,32 +41,19 @@ public: sem.Post(); } - std::unique_ptr GetStatus() - { - return std::move(get_underlying_safe(pStatus_)); - } + std::unique_ptr GetStatus() { return std::move(get_underlying_safe(pStatus_)); } - std::unique_ptr GetResponse() - { - return std::move(get_underlying_safe(pResponse_)); - } + std::unique_ptr GetResponse() { return std::move(get_underlying_safe(pResponse_)); } - std::unique_ptr GetHosts() - { - return std::move(get_underlying_safe(pHostList_)); - } + std::unique_ptr GetHosts() { return std::move(get_underlying_safe(pHostList_)); } - void WaitForResponse() - { - sem.Wait(); - } + void WaitForResponse() { sem.Wait(); } private: - edm::propagate_const> pStatus_; - edm::propagate_const> pResponse_; - edm::propagate_const> pHostList_; - Semaphore sem; + edm::propagate_const> pResponse_; + edm::propagate_const> pHostList_; + Semaphore sem; }; #endif diff --git a/Utilities/XrdAdaptor/src/XrdRequest.cc b/Utilities/XrdAdaptor/src/XrdRequest.cc index eb4c6e19bf7e5..f667a30d3b5f2 100644 --- a/Utilities/XrdAdaptor/src/XrdRequest.cc +++ b/Utilities/XrdAdaptor/src/XrdRequest.cc @@ -12,112 +12,94 @@ using namespace XrdAdaptor; // If you define XRD_FAKE_ERROR, 1/5 read requests should fail. #ifdef XRD_FAKE_ERROR #define FAKE_ERROR_COUNTER 5 -static std::atomic g_fakeError {0}; +static std::atomic g_fakeError{0}; #else #define FAKE_ERROR_COUNTER 0 -static std::atomic g_fakeError {0}; +static std::atomic g_fakeError{0}; #endif XrdAdaptor::ClientRequest::~ClientRequest() {} -void -XrdAdaptor::ClientRequest::HandleResponse(XrdCl::XRootDStatus *stat, XrdCl::AnyObject *resp) -{ - std::unique_ptr response(resp); - std::unique_ptr status(stat); - std::shared_ptr self_ref = self_reference(); - m_self_reference = nullptr; // propagate_const has no reset() function - { - QualityMetricWatch qmw; - m_qmw.swap(qmw); - } - m_stats = nullptr; // propagate_const has no reset() function +void XrdAdaptor::ClientRequest::HandleResponse(XrdCl::XRootDStatus *stat, XrdCl::AnyObject *resp) { + std::unique_ptr response(resp); + std::unique_ptr status(stat); + std::shared_ptr self_ref = self_reference(); + m_self_reference = nullptr; // propagate_const has no reset() function + { + QualityMetricWatch qmw; + m_qmw.swap(qmw); + } + m_stats = nullptr; // propagate_const has no reset() function - if ((!FAKE_ERROR_COUNTER || ((++g_fakeError % FAKE_ERROR_COUNTER) != 0)) && (status->IsOK() && resp)) - { - if (m_into) - { - XrdCl::ChunkInfo *read_info; - response->Get(read_info); - if( read_info == nullptr) { - edm::Exception ex(edm::errors::FileReadError); - ex<<"nullptr returned from response->Get()."; - ex.addContext("XrdAdaptor::ClientRequest::HandleResponse() called." ); - m_promise.set_exception( std::make_exception_ptr(ex)); - return; - } - if( read_info->length == 0) {edm::LogWarning("XrdAdaptorInternal") << "XrdAdaptor::ClientRequest::HandleResponse: While reading from\n " - << m_manager.getFilename() << "\n received a read_info->length = 0 and read_info->offset = "<offset; - } - m_promise.set_value(read_info->length); - } - else - { - XrdCl::VectorReadInfo *read_info; - response->Get(read_info); - if( read_info == nullptr) { - edm::Exception ex(edm::errors::FileReadError); - ex<<"nullptr returned from response->Get() from vector read."; - ex.addContext("XrdAdaptor::ClientRequest::HandleResponse() called." ); - m_promise.set_exception( std::make_exception_ptr(ex)); - return; - } - if( read_info->GetSize() == 0) {edm::LogWarning("XrdAdaptorInternal") << "XrdAdaptor::ClientRequest::HandleResponse: While reading from\n " - << m_manager.getFilename() << "\n received a read_info->GetSize() = 0"; - } + if ((!FAKE_ERROR_COUNTER || ((++g_fakeError % FAKE_ERROR_COUNTER) != 0)) && (status->IsOK() && resp)) { + if (m_into) { + XrdCl::ChunkInfo *read_info; + response->Get(read_info); + if (read_info == nullptr) { + edm::Exception ex(edm::errors::FileReadError); + ex << "nullptr returned from response->Get()."; + ex.addContext("XrdAdaptor::ClientRequest::HandleResponse() called."); + m_promise.set_exception(std::make_exception_ptr(ex)); + return; + } + if (read_info->length == 0) { + edm::LogWarning("XrdAdaptorInternal") + << "XrdAdaptor::ClientRequest::HandleResponse: While reading from\n " << m_manager.getFilename() + << "\n received a read_info->length = 0 and read_info->offset = " << read_info->offset; + } + m_promise.set_value(read_info->length); + } else { + XrdCl::VectorReadInfo *read_info; + response->Get(read_info); + if (read_info == nullptr) { + edm::Exception ex(edm::errors::FileReadError); + ex << "nullptr returned from response->Get() from vector read."; + ex.addContext("XrdAdaptor::ClientRequest::HandleResponse() called."); + m_promise.set_exception(std::make_exception_ptr(ex)); + return; + } + if (read_info->GetSize() == 0) { + edm::LogWarning("XrdAdaptorInternal") << "XrdAdaptor::ClientRequest::HandleResponse: While reading from\n " + << m_manager.getFilename() << "\n received a read_info->GetSize() = 0"; + } - m_promise.set_value(read_info->GetSize()); - } + m_promise.set_value(read_info->GetSize()); } - else - { - Source *source = m_source.get(); - edm::LogWarning("XrdAdaptorInternal") << "XrdRequestManager::handle(name='" - << m_manager.getFilename() << ") failure when reading from " - << (source ? source->PrettyID() : "(unknown source)") - << "; failed with error '" << status->ToStr() << "' (errno=" - << status->errNo << ", code=" << status->code << ")."; - m_failure_count++; - try - { - try - { - m_manager.requestFailure(self_ref, *status); - return; - } - catch (XrootdException& ex) - { - if (ex.getCode() == XrdCl::errInvalidResponse) - { - m_promise.set_exception(std::current_exception()); - } - else throw; - } - } - catch (edm::Exception& ex) - { - ex.addContext("XrdAdaptor::ClientRequest::HandleResponse() failure while running connection recovery"); - std::stringstream ss; - ss << "Original error: '" << status->ToStr() << "' (errno=" - << status->errNo << ", code=" << status->code << ", source=" << (source ? source->PrettyID() : "(unknown source)") << ")."; - ex.addAdditionalInfo(ss.str()); - m_promise.set_exception(std::current_exception()); - edm::LogWarning("XrdAdaptorInternal") << "Caught a CMSSW exception when running connection recovery."; - } - catch (std::exception const&) - { - edm::Exception ex(edm::errors::FileReadError); - ex << "XrdRequestManager::handle(name='" << m_manager.getFilename() - << ") failed with error '" << status->ToStr() - << "' (errno=" << status->errNo << ", code=" - << status->code << "). Unknown exception occurred when running" - << " connection recovery."; - ex.addContext("Calling XrdRequestManager::handle()"); - m_manager.addConnections(ex); - ex.addAdditionalInfo("Original source of error is " + (source ? source->PrettyID() : "(unknown source)")); - m_promise.set_exception(std::make_exception_ptr(ex)); - edm::LogWarning("XrdAdaptorInternal") << "Caught a new exception when running connection recovery."; - } + } else { + Source *source = m_source.get(); + edm::LogWarning("XrdAdaptorInternal") + << "XrdRequestManager::handle(name='" << m_manager.getFilename() << ") failure when reading from " + << (source ? source->PrettyID() : "(unknown source)") << "; failed with error '" << status->ToStr() + << "' (errno=" << status->errNo << ", code=" << status->code << ")."; + m_failure_count++; + try { + try { + m_manager.requestFailure(self_ref, *status); + return; + } catch (XrootdException &ex) { + if (ex.getCode() == XrdCl::errInvalidResponse) { + m_promise.set_exception(std::current_exception()); + } else + throw; + } + } catch (edm::Exception &ex) { + ex.addContext("XrdAdaptor::ClientRequest::HandleResponse() failure while running connection recovery"); + std::stringstream ss; + ss << "Original error: '" << status->ToStr() << "' (errno=" << status->errNo << ", code=" << status->code + << ", source=" << (source ? source->PrettyID() : "(unknown source)") << ")."; + ex.addAdditionalInfo(ss.str()); + m_promise.set_exception(std::current_exception()); + edm::LogWarning("XrdAdaptorInternal") << "Caught a CMSSW exception when running connection recovery."; + } catch (std::exception const &) { + edm::Exception ex(edm::errors::FileReadError); + ex << "XrdRequestManager::handle(name='" << m_manager.getFilename() << ") failed with error '" << status->ToStr() + << "' (errno=" << status->errNo << ", code=" << status->code << "). Unknown exception occurred when running" + << " connection recovery."; + ex.addContext("Calling XrdRequestManager::handle()"); + m_manager.addConnections(ex); + ex.addAdditionalInfo("Original source of error is " + (source ? source->PrettyID() : "(unknown source)")); + m_promise.set_exception(std::make_exception_ptr(ex)); + edm::LogWarning("XrdAdaptorInternal") << "Caught a new exception when running connection recovery."; } + } } - diff --git a/Utilities/XrdAdaptor/src/XrdRequest.h b/Utilities/XrdAdaptor/src/XrdRequest.h index 0e4b28599319f..b1d64135a81de 100644 --- a/Utilities/XrdAdaptor/src/XrdRequest.h +++ b/Utilities/XrdAdaptor/src/XrdRequest.h @@ -14,76 +14,53 @@ namespace XrdAdaptor { -class Source; + class Source; -class RequestManager; + class RequestManager; -class XrdReadStatistics; + class XrdReadStatistics; -class ClientRequest : boost::noncopyable, public XrdCl::ResponseHandler { - -friend class Source; - -public: + class ClientRequest : boost::noncopyable, public XrdCl::ResponseHandler { + friend class Source; + public: ClientRequest(RequestManager &manager, void *into, IOSize size, IOOffset off) - : m_failure_count(0), - m_into(into), - m_size(size), - m_off(off), - m_iolist(nullptr), - m_manager(manager) - { - } + : m_failure_count(0), m_into(into), m_size(size), m_off(off), m_iolist(nullptr), m_manager(manager) {} - ClientRequest(RequestManager &manager, std::shared_ptr > iolist, IOSize size=0) - : m_failure_count(0), - m_into(nullptr), - m_size(size), - m_off(0), - m_iolist(iolist), - m_manager(manager) - { - if (!m_iolist->empty() && !m_size) - { - for (IOPosBuffer const & buf : *m_iolist) - { - m_size += buf.size(); - } + ClientRequest(RequestManager &manager, std::shared_ptr> iolist, IOSize size = 0) + : m_failure_count(0), m_into(nullptr), m_size(size), m_off(0), m_iolist(iolist), m_manager(manager) { + if (!m_iolist->empty() && !m_size) { + for (IOPosBuffer const &buf : *m_iolist) { + m_size += buf.size(); } + } } - void setStatistics(std::shared_ptr stats) - { - m_stats = stats; - } + void setStatistics(std::shared_ptr stats) { m_stats = stats; } ~ClientRequest() override; - std::future get_future() - { - return m_promise.get_future(); - } + std::future get_future() { return m_promise.get_future(); } /** * Handle the response from the Xrootd server. */ void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override; - IOSize getSize() const {return m_size;} + IOSize getSize() const { return m_size; } - size_t getCount() const {return m_into ? 1 : m_iolist->size();} + size_t getCount() const { return m_into ? 1 : m_iolist->size(); } /** * Returns a pointer to the current source; may be nullptr * if there is no outstanding IO */ - std::shared_ptr getCurrentSource() const {return get_underlying_safe(m_source);} - std::shared_ptr& getCurrentSource() {return get_underlying_safe(m_source);} + std::shared_ptr getCurrentSource() const { return get_underlying_safe(m_source); } + std::shared_ptr &getCurrentSource() { return get_underlying_safe(m_source); } -private: - std::shared_ptr self_reference() const {return get_underlying_safe(m_self_reference);} - std::shared_ptr& self_reference() {return get_underlying_safe(m_self_reference);} + private: + std::shared_ptr self_reference() const { return get_underlying_safe(m_self_reference); } + std::shared_ptr &self_reference() { return get_underlying_safe(m_self_reference); } unsigned m_failure_count; void *m_into; @@ -104,8 +81,8 @@ friend class Source; std::promise m_promise; QualityMetricWatch m_qmw; -}; + }; -} +} // namespace XrdAdaptor #endif diff --git a/Utilities/XrdAdaptor/src/XrdRequestManager.cc b/Utilities/XrdAdaptor/src/XrdRequestManager.cc index 0f1856fd71061..61b1751eabda0 100644 --- a/Utilities/XrdAdaptor/src/XrdRequestManager.cc +++ b/Utilities/XrdAdaptor/src/XrdRequestManager.cc @@ -20,7 +20,7 @@ #include "Utilities/XrdAdaptor/src/XrdRequestManager.h" #include "Utilities/XrdAdaptor/src/XrdHostHandler.hh" -#define XRD_CL_MAX_CHUNK 512*1024 +#define XRD_CL_MAX_CHUNK 512 * 1024 #define XRD_ADAPTOR_SHORT_OPEN_DELAY 5 @@ -31,35 +31,32 @@ #define XRD_ADAPTOR_SOURCE_QUALITY_FUDGE 0 #else #define XRD_ADAPTOR_OPEN_PROBE_PERCENT 10 -#define XRD_ADAPTOR_LONG_OPEN_DELAY 2*60 +#define XRD_ADAPTOR_LONG_OPEN_DELAY 2 * 60 #define XRD_ADAPTOR_SOURCE_QUALITY_FUDGE 100 #endif #define XRD_ADAPTOR_CHUNK_THRESHOLD 1000 - #ifdef __MACH__ #include #include -#define GET_CLOCK_MONOTONIC(ts) \ -{ \ - clock_serv_t cclock; \ - mach_timespec_t mts; \ - host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); \ - clock_get_time(cclock, &mts); \ - mach_port_deallocate(mach_task_self(), cclock); \ - ts.tv_sec = mts.tv_sec; \ - ts.tv_nsec = mts.tv_nsec; \ -} +#define GET_CLOCK_MONOTONIC(ts) \ + { \ + clock_serv_t cclock; \ + mach_timespec_t mts; \ + host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); \ + clock_get_time(cclock, &mts); \ + mach_port_deallocate(mach_task_self(), cclock); \ + ts.tv_sec = mts.tv_sec; \ + ts.tv_nsec = mts.tv_nsec; \ + } #else -#define GET_CLOCK_MONOTONIC(ts) \ - clock_gettime(CLOCK_MONOTONIC, &ts); +#define GET_CLOCK_MONOTONIC(ts) clock_gettime(CLOCK_MONOTONIC, &ts); #endif using namespace XrdAdaptor; -long long timeDiffMS(const timespec &a, const timespec &b) -{ +long long timeDiffMS(const timespec &a, const timespec &b) { long long diff = (a.tv_sec - b.tv_sec) * 1000; diff += (a.tv_nsec - b.tv_nsec) / 1e6; return diff; @@ -69,51 +66,45 @@ long long timeDiffMS(const timespec &a, const timespec &b) * We do not care about the response of sending the monitoring information; * this handler class simply frees any returned buffer to prevent memory leaks. */ -class SendMonitoringInfoHandler : boost::noncopyable, public XrdCl::ResponseHandler -{ - void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override - { - if (response) - { - XrdCl::Buffer *buffer = nullptr; - response->Get(buffer); - response->Set(static_cast(nullptr)); - delete buffer; - } - // Send Info has a response object; we must delete it. - delete response; - delete status; +class SendMonitoringInfoHandler : boost::noncopyable, public XrdCl::ResponseHandler { + void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override { + if (response) { + XrdCl::Buffer *buffer = nullptr; + response->Get(buffer); + response->Set(static_cast(nullptr)); + delete buffer; } + // Send Info has a response object; we must delete it. + delete response; + delete status; + } }; CMS_THREAD_SAFE SendMonitoringInfoHandler nullHandler; +static void SendMonitoringInfo(XrdCl::File &file) { + // Do not send this to a dCache data server as they return an error. + // In some versions of dCache, sending the monitoring information causes + // the server to close the connection - resulting in failures. + if (Source::isDCachePool(file)) { + return; + } -static void -SendMonitoringInfo(XrdCl::File &file) -{ - // Do not send this to a dCache data server as they return an error. - // In some versions of dCache, sending the monitoring information causes - // the server to close the connection - resulting in failures. - if (Source::isDCachePool(file)) {return;} - - // Send the monitoring info, if available. - const char * jobId = edm::storage::StatisticsSenderService::getJobID(); - std::string lastUrl; - file.GetProperty("LastURL", lastUrl); - if (jobId && !lastUrl.empty()) - { - XrdCl::URL url(lastUrl); - XrdCl::FileSystem fs(url); - if (!(fs.SendInfo(jobId, &nullHandler, 30).IsOK())) - { - edm::LogWarning("XrdAdaptorInternal") << "Failed to send the monitoring information, monitoring ID is " << jobId << "."; - } - edm::LogInfo("XrdAdaptorInternal") << "Set monitoring ID to " << jobId << "."; + // Send the monitoring info, if available. + const char *jobId = edm::storage::StatisticsSenderService::getJobID(); + std::string lastUrl; + file.GetProperty("LastURL", lastUrl); + if (jobId && !lastUrl.empty()) { + XrdCl::URL url(lastUrl); + XrdCl::FileSystem fs(url); + if (!(fs.SendInfo(jobId, &nullHandler, 30).IsOK())) { + edm::LogWarning("XrdAdaptorInternal") + << "Failed to send the monitoring information, monitoring ID is " << jobId << "."; } + edm::LogInfo("XrdAdaptorInternal") << "Set monitoring ID to " << jobId << "."; + } } - RequestManager::RequestManager(const std::string &filename, XrdCl::OpenFlags::Flags flags, XrdCl::Access::Mode perms) : m_serverToAdvertise(nullptr), m_timeout(XRD_DEFAULT_TIMEOUT), @@ -121,26 +112,21 @@ RequestManager::RequestManager(const std::string &filename, XrdCl::OpenFlags::Fl m_name(filename), m_flags(flags), m_perms(perms), - m_distribution(0,100), - m_excluded_active_count(0) -{ -} - + m_distribution(0, 100), + m_excluded_active_count(0) {} -void -RequestManager::initialize(std::weak_ptr self) -{ +void RequestManager::initialize(std::weak_ptr self) { m_open_handler = OpenHandler::getInstance(self); XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); - if (env) {env->GetInt("StreamErrorWindow", m_timeout);} + if (env) { + env->GetInt("StreamErrorWindow", m_timeout); + } std::string orig_site; - if (!Source::getXrootdSiteFromURL(m_name, orig_site) && (orig_site.find(".") == std::string::npos)) - { + if (!Source::getXrootdSiteFromURL(m_name, orig_site) && (orig_site.find(".") == std::string::npos)) { std::string hostname; - if (Source::getHostname(orig_site, hostname)) - { + if (Source::getHostname(orig_site, hostname)) { Source::getDomain(hostname, orig_site); } } @@ -150,26 +136,24 @@ RequestManager::initialize(std::weak_ptr self) bool validFile = false; const int retries = 5; std::string excludeString; - for (int idx=0; idxOpen(new_filename, m_flags, m_perms, &handler); - if (!openStatus.IsOK()) - { // In this case, we failed immediately - this indicates we have previously tried to talk to this + if (!openStatus + .IsOK()) { // In this case, we failed immediately - this indicates we have previously tried to talk to this // server and it was marked bad - xrootd couldn't even queue up the request internally! // In practice, we obsere this happening when the call to getXrootdSiteFromURL fails due to the // redirector being down or authentication failures. ex.clearMessage(); ex.clearContext(); ex.clearAdditionalInfo(); - ex << "XrdCl::File::Open(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ") => error '" << openStatus.ToStr() - << "' (errno=" << openStatus.errNo << ", code=" << openStatus.code << ")"; + ex << "XrdCl::File::Open(name='" << m_name << "', flags=0x" << std::hex << m_flags << ", permissions=0" + << std::oct << m_perms << std::dec << ") => error '" << openStatus.ToStr() << "' (errno=" << openStatus.errNo + << ", code=" << openStatus.code << ")"; ex.addContext("Calling XrdFile::open()"); ex.addAdditionalInfo("Remote server already encountered a fatal error; no redirections were performed."); throw ex; @@ -179,56 +163,46 @@ RequestManager::initialize(std::weak_ptr self) std::unique_ptr hostList = handler.GetHosts(); Source::determineHostExcludeString(*file, hostList.get(), excludeString); assert(status); - if (status->IsOK()) - { + if (status->IsOK()) { validFile = true; break; - } - else - { + } else { ex.clearMessage(); ex.clearContext(); ex.clearAdditionalInfo(); - ex << "XrdCl::File::Open(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ") => error '" << status->ToStr() - << "' (errno=" << status->errNo << ", code=" << status->code << ")"; + ex << "XrdCl::File::Open(name='" << m_name << "', flags=0x" << std::hex << m_flags << ", permissions=0" + << std::oct << m_perms << std::dec << ") => error '" << status->ToStr() << "' (errno=" << status->errNo + << ", code=" << status->code << ")"; ex.addContext("Calling XrdFile::open()"); addConnections(ex); std::string dataServer, lastUrl; file->GetProperty("DataServer", dataServer); file->GetProperty("LastURL", lastUrl); - if (!dataServer.empty()) - { + if (!dataServer.empty()) { ex.addAdditionalInfo("Problematic data server: " + dataServer); } - if (!lastUrl.empty()) - { + if (!lastUrl.empty()) { ex.addAdditionalInfo("Last URL tried: " + lastUrl); edm::LogWarning("XrdAdaptorInternal") << "Failed to open file at URL " << lastUrl << "."; } - if (std::find(m_disabledSourceStrings.begin(), m_disabledSourceStrings.end(), dataServer) != m_disabledSourceStrings.end()) - { + if (std::find(m_disabledSourceStrings.begin(), m_disabledSourceStrings.end(), dataServer) != + m_disabledSourceStrings.end()) { ex << ". No additional data servers were found."; throw ex; } - if (!dataServer.empty()) - { + if (!dataServer.empty()) { m_disabledSourceStrings.insert(dataServer); m_disabledExcludeStrings.insert(excludeString); } // In this case, we didn't go anywhere - we stayed at the redirector and it gave us a file-not-found. - if (lastUrl == new_filename) - { + if (lastUrl == new_filename) { edm::LogWarning("XrdAdaptorInternal") << lastUrl << ", " << new_filename; throw ex; } } } - if (!validFile) - { - throw ex; + if (!validFile) { + throw ex; } SendMonitoringInfo(*file); @@ -257,221 +231,223 @@ RequestManager::initialize(std::weak_ptr self) * from an edm-managed thread. It CANNOT be called from an Xrootd-managed * thread. */ -void -RequestManager::updateCurrentServer() -{ - // NOTE: we use memory_order_relaxed here, meaning that we may actually miss - // a pending update. *However*, since we call this for every read, we'll get it - // eventually. - if (LIKELY(!m_serverToAdvertise.load(std::memory_order_relaxed))) {return;} - std::string *hostname_ptr; - if ((hostname_ptr = m_serverToAdvertise.exchange(nullptr))) - { - std::unique_ptr hostname(hostname_ptr); - edm::Service statsService; - if (statsService.isAvailable()) { - statsService->setCurrentServer(*hostname_ptr); - } +void RequestManager::updateCurrentServer() { + // NOTE: we use memory_order_relaxed here, meaning that we may actually miss + // a pending update. *However*, since we call this for every read, we'll get it + // eventually. + if (LIKELY(!m_serverToAdvertise.load(std::memory_order_relaxed))) { + return; + } + std::string *hostname_ptr; + if ((hostname_ptr = m_serverToAdvertise.exchange(nullptr))) { + std::unique_ptr hostname(hostname_ptr); + edm::Service statsService; + if (statsService.isAvailable()) { + statsService->setCurrentServer(*hostname_ptr); } + } } - -void -RequestManager::queueUpdateCurrentServer(const std::string &id) -{ - auto hostname = std::make_unique(id); - if (Source::getHostname(id, *hostname)) - { - std::string *null_hostname = nullptr; - if (m_serverToAdvertise.compare_exchange_strong(null_hostname, hostname.get())) - { - hostname.release(); - } +void RequestManager::queueUpdateCurrentServer(const std::string &id) { + auto hostname = std::make_unique(id); + if (Source::getHostname(id, *hostname)) { + std::string *null_hostname = nullptr; + if (m_serverToAdvertise.compare_exchange_strong(null_hostname, hostname.get())) { + hostname.release(); } + } } -namespace { - std::string formatSites(std::vector > const& iSources) { +namespace { + std::string formatSites(std::vector> const &iSources) { std::string siteA, siteB; - if (!iSources.empty()) {siteA = iSources[0]->Site();} - if (iSources.size() == 2) {siteB = iSources[1]->Site();} + if (!iSources.empty()) { + siteA = iSources[0]->Site(); + } + if (iSources.size() == 2) { + siteB = iSources[1]->Site(); + } std::string siteList = siteA; - if (!siteB.empty() && (siteB != siteA)) {siteList = siteA + ", " + siteB;} + if (!siteB.empty() && (siteB != siteA)) { + siteList = siteA + ", " + siteB; + } return siteList; } -} - -void -RequestManager::reportSiteChange(std::vector > const& iOld, - std::vector > const& iNew, - std::string orig_site) const -{ +} // namespace + +void RequestManager::reportSiteChange(std::vector> const &iOld, + std::vector> const &iNew, + std::string orig_site) const { auto siteList = formatSites(iNew); - if (!orig_site.empty() && (orig_site != siteList)) - { + if (!orig_site.empty() && (orig_site != siteList)) { edm::LogWarning("XrdAdaptor") << "Data is served from " << siteList << " instead of original site " << orig_site; - } - else { + } else { auto oldSites = formatSites(iOld); - if (orig_site.empty() && (siteList != oldSites)) - { - if (!oldSites.empty() ) + if (orig_site.empty() && (siteList != oldSites)) { + if (!oldSites.empty()) edm::LogWarning("XrdAdaptor") << "Data is now served from " << siteList << " instead of previous " << oldSites; } } } - -void -RequestManager::checkSources(timespec &now, IOSize requestSize, - std::vector>& activeSources, - std::vector>& inactiveSources) -{ - edm::LogVerbatim("XrdAdaptorInternal") << "Time since last check " - << timeDiffMS(now, m_lastSourceCheck) << "; last check " - << m_lastSourceCheck.tv_sec << "; now " < 1000) - { - { // Be more aggressive about getting rid of very bad sources. +void RequestManager::checkSources(timespec &now, + IOSize requestSize, + std::vector> &activeSources, + std::vector> &inactiveSources) { + edm::LogVerbatim("XrdAdaptorInternal") << "Time since last check " << timeDiffMS(now, m_lastSourceCheck) + << "; last check " << m_lastSourceCheck.tv_sec << "; now " << now.tv_sec + << "; next check " << m_nextActiveSourceCheck.tv_sec << std::endl; + if (timeDiffMS(now, m_lastSourceCheck) > 1000) { + { // Be more aggressive about getting rid of very bad sources. compareSources(now, 0, 1, activeSources, inactiveSources); compareSources(now, 1, 0, activeSources, inactiveSources); } - if (timeDiffMS(now, m_nextActiveSourceCheck) > 0) - { + if (timeDiffMS(now, m_nextActiveSourceCheck) > 0) { checkSourcesImpl(now, requestSize, activeSources, inactiveSources); } } } - -bool -RequestManager::compareSources(const timespec &now, unsigned a, unsigned b, - std::vector>& activeSources, - std::vector>& inactiveSources) const -{ - if (activeSources.size() < std::max(a, b)+1) {return false;} +bool RequestManager::compareSources(const timespec &now, + unsigned a, + unsigned b, + std::vector> &activeSources, + std::vector> &inactiveSources) const { + if (activeSources.size() < std::max(a, b) + 1) { + return false; + } bool findNewSource = false; if ((activeSources[a]->getQuality() > 5130) || - ((activeSources[a]->getQuality() > 260) && (activeSources[b]->getQuality()*4 < activeSources[a]->getQuality()))) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Removing " - << activeSources[a]->PrettyID() << " from active sources due to poor quality (" - << activeSources[a]->getQuality() << " vs " << activeSources[b]->getQuality() << ")" << std::endl; - if (activeSources[a]->getLastDowngrade().tv_sec != 0) {findNewSource = true;} + ((activeSources[a]->getQuality() > 260) && + (activeSources[b]->getQuality() * 4 < activeSources[a]->getQuality()))) { + edm::LogVerbatim("XrdAdaptorInternal") + << "Removing " << activeSources[a]->PrettyID() << " from active sources due to poor quality (" + << activeSources[a]->getQuality() << " vs " << activeSources[b]->getQuality() << ")" << std::endl; + if (activeSources[a]->getLastDowngrade().tv_sec != 0) { + findNewSource = true; + } activeSources[a]->setLastDowngrade(now); inactiveSources.emplace_back(activeSources[a]); auto oldSources = activeSources; - activeSources.erase(activeSources.begin()+a); - reportSiteChange(oldSources,activeSources); + activeSources.erase(activeSources.begin() + a); + reportSiteChange(oldSources, activeSources); } return findNewSource; } -void -RequestManager::checkSourcesImpl(timespec &now, - IOSize requestSize, - std::vector>& activeSources, - std::vector>& inactiveSources) -{ - +void RequestManager::checkSourcesImpl(timespec &now, + IOSize requestSize, + std::vector> &activeSources, + std::vector> &inactiveSources) { bool findNewSource = false; - if (activeSources.size() <= 1) - { + if (activeSources.size() <= 1) { findNewSource = true; - } - else if (activeSources.size() > 1) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Source 0 quality " << activeSources[0]->getQuality() << ", source 1 quality " << activeSources[1]->getQuality() << std::endl; - findNewSource |= compareSources(now, 0, 1, activeSources,inactiveSources); - findNewSource |= compareSources(now, 1, 0,activeSources, inactiveSources); + } else if (activeSources.size() > 1) { + edm::LogVerbatim("XrdAdaptorInternal") << "Source 0 quality " << activeSources[0]->getQuality() + << ", source 1 quality " << activeSources[1]->getQuality() << std::endl; + findNewSource |= compareSources(now, 0, 1, activeSources, inactiveSources); + findNewSource |= compareSources(now, 1, 0, activeSources, inactiveSources); // NOTE: We could probably replace the copy with a better sort function. // However, there are typically very few sources and the correctness is more obvious right now. - std::vector > eligibleInactiveSources; eligibleInactiveSources.reserve(inactiveSources.size()); - for (const auto & source : inactiveSources) - { - if (timeDiffMS(now, source->getLastDowngrade()) > (XRD_ADAPTOR_SHORT_OPEN_DELAY-1)*1000) {eligibleInactiveSources.push_back(source);} - } - auto bestInactiveSource = std::min_element(eligibleInactiveSources.begin(), eligibleInactiveSources.end(), - [](const std::shared_ptr &s1, const std::shared_ptr &s2) {return s1->getQuality() < s2->getQuality();}); - auto worstActiveSource = std::max_element(activeSources.cbegin(), activeSources.cend(), - [](const std::shared_ptr &s1, const std::shared_ptr &s2) {return s1->getQuality() < s2->getQuality();}); - if (bestInactiveSource != eligibleInactiveSources.end() && bestInactiveSource->get()) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Best inactive source: " <<(*bestInactiveSource)->PrettyID() - << ", quality " << (*bestInactiveSource)->getQuality(); + std::vector> eligibleInactiveSources; + eligibleInactiveSources.reserve(inactiveSources.size()); + for (const auto &source : inactiveSources) { + if (timeDiffMS(now, source->getLastDowngrade()) > (XRD_ADAPTOR_SHORT_OPEN_DELAY - 1) * 1000) { + eligibleInactiveSources.push_back(source); + } } - edm::LogVerbatim("XrdAdaptorInternal") << "Worst active source: " <<(*worstActiveSource)->PrettyID() - << ", quality " << (*worstActiveSource)->getQuality(); - // Only upgrade the source if we only have one source and the best inactive one isn't too horrible. - // Regardless, we will want to re-evaluate the new source quickly (within 5s). - if ((bestInactiveSource != eligibleInactiveSources.end()) && activeSources.size() == 1 && ((*bestInactiveSource)->getQuality() < 4*activeSources[0]->getQuality())) - { - auto oldSources = activeSources; - activeSources.push_back(*bestInactiveSource); - reportSiteChange(oldSources, activeSources); - for (auto it = inactiveSources.begin(); it != inactiveSources.end(); it++) if (it->get() == bestInactiveSource->get()) {inactiveSources.erase(it); break;} + auto bestInactiveSource = + std::min_element(eligibleInactiveSources.begin(), + eligibleInactiveSources.end(), + [](const std::shared_ptr &s1, const std::shared_ptr &s2) { + return s1->getQuality() < s2->getQuality(); + }); + auto worstActiveSource = std::max_element(activeSources.cbegin(), + activeSources.cend(), + [](const std::shared_ptr &s1, const std::shared_ptr &s2) { + return s1->getQuality() < s2->getQuality(); + }); + if (bestInactiveSource != eligibleInactiveSources.end() && bestInactiveSource->get()) { + edm::LogVerbatim("XrdAdaptorInternal") << "Best inactive source: " << (*bestInactiveSource)->PrettyID() + << ", quality " << (*bestInactiveSource)->getQuality(); } - else while ((bestInactiveSource != eligibleInactiveSources.end()) && (*worstActiveSource)->getQuality() > (*bestInactiveSource)->getQuality()+XRD_ADAPTOR_SOURCE_QUALITY_FUDGE) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Removing " << (*worstActiveSource)->PrettyID() - << " from active sources due to quality (" << (*worstActiveSource)->getQuality() - << ") and promoting " << (*bestInactiveSource)->PrettyID() << " (quality: " - << (*bestInactiveSource)->getQuality() << ")" << std::endl; + edm::LogVerbatim("XrdAdaptorInternal") << "Worst active source: " << (*worstActiveSource)->PrettyID() + << ", quality " << (*worstActiveSource)->getQuality(); + // Only upgrade the source if we only have one source and the best inactive one isn't too horrible. + // Regardless, we will want to re-evaluate the new source quickly (within 5s). + if ((bestInactiveSource != eligibleInactiveSources.end()) && activeSources.size() == 1 && + ((*bestInactiveSource)->getQuality() < 4 * activeSources[0]->getQuality())) { + auto oldSources = activeSources; + activeSources.push_back(*bestInactiveSource); + reportSiteChange(oldSources, activeSources); + for (auto it = inactiveSources.begin(); it != inactiveSources.end(); it++) + if (it->get() == bestInactiveSource->get()) { + inactiveSources.erase(it); + break; + } + } else + while ((bestInactiveSource != eligibleInactiveSources.end()) && + (*worstActiveSource)->getQuality() > + (*bestInactiveSource)->getQuality() + XRD_ADAPTOR_SOURCE_QUALITY_FUDGE) { + edm::LogVerbatim("XrdAdaptorInternal") + << "Removing " << (*worstActiveSource)->PrettyID() << " from active sources due to quality (" + << (*worstActiveSource)->getQuality() << ") and promoting " << (*bestInactiveSource)->PrettyID() + << " (quality: " << (*bestInactiveSource)->getQuality() << ")" << std::endl; (*worstActiveSource)->setLastDowngrade(now); - for (auto it = inactiveSources.begin(); it != inactiveSources.end(); it++) if (it->get() == bestInactiveSource->get()) {inactiveSources.erase(it); break;} + for (auto it = inactiveSources.begin(); it != inactiveSources.end(); it++) + if (it->get() == bestInactiveSource->get()) { + inactiveSources.erase(it); + break; + } inactiveSources.emplace_back(std::move(*worstActiveSource)); auto oldSources = activeSources; activeSources.erase(worstActiveSource); activeSources.emplace_back(std::move(*bestInactiveSource)); reportSiteChange(oldSources, activeSources); eligibleInactiveSources.clear(); - for (const auto & source : inactiveSources) if (timeDiffMS(now, source->getLastDowngrade()) > (XRD_ADAPTOR_LONG_OPEN_DELAY-1)*1000) eligibleInactiveSources.push_back(source); - bestInactiveSource = std::min_element(eligibleInactiveSources.begin(), eligibleInactiveSources.end(), - [](const std::shared_ptr &s1, const std::shared_ptr &s2) {return s1->getQuality() < s2->getQuality();}); - worstActiveSource = std::max_element(activeSources.begin(), activeSources.end(), - [](const std::shared_ptr &s1, const std::shared_ptr &s2) {return s1->getQuality() < s2->getQuality();}); - } - if (!findNewSource && (timeDiffMS(now, m_lastSourceCheck) > 1000*XRD_ADAPTOR_LONG_OPEN_DELAY)) - { - float r = m_distribution(m_generator); - if (r < XRD_ADAPTOR_OPEN_PROBE_PERCENT) - { - findNewSource = true; - } + for (const auto &source : inactiveSources) + if (timeDiffMS(now, source->getLastDowngrade()) > (XRD_ADAPTOR_LONG_OPEN_DELAY - 1) * 1000) + eligibleInactiveSources.push_back(source); + bestInactiveSource = std::min_element(eligibleInactiveSources.begin(), + eligibleInactiveSources.end(), + [](const std::shared_ptr &s1, const std::shared_ptr &s2) { + return s1->getQuality() < s2->getQuality(); + }); + worstActiveSource = std::max_element(activeSources.begin(), + activeSources.end(), + [](const std::shared_ptr &s1, const std::shared_ptr &s2) { + return s1->getQuality() < s2->getQuality(); + }); + } + if (!findNewSource && (timeDiffMS(now, m_lastSourceCheck) > 1000 * XRD_ADAPTOR_LONG_OPEN_DELAY)) { + float r = m_distribution(m_generator); + if (r < XRD_ADAPTOR_OPEN_PROBE_PERCENT) { + findNewSource = true; + } } } - if (findNewSource) - { + if (findNewSource) { m_open_handler->open(); m_lastSourceCheck = now; } // Only aggressively look for new sources if we don't have two. - if (activeSources.size() == 2) - { + if (activeSources.size() == 2) { now.tv_sec += XRD_ADAPTOR_LONG_OPEN_DELAY - XRD_ADAPTOR_SHORT_OPEN_DELAY; - } - else - { + } else { now.tv_sec += XRD_ADAPTOR_SHORT_OPEN_DELAY; } m_nextActiveSourceCheck = now; } -std::shared_ptr -RequestManager::getActiveFile() const -{ +std::shared_ptr RequestManager::getActiveFile() const { std::lock_guard sentry(m_source_mutex); - if (m_activeSources.empty()) - { + if (m_activeSources.empty()) { edm::Exception ex(edm::errors::FileReadError); - ex << "XrdAdaptor::RequestManager::getActiveFile(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ") => Source used after fatal exception."; + ex << "XrdAdaptor::RequestManager::getActiveFile(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec << ") => Source used after fatal exception."; ex.addContext("In XrdAdaptor::RequestManager::handle()"); addConnections(ex); throw ex; @@ -479,93 +455,69 @@ RequestManager::getActiveFile() const return m_activeSources[0]->getFileHandle(); } -void -RequestManager::getActiveSourceNames(std::vector & sources) const -{ +void RequestManager::getActiveSourceNames(std::vector &sources) const { std::lock_guard sentry(m_source_mutex); sources.reserve(m_activeSources.size()); - for (auto const& source : m_activeSources) { + for (auto const &source : m_activeSources) { sources.push_back(source->ID()); } } -void -RequestManager::getPrettyActiveSourceNames(std::vector & sources) const -{ +void RequestManager::getPrettyActiveSourceNames(std::vector &sources) const { std::lock_guard sentry(m_source_mutex); sources.reserve(m_activeSources.size()); - for (auto const& source : m_activeSources) { + for (auto const &source : m_activeSources) { sources.push_back(source->PrettyID()); } } -void -RequestManager::getDisabledSourceNames(std::vector & sources) const -{ +void RequestManager::getDisabledSourceNames(std::vector &sources) const { sources.reserve(m_disabledSourceStrings.size()); - for (auto const& source : m_disabledSourceStrings) { + for (auto const &source : m_disabledSourceStrings) { sources.push_back(source); } } -void -RequestManager::addConnections(cms::Exception &ex) const -{ +void RequestManager::addConnections(cms::Exception &ex) const { std::vector sources; getPrettyActiveSourceNames(sources); - for (auto const& source : sources) - { + for (auto const &source : sources) { ex.addAdditionalInfo("Active source: " + source); } sources.clear(); getDisabledSourceNames(sources); - for (auto const& source : sources) - { + for (auto const &source : sources) { ex.addAdditionalInfo("Disabled source: " + source); } } -std::shared_ptr -RequestManager::pickSingleSource() -{ +std::shared_ptr RequestManager::pickSingleSource() { std::shared_ptr source = nullptr; { std::lock_guard sentry(m_source_mutex); - if (m_activeSources.size() == 2) - { - if (m_nextInitialSourceToggle) - { - source = m_activeSources[0]; - m_nextInitialSourceToggle = false; - } - else - { - source = m_activeSources[1]; - m_nextInitialSourceToggle = true; - } - } - else if (m_activeSources.empty()) - { - edm::Exception ex(edm::errors::FileReadError); - ex << "XrdAdaptor::RequestManager::handle read(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ") => Source used after fatal exception."; - ex.addContext("In XrdAdaptor::RequestManager::handle()"); - addConnections(ex); - throw ex; - } - else - { + if (m_activeSources.size() == 2) { + if (m_nextInitialSourceToggle) { source = m_activeSources[0]; + m_nextInitialSourceToggle = false; + } else { + source = m_activeSources[1]; + m_nextInitialSourceToggle = true; + } + } else if (m_activeSources.empty()) { + edm::Exception ex(edm::errors::FileReadError); + ex << "XrdAdaptor::RequestManager::handle read(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec << ") => Source used after fatal exception."; + ex.addContext("In XrdAdaptor::RequestManager::handle()"); + addConnections(ex); + throw ex; + } else { + source = m_activeSources[0]; } } return source; } -std::future -RequestManager::handle(std::shared_ptr c_ptr) -{ +std::future RequestManager::handle(std::shared_ptr c_ptr) { assert(c_ptr.get()); timespec now; GET_CLOCK_MONOTONIC(now); @@ -578,7 +530,7 @@ RequestManager::handle(std::shared_ptr c_ptr) } { //make sure we update values before calling pickSingelSource - std::shared_ptr guard(nullptr, [this, &activeSources, &inactiveSources](void *) { + std::shared_ptr guard(nullptr, [this, &activeSources, &inactiveSources](void *) { std::lock_guard sentry(m_source_mutex); m_activeSources = std::move(activeSources); m_inactiveSources = std::move(inactiveSources); @@ -586,491 +538,435 @@ RequestManager::handle(std::shared_ptr c_ptr) checkSources(now, c_ptr->getSize(), activeSources, inactiveSources); } - + std::shared_ptr source = pickSingleSource(); source->handle(c_ptr); return c_ptr->get_future(); } -std::string -RequestManager::prepareOpaqueString() const -{ - std::stringstream ss; - ss << "tried="; - size_t count = 0; - { - std::lock_guard sentry(m_source_mutex); - - for ( const auto & it : m_activeSources ) - { - count++; - ss << it->ExcludeID().substr(0, it->ExcludeID().find(":")) << ","; - } - for ( const auto & it : m_inactiveSources ) - { - count++; - ss << it->ExcludeID().substr(0, it->ExcludeID().find(":")) << ","; - } - } - for ( const auto & it : m_disabledExcludeStrings ) - { - count++; - ss << it.substr(0, it.find(":")) << ","; +std::string RequestManager::prepareOpaqueString() const { + std::stringstream ss; + ss << "tried="; + size_t count = 0; + { + std::lock_guard sentry(m_source_mutex); + + for (const auto &it : m_activeSources) { + count++; + ss << it->ExcludeID().substr(0, it->ExcludeID().find(":")) << ","; } - if (count) - { - std::string tmp_str = ss.str(); - return tmp_str.substr(0, tmp_str.size()-1); + for (const auto &it : m_inactiveSources) { + count++; + ss << it->ExcludeID().substr(0, it->ExcludeID().find(":")) << ","; } - return ""; + } + for (const auto &it : m_disabledExcludeStrings) { + count++; + ss << it.substr(0, it.find(":")) << ","; + } + if (count) { + std::string tmp_str = ss.str(); + return tmp_str.substr(0, tmp_str.size() - 1); + } + return ""; } -void -XrdAdaptor::RequestManager::handleOpen(XrdCl::XRootDStatus &status, std::shared_ptr source) -{ - std::lock_guard sentry(m_source_mutex); - if (status.IsOK()) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Successfully opened new source: " << source->PrettyID() << std::endl; - for (const auto & s : m_activeSources) - { - if (source->ID() == s->ID()) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Xrootd server returned excluded source " << source->PrettyID() - << "; ignoring" << std::endl; - unsigned returned_count = ++m_excluded_active_count; - m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_SHORT_OPEN_DELAY; - if (returned_count >= 3) {m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_LONG_OPEN_DELAY - 2*XRD_ADAPTOR_SHORT_OPEN_DELAY;} - return; - } - } - for (const auto & s : m_inactiveSources) - { - if (source->ID() == s->ID()) - { - edm::LogVerbatim("XrdAdaptorInternal") << "Xrootd server returned excluded inactive source " << source->PrettyID() - << "; ignoring" << std::endl; - m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_LONG_OPEN_DELAY - XRD_ADAPTOR_SHORT_OPEN_DELAY; - return; - } - } - if (m_activeSources.size() < 2) - { - auto oldSources = m_activeSources; - m_activeSources.push_back(source); - reportSiteChange(oldSources, m_activeSources); - queueUpdateCurrentServer(source->ID()); - } - else - { - m_inactiveSources.push_back(source); +void XrdAdaptor::RequestManager::handleOpen(XrdCl::XRootDStatus &status, std::shared_ptr source) { + std::lock_guard sentry(m_source_mutex); + if (status.IsOK()) { + edm::LogVerbatim("XrdAdaptorInternal") << "Successfully opened new source: " << source->PrettyID() << std::endl; + for (const auto &s : m_activeSources) { + if (source->ID() == s->ID()) { + edm::LogVerbatim("XrdAdaptorInternal") + << "Xrootd server returned excluded source " << source->PrettyID() << "; ignoring" << std::endl; + unsigned returned_count = ++m_excluded_active_count; + m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_SHORT_OPEN_DELAY; + if (returned_count >= 3) { + m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_LONG_OPEN_DELAY - 2 * XRD_ADAPTOR_SHORT_OPEN_DELAY; } + return; + } } - else - { // File-open failure - wait at least 120s before next attempt. - edm::LogVerbatim("XrdAdaptorInternal") << "Got failure when trying to open a new source" << std::endl; + for (const auto &s : m_inactiveSources) { + if (source->ID() == s->ID()) { + edm::LogVerbatim("XrdAdaptorInternal") + << "Xrootd server returned excluded inactive source " << source->PrettyID() << "; ignoring" << std::endl; m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_LONG_OPEN_DELAY - XRD_ADAPTOR_SHORT_OPEN_DELAY; + return; + } + } + if (m_activeSources.size() < 2) { + auto oldSources = m_activeSources; + m_activeSources.push_back(source); + reportSiteChange(oldSources, m_activeSources); + queueUpdateCurrentServer(source->ID()); + } else { + m_inactiveSources.push_back(source); } + } else { // File-open failure - wait at least 120s before next attempt. + edm::LogVerbatim("XrdAdaptorInternal") << "Got failure when trying to open a new source" << std::endl; + m_nextActiveSourceCheck.tv_sec += XRD_ADAPTOR_LONG_OPEN_DELAY - XRD_ADAPTOR_SHORT_OPEN_DELAY; + } } -std::future -XrdAdaptor::RequestManager::handle(std::shared_ptr > iolist) -{ - //Use a copy of m_activeSources and m_inactiveSources throughout this function - // in order to avoid holding the lock a long time and causing a deadlock. - // When the function is over we will update the values of the containers - std::vector> activeSources, inactiveSources; - { - std::lock_guard sentry(m_source_mutex); - activeSources = m_activeSources; - inactiveSources = m_inactiveSources; - } - //Make sure we update changes when we leave the function - std::shared_ptr guard(nullptr, [this, &activeSources, &inactiveSources](void *) { - std::lock_guard sentry(m_source_mutex); - m_activeSources = std::move(activeSources); - m_inactiveSources = std::move(inactiveSources); - }); - - updateCurrentServer(); +std::future XrdAdaptor::RequestManager::handle(std::shared_ptr> iolist) { + //Use a copy of m_activeSources and m_inactiveSources throughout this function + // in order to avoid holding the lock a long time and causing a deadlock. + // When the function is over we will update the values of the containers + std::vector> activeSources, inactiveSources; + { + std::lock_guard sentry(m_source_mutex); + activeSources = m_activeSources; + inactiveSources = m_inactiveSources; + } + //Make sure we update changes when we leave the function + std::shared_ptr guard(nullptr, [this, &activeSources, &inactiveSources](void *) { + std::lock_guard sentry(m_source_mutex); + m_activeSources = std::move(activeSources); + m_inactiveSources = std::move(inactiveSources); + }); - timespec now; - GET_CLOCK_MONOTONIC(now); + updateCurrentServer(); - edm::CPUTimer timer; - timer.start(); + timespec now; + GET_CLOCK_MONOTONIC(now); - if (activeSources.size() == 1) - { - auto c_ptr = std::make_shared(*this, iolist); - checkSources(now, c_ptr->getSize(), activeSources,inactiveSources); - activeSources[0]->handle(c_ptr); - return c_ptr->get_future(); - } - // Make sure active - else if (activeSources.empty()) - { - edm::Exception ex(edm::errors::FileReadError); - ex << "XrdAdaptor::RequestManager::handle readv(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ") => Source used after fatal exception."; - ex.addContext("In XrdAdaptor::RequestManager::handle()"); - addConnections(ex); - throw ex; - } + edm::CPUTimer timer; + timer.start(); - assert(iolist.get()); - auto req1 = std::make_shared>(); - auto req2 = std::make_shared>(); - splitClientRequest(*iolist, *req1, *req2, activeSources); - - checkSources(now, req1->size() + req2->size(), activeSources, inactiveSources); - // CheckSources may have removed a source - if (activeSources.size() == 1) - { - auto c_ptr = std::make_shared(*this, iolist); - activeSources[0]->handle(c_ptr); - return c_ptr->get_future(); - } + if (activeSources.size() == 1) { + auto c_ptr = std::make_shared(*this, iolist); + checkSources(now, c_ptr->getSize(), activeSources, inactiveSources); + activeSources[0]->handle(c_ptr); + return c_ptr->get_future(); + } + // Make sure active + else if (activeSources.empty()) { + edm::Exception ex(edm::errors::FileReadError); + ex << "XrdAdaptor::RequestManager::handle readv(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec << ") => Source used after fatal exception."; + ex.addContext("In XrdAdaptor::RequestManager::handle()"); + addConnections(ex); + throw ex; + } - std::shared_ptr c_ptr1, c_ptr2; - std::future future1, future2; - if (!req1->empty()) - { - c_ptr1.reset(new XrdAdaptor::ClientRequest(*this, req1)); - activeSources[0]->handle(c_ptr1); - future1 = c_ptr1->get_future(); - } - if (!req2->empty()) - { - c_ptr2.reset(new XrdAdaptor::ClientRequest(*this, req2)); - activeSources[1]->handle(c_ptr2); - future2 = c_ptr2->get_future(); - } - if (!req1->empty() && !req2->empty()) - { - std::future task = std::async(std::launch::deferred, - [](std::future a, std::future b){ - // Wait until *both* results are available. This is essential - // as the callback may try referencing the RequestManager. If one - // throws an exception (causing the RequestManager to be destroyed by - // XrdFile) and the other has a failure, then the recovery code will - // reference the destroyed RequestManager. - // - // Unlike other places where we use shared/weak ptrs to maintain object - // lifetime and destruction asynchronously, we *cannot* destroy the request - // asynchronously as it is associated with a ROOT buffer. We must wait until we - // are guaranteed that XrdCl will not write into the ROOT buffer before we - // can return. - b.wait(); a.wait(); - return b.get() + a.get(); - }, - std::move(future1), - std::move(future2)); - timer.stop(); - //edm::LogVerbatim("XrdAdaptorInternal") << "Total time to create requests " << static_cast(1000*timer.realTime()) << std::endl; - return task; - } - else if (!req1->empty()) { return future1; } - else if (!req2->empty()) { return future2; } - else - { // Degenerate case - no bytes to read. - std::promise p; p.set_value(0); - return p.get_future(); - } + assert(iolist.get()); + auto req1 = std::make_shared>(); + auto req2 = std::make_shared>(); + splitClientRequest(*iolist, *req1, *req2, activeSources); + + checkSources(now, req1->size() + req2->size(), activeSources, inactiveSources); + // CheckSources may have removed a source + if (activeSources.size() == 1) { + auto c_ptr = std::make_shared(*this, iolist); + activeSources[0]->handle(c_ptr); + return c_ptr->get_future(); + } + + std::shared_ptr c_ptr1, c_ptr2; + std::future future1, future2; + if (!req1->empty()) { + c_ptr1.reset(new XrdAdaptor::ClientRequest(*this, req1)); + activeSources[0]->handle(c_ptr1); + future1 = c_ptr1->get_future(); + } + if (!req2->empty()) { + c_ptr2.reset(new XrdAdaptor::ClientRequest(*this, req2)); + activeSources[1]->handle(c_ptr2); + future2 = c_ptr2->get_future(); + } + if (!req1->empty() && !req2->empty()) { + std::future task = + std::async(std::launch::deferred, + [](std::future a, std::future b) { + // Wait until *both* results are available. This is essential + // as the callback may try referencing the RequestManager. If one + // throws an exception (causing the RequestManager to be destroyed by + // XrdFile) and the other has a failure, then the recovery code will + // reference the destroyed RequestManager. + // + // Unlike other places where we use shared/weak ptrs to maintain object + // lifetime and destruction asynchronously, we *cannot* destroy the request + // asynchronously as it is associated with a ROOT buffer. We must wait until we + // are guaranteed that XrdCl will not write into the ROOT buffer before we + // can return. + b.wait(); + a.wait(); + return b.get() + a.get(); + }, + std::move(future1), + std::move(future2)); + timer.stop(); + //edm::LogVerbatim("XrdAdaptorInternal") << "Total time to create requests " << static_cast(1000*timer.realTime()) << std::endl; + return task; + } else if (!req1->empty()) { + return future1; + } else if (!req2->empty()) { + return future2; + } else { // Degenerate case - no bytes to read. + std::promise p; + p.set_value(0); + return p.get_future(); + } } -void -RequestManager::requestFailure(std::shared_ptr c_ptr, XrdCl::Status &c_status) -{ - std::shared_ptr source_ptr = c_ptr->getCurrentSource(); - - // Fail early for invalid responses - XrdFile has a separate path for handling this. - if (c_status.code == XrdCl::errInvalidResponse) - { - edm::LogWarning("XrdAdaptorInternal") << "Invalid response when reading from " << source_ptr->PrettyID(); - XrootdException ex(c_status, edm::errors::FileReadError); - ex << "XrdAdaptor::RequestManager::requestFailure readv(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ", old source=" << source_ptr->PrettyID() - << ") => Invalid ReadV response from server"; - ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); - addConnections(ex); - throw ex; - } - edm::LogWarning("XrdAdaptorInternal") << "Request failure when reading from " << source_ptr->PrettyID(); - - // Note that we do not delete the Source itself. That is because this - // function may be called from within XrdCl::ResponseHandler::HandleResponseWithHosts - // In such a case, if you close a file in the handler, it will deadlock - m_disabledSourceStrings.insert(source_ptr->ID()); - m_disabledExcludeStrings.insert(source_ptr->ExcludeID()); - m_disabledSources.insert(source_ptr); - - std::unique_lock sentry(m_source_mutex); - if ((!m_activeSources.empty()) && (m_activeSources[0].get() == source_ptr.get())) - { - auto oldSources = m_activeSources; - m_activeSources.erase(m_activeSources.begin()); - reportSiteChange(oldSources, m_activeSources); - } - else if ((m_activeSources.size() > 1) && (m_activeSources[1].get() == source_ptr.get())) - { - auto oldSources = m_activeSources; - m_activeSources.erase(m_activeSources.begin()+1); - reportSiteChange(oldSources, m_activeSources); +void RequestManager::requestFailure(std::shared_ptr c_ptr, XrdCl::Status &c_status) { + std::shared_ptr source_ptr = c_ptr->getCurrentSource(); + + // Fail early for invalid responses - XrdFile has a separate path for handling this. + if (c_status.code == XrdCl::errInvalidResponse) { + edm::LogWarning("XrdAdaptorInternal") << "Invalid response when reading from " << source_ptr->PrettyID(); + XrootdException ex(c_status, edm::errors::FileReadError); + ex << "XrdAdaptor::RequestManager::requestFailure readv(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec << ", old source=" << source_ptr->PrettyID() + << ") => Invalid ReadV response from server"; + ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); + addConnections(ex); + throw ex; + } + edm::LogWarning("XrdAdaptorInternal") << "Request failure when reading from " << source_ptr->PrettyID(); + + // Note that we do not delete the Source itself. That is because this + // function may be called from within XrdCl::ResponseHandler::HandleResponseWithHosts + // In such a case, if you close a file in the handler, it will deadlock + m_disabledSourceStrings.insert(source_ptr->ID()); + m_disabledExcludeStrings.insert(source_ptr->ExcludeID()); + m_disabledSources.insert(source_ptr); + + std::unique_lock sentry(m_source_mutex); + if ((!m_activeSources.empty()) && (m_activeSources[0].get() == source_ptr.get())) { + auto oldSources = m_activeSources; + m_activeSources.erase(m_activeSources.begin()); + reportSiteChange(oldSources, m_activeSources); + } else if ((m_activeSources.size() > 1) && (m_activeSources[1].get() == source_ptr.get())) { + auto oldSources = m_activeSources; + m_activeSources.erase(m_activeSources.begin() + 1); + reportSiteChange(oldSources, m_activeSources); + } + std::shared_ptr new_source; + if (m_activeSources.empty()) { + std::shared_future> future = m_open_handler->open(); + timespec now; + GET_CLOCK_MONOTONIC(now); + m_lastSourceCheck = now; + // Note we only wait for 180 seconds here. This is because we've already failed + // once and the likelihood the program has some inconsistent state is decent. + // We'd much rather fail hard than deadlock! + sentry.unlock(); + std::future_status status = future.wait_for(std::chrono::seconds(m_timeout + 10)); + if (status == std::future_status::timeout) { + XrootdException ex(c_status, edm::errors::FileOpenError); + ex << "XrdAdaptor::RequestManager::requestFailure Open(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec << ", old source=" << source_ptr->PrettyID() + << ") => timeout when waiting for file open"; + ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); + addConnections(ex); + throw ex; + } else { + try { + new_source = future.get(); + } catch (edm::Exception &ex) { + ex.addContext("Handling XrdAdaptor::RequestManager::requestFailure()"); + ex.addAdditionalInfo("Original failed source is " + source_ptr->PrettyID()); + throw; + } } - std::shared_ptr new_source; - if (m_activeSources.empty()) - { - std::shared_future > future = m_open_handler->open(); - timespec now; - GET_CLOCK_MONOTONIC(now); - m_lastSourceCheck = now; - // Note we only wait for 180 seconds here. This is because we've already failed - // once and the likelihood the program has some inconsistent state is decent. - // We'd much rather fail hard than deadlock! - sentry.unlock(); - std::future_status status = future.wait_for(std::chrono::seconds(m_timeout+10)); - if (status == std::future_status::timeout) - { - XrootdException ex(c_status, edm::errors::FileOpenError); - ex << "XrdAdaptor::RequestManager::requestFailure Open(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ", old source=" << source_ptr->PrettyID() - << ") => timeout when waiting for file open"; - ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); - addConnections(ex); - throw ex; - } - else - { - try - { - new_source = future.get(); - } - catch (edm::Exception &ex) - { - ex.addContext("Handling XrdAdaptor::RequestManager::requestFailure()"); - ex.addAdditionalInfo("Original failed source is " + source_ptr->PrettyID()); - throw; - } - } - - if (std::find(m_disabledSourceStrings.begin(), m_disabledSourceStrings.end(), new_source->ID()) != m_disabledSourceStrings.end()) - { - // The server gave us back a data node we requested excluded. Fatal! - XrootdException ex(c_status, edm::errors::FileOpenError); - ex << "XrdAdaptor::RequestManager::requestFailure Open(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ", old source=" << source_ptr->PrettyID() - << ", new source=" << new_source->PrettyID() << ") => Xrootd server returned an excluded source"; - ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); - addConnections(ex); - throw ex; - } - sentry.lock(); - auto oldSources = m_activeSources; - m_activeSources.push_back(new_source); - reportSiteChange(oldSources,m_activeSources); - } - else - { - new_source = m_activeSources[0]; + if (std::find(m_disabledSourceStrings.begin(), m_disabledSourceStrings.end(), new_source->ID()) != + m_disabledSourceStrings.end()) { + // The server gave us back a data node we requested excluded. Fatal! + XrootdException ex(c_status, edm::errors::FileOpenError); + ex << "XrdAdaptor::RequestManager::requestFailure Open(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec << ", old source=" << source_ptr->PrettyID() + << ", new source=" << new_source->PrettyID() << ") => Xrootd server returned an excluded source"; + ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); + addConnections(ex); + throw ex; } - new_source->handle(c_ptr); + sentry.lock(); + + auto oldSources = m_activeSources; + m_activeSources.push_back(new_source); + reportSiteChange(oldSources, m_activeSources); + } else { + new_source = m_activeSources[0]; + } + new_source->handle(c_ptr); } -static void -consumeChunkFront(size_t &front, std::vector &input, std::vector &output, IOSize chunksize) -{ - while ((chunksize > 0) && (front < input.size()) && (output.size() <= XRD_ADAPTOR_CHUNK_THRESHOLD)) - { - IOPosBuffer &io = input[front]; - IOPosBuffer &outio = output.back(); - if (io.size() > chunksize) - { - IOSize consumed; - if (!output.empty() && (outio.size() < XRD_CL_MAX_CHUNK) && (outio.offset() + static_cast(outio.size()) == io.offset())) - { - if (outio.size() + chunksize > XRD_CL_MAX_CHUNK) - { - consumed = (XRD_CL_MAX_CHUNK - outio.size()); - outio.set_size(XRD_CL_MAX_CHUNK); - } - else - { - consumed = chunksize; - outio.set_size(outio.size() + consumed); - } - } - else - { - consumed = chunksize; - output.emplace_back(IOPosBuffer(io.offset(), io.data(), chunksize)); - } - chunksize -= consumed; - IOSize newsize = io.size() - consumed; - IOOffset newoffset = io.offset() + consumed; - void* newdata = static_cast(io.data()) + consumed; - io.set_offset(newoffset); - io.set_data(newdata); - io.set_size(newsize); - } - else if (io.size() == 0) - { - front++; - } - else - { - output.push_back(io); - chunksize -= io.size(); - front++; +static void consumeChunkFront(size_t &front, + std::vector &input, + std::vector &output, + IOSize chunksize) { + while ((chunksize > 0) && (front < input.size()) && (output.size() <= XRD_ADAPTOR_CHUNK_THRESHOLD)) { + IOPosBuffer &io = input[front]; + IOPosBuffer &outio = output.back(); + if (io.size() > chunksize) { + IOSize consumed; + if (!output.empty() && (outio.size() < XRD_CL_MAX_CHUNK) && + (outio.offset() + static_cast(outio.size()) == io.offset())) { + if (outio.size() + chunksize > XRD_CL_MAX_CHUNK) { + consumed = (XRD_CL_MAX_CHUNK - outio.size()); + outio.set_size(XRD_CL_MAX_CHUNK); + } else { + consumed = chunksize; + outio.set_size(outio.size() + consumed); } + } else { + consumed = chunksize; + output.emplace_back(IOPosBuffer(io.offset(), io.data(), chunksize)); + } + chunksize -= consumed; + IOSize newsize = io.size() - consumed; + IOOffset newoffset = io.offset() + consumed; + void *newdata = static_cast(io.data()) + consumed; + io.set_offset(newoffset); + io.set_data(newdata); + io.set_size(newsize); + } else if (io.size() == 0) { + front++; + } else { + output.push_back(io); + chunksize -= io.size(); + front++; } + } } -static void -consumeChunkBack(size_t front, std::vector &input, std::vector &output, IOSize chunksize) -{ - while ((chunksize > 0) && (front < input.size()) && (output.size() <= XRD_ADAPTOR_CHUNK_THRESHOLD)) - { - IOPosBuffer &io = input.back(); - IOPosBuffer &outio = output.back(); - if (io.size() > chunksize) - { - IOSize consumed; - if (!output.empty() && (outio.size() < XRD_CL_MAX_CHUNK) && (outio.offset() + static_cast(outio.size()) == io.offset())) - { - if (outio.size() + chunksize > XRD_CL_MAX_CHUNK) - { - consumed = (XRD_CL_MAX_CHUNK - outio.size()); - outio.set_size(XRD_CL_MAX_CHUNK); - } - else - { - consumed = chunksize; - outio.set_size(outio.size() + consumed); - } - } - else - { - consumed = chunksize; - output.emplace_back(IOPosBuffer(io.offset(), io.data(), chunksize)); - } - chunksize -= consumed; - IOSize newsize = io.size() - consumed; - IOOffset newoffset = io.offset() + consumed; - void* newdata = static_cast(io.data()) + consumed; - io.set_offset(newoffset); - io.set_data(newdata); - io.set_size(newsize); - } - else if (io.size() == 0) - { - input.pop_back(); - } - else - { - output.push_back(io); - chunksize -= io.size(); - input.pop_back(); +static void consumeChunkBack(size_t front, + std::vector &input, + std::vector &output, + IOSize chunksize) { + while ((chunksize > 0) && (front < input.size()) && (output.size() <= XRD_ADAPTOR_CHUNK_THRESHOLD)) { + IOPosBuffer &io = input.back(); + IOPosBuffer &outio = output.back(); + if (io.size() > chunksize) { + IOSize consumed; + if (!output.empty() && (outio.size() < XRD_CL_MAX_CHUNK) && + (outio.offset() + static_cast(outio.size()) == io.offset())) { + if (outio.size() + chunksize > XRD_CL_MAX_CHUNK) { + consumed = (XRD_CL_MAX_CHUNK - outio.size()); + outio.set_size(XRD_CL_MAX_CHUNK); + } else { + consumed = chunksize; + outio.set_size(outio.size() + consumed); } + } else { + consumed = chunksize; + output.emplace_back(IOPosBuffer(io.offset(), io.data(), chunksize)); + } + chunksize -= consumed; + IOSize newsize = io.size() - consumed; + IOOffset newoffset = io.offset() + consumed; + void *newdata = static_cast(io.data()) + consumed; + io.set_offset(newoffset); + io.set_data(newdata); + io.set_size(newsize); + } else if (io.size() == 0) { + input.pop_back(); + } else { + output.push_back(io); + chunksize -= io.size(); + input.pop_back(); } + } } -static IOSize validateList(const std::vector req) -{ - IOSize total = 0; - off_t last_offset = -1; - for (const auto & it : req) - { - total += it.size(); - assert(it.offset() > last_offset); - last_offset = it.offset(); - assert(it.size() <= XRD_CL_MAX_CHUNK); - assert(it.offset() < 0x1ffffffffff); - } - assert(req.size() <= 1024); - return total; +static IOSize validateList(const std::vector req) { + IOSize total = 0; + off_t last_offset = -1; + for (const auto &it : req) { + total += it.size(); + assert(it.offset() > last_offset); + last_offset = it.offset(); + assert(it.size() <= XRD_CL_MAX_CHUNK); + assert(it.offset() < 0x1ffffffffff); + } + assert(req.size() <= 1024); + return total; } -void -XrdAdaptor::RequestManager::splitClientRequest(const std::vector &iolist, std::vector &req1, std::vector &req2, std::vector> const& activeSources) const -{ - if (iolist.empty()) return; - std::vector tmp_iolist(iolist.begin(), iolist.end()); - req1.reserve(iolist.size()/2+1); - req2.reserve(iolist.size()/2+1); - size_t front=0; - - // The quality of both is increased by 5 to prevent strange effects if quality is 0 for one source. - float q1 = static_cast(activeSources[0]->getQuality())+5; - float q2 = static_cast(activeSources[1]->getQuality())+5; - IOSize chunk1, chunk2; - // Make sure the chunk size is at least 1024; little point to reads less than that size. - chunk1 = std::max(static_cast(static_cast(XRD_CL_MAX_CHUNK)*(q2*q2/(q1*q1+q2*q2))), static_cast(1024)); - chunk2 = std::max(static_cast(static_cast(XRD_CL_MAX_CHUNK)*(q1*q1/(q1*q1+q2*q2))), static_cast(1024)); - - IOSize size_orig = 0; - for (const auto & it : iolist) size_orig += it.size(); - - while (tmp_iolist.size()-front > 0) - { - if ((req1.size() >= XRD_ADAPTOR_CHUNK_THRESHOLD) && (req2.size() >= XRD_ADAPTOR_CHUNK_THRESHOLD)) - { // The XrdFile::readv implementation should guarantee that no more than approximately 1024 chunks - // are passed to the request manager. However, because we have a max chunk size, we increase - // the total number slightly. Theoretically, it's possible an individual readv of total size >2GB where - // each individual chunk is >1MB could result in this firing. However, within the context of CMSSW, - // this cannot happen (ROOT uses readv for TTreeCache; TTreeCache size is 20MB). - edm::Exception ex(edm::errors::FileReadError); - ex << "XrdAdaptor::RequestManager::splitClientRequest(name='" << m_name - << "', flags=0x" << std::hex << m_flags - << ", permissions=0" << std::oct << m_perms << std::dec - << ") => Unable to split request between active servers. This is an unexpected internal error and should be reported to CMSSW developers."; - ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); - addConnections(ex); - std::stringstream ss; ss << "Original request size " << iolist.size() << "(" << size_orig << " bytes)"; - ex.addAdditionalInfo(ss.str()); - std::stringstream ss2; ss2 << "Quality source 1 " << q1-5 << ", quality source 2: " << q2-5; - ex.addAdditionalInfo(ss2.str()); - throw ex; - } - if (req1.size() < XRD_ADAPTOR_CHUNK_THRESHOLD) {consumeChunkFront(front, tmp_iolist, req1, chunk1);} - if (req2.size() < XRD_ADAPTOR_CHUNK_THRESHOLD) {consumeChunkBack(front, tmp_iolist, req2, chunk2);} +void XrdAdaptor::RequestManager::splitClientRequest(const std::vector &iolist, + std::vector &req1, + std::vector &req2, + std::vector> const &activeSources) const { + if (iolist.empty()) + return; + std::vector tmp_iolist(iolist.begin(), iolist.end()); + req1.reserve(iolist.size() / 2 + 1); + req2.reserve(iolist.size() / 2 + 1); + size_t front = 0; + + // The quality of both is increased by 5 to prevent strange effects if quality is 0 for one source. + float q1 = static_cast(activeSources[0]->getQuality()) + 5; + float q2 = static_cast(activeSources[1]->getQuality()) + 5; + IOSize chunk1, chunk2; + // Make sure the chunk size is at least 1024; little point to reads less than that size. + chunk1 = std::max(static_cast(static_cast(XRD_CL_MAX_CHUNK) * (q2 * q2 / (q1 * q1 + q2 * q2))), + static_cast(1024)); + chunk2 = std::max(static_cast(static_cast(XRD_CL_MAX_CHUNK) * (q1 * q1 / (q1 * q1 + q2 * q2))), + static_cast(1024)); + + IOSize size_orig = 0; + for (const auto &it : iolist) + size_orig += it.size(); + + while (tmp_iolist.size() - front > 0) { + if ((req1.size() >= XRD_ADAPTOR_CHUNK_THRESHOLD) && + (req2.size() >= + XRD_ADAPTOR_CHUNK_THRESHOLD)) { // The XrdFile::readv implementation should guarantee that no more than approximately 1024 chunks + // are passed to the request manager. However, because we have a max chunk size, we increase + // the total number slightly. Theoretically, it's possible an individual readv of total size >2GB where + // each individual chunk is >1MB could result in this firing. However, within the context of CMSSW, + // this cannot happen (ROOT uses readv for TTreeCache; TTreeCache size is 20MB). + edm::Exception ex(edm::errors::FileReadError); + ex << "XrdAdaptor::RequestManager::splitClientRequest(name='" << m_name << "', flags=0x" << std::hex << m_flags + << ", permissions=0" << std::oct << m_perms << std::dec + << ") => Unable to split request between active servers. This is an unexpected internal error and should be " + "reported to CMSSW developers."; + ex.addContext("In XrdAdaptor::RequestManager::requestFailure()"); + addConnections(ex); + std::stringstream ss; + ss << "Original request size " << iolist.size() << "(" << size_orig << " bytes)"; + ex.addAdditionalInfo(ss.str()); + std::stringstream ss2; + ss2 << "Quality source 1 " << q1 - 5 << ", quality source 2: " << q2 - 5; + ex.addAdditionalInfo(ss2.str()); + throw ex; } - std::sort(req1.begin(), req1.end(), [](const IOPosBuffer & left, const IOPosBuffer & right){return left.offset() < right.offset();}); - std::sort(req2.begin(), req2.end(), [](const IOPosBuffer & left, const IOPosBuffer & right){return left.offset() < right.offset();}); - - IOSize size1 = validateList(req1); - IOSize size2 = validateList(req2); + if (req1.size() < XRD_ADAPTOR_CHUNK_THRESHOLD) { + consumeChunkFront(front, tmp_iolist, req1, chunk1); + } + if (req2.size() < XRD_ADAPTOR_CHUNK_THRESHOLD) { + consumeChunkBack(front, tmp_iolist, req2, chunk2); + } + } + std::sort(req1.begin(), req1.end(), [](const IOPosBuffer &left, const IOPosBuffer &right) { + return left.offset() < right.offset(); + }); + std::sort(req2.begin(), req2.end(), [](const IOPosBuffer &left, const IOPosBuffer &right) { + return left.offset() < right.offset(); + }); - assert(size_orig == size1 + size2); + IOSize size1 = validateList(req1); + IOSize size2 = validateList(req2); - edm::LogVerbatim("XrdAdaptorInternal") << "Original request size " << iolist.size() << " (" << size_orig << " bytes) split into requests size " << req1.size() << " (" << size1 << " bytes) and " << req2.size() << " (" << size2 << " bytes)" << std::endl; -} + assert(size_orig == size1 + size2); -XrdAdaptor::RequestManager::OpenHandler::OpenHandler(std::weak_ptr manager) - : m_manager(manager) -{ + edm::LogVerbatim("XrdAdaptorInternal") << "Original request size " << iolist.size() << " (" << size_orig + << " bytes) split into requests size " << req1.size() << " (" << size1 + << " bytes) and " << req2.size() << " (" << size2 << " bytes)" << std::endl; } +XrdAdaptor::RequestManager::OpenHandler::OpenHandler(std::weak_ptr manager) : m_manager(manager) {} - // Cannot use ~OpenHandler=default as XrdCl::File is not fully - // defined in the header. -XrdAdaptor::RequestManager::OpenHandler::~OpenHandler() -{ -} - +// Cannot use ~OpenHandler=default as XrdCl::File is not fully +// defined in the header. +XrdAdaptor::RequestManager::OpenHandler::~OpenHandler() {} -void -XrdAdaptor::RequestManager::OpenHandler::HandleResponseWithHosts(XrdCl::XRootDStatus *status_ptr, XrdCl::AnyObject *, XrdCl::HostList *hostList_ptr) -{ +void XrdAdaptor::RequestManager::OpenHandler::HandleResponseWithHosts(XrdCl::XRootDStatus *status_ptr, + XrdCl::AnyObject *, + XrdCl::HostList *hostList_ptr) { // Make sure we get rid of the strong self-reference when the callback finishes. std::shared_ptr self = m_self; m_self.reset(); @@ -1078,17 +974,17 @@ XrdAdaptor::RequestManager::OpenHandler::HandleResponseWithHosts(XrdCl::XRootDSt // NOTE: as in XrdCl::File (synchronous), we ignore the response object. // Make sure that we set m_outstanding_open to false on exit from this function. // NOTE: we need to pass non-nullptr to unique_ptr in order for the guard to run - std::unique_ptr> outstanding_guard(this, [&](OpenHandler*){m_outstanding_open=false;}); + std::unique_ptr> outstanding_guard( + this, [&](OpenHandler *) { m_outstanding_open = false; }); std::shared_ptr source; std::unique_ptr status(status_ptr); std::unique_ptr hostList(hostList_ptr); auto manager = m_manager.lock(); - // Manager object has already been deleted. Cleanup the - // response objects, remove our self-reference, and ignore the response. - if (!manager) - { + // Manager object has already been deleted. Cleanup the + // response objects, remove our self-reference, and ignore the response. + if (!manager) { return; } //if we need to delete the File object we must do it outside @@ -1097,119 +993,107 @@ XrdAdaptor::RequestManager::OpenHandler::HandleResponseWithHosts(XrdCl::XRootDSt { std::lock_guard sentry(m_mutex); - if (status->IsOK()) - { - SendMonitoringInfo(*m_file); - timespec now; - GET_CLOCK_MONOTONIC(now); + if (status->IsOK()) { + SendMonitoringInfo(*m_file); + timespec now; + GET_CLOCK_MONOTONIC(now); - std::string excludeString; - Source::determineHostExcludeString(*m_file, hostList.get(), excludeString); + std::string excludeString; + Source::determineHostExcludeString(*m_file, hostList.get(), excludeString); - source.reset(new Source(now, std::move(m_file), excludeString)); - m_promise.set_value(source); - } - else - { - releaseFile = std::move(m_file); - edm::Exception ex(edm::errors::FileOpenError); - ex << "XrdCl::File::Open(name='" << manager->m_name - << "', flags=0x" << std::hex << manager->m_flags - << ", permissions=0" << std::oct << manager->m_perms << std::dec - << ") => error '" << status->ToStr() - << "' (errno=" << status->errNo << ", code=" << status->code << ")"; - ex.addContext("In XrdAdaptor::RequestManager::OpenHandler::HandleResponseWithHosts()"); - manager->addConnections(ex); - - m_promise.set_exception(std::make_exception_ptr(ex)); + source.reset(new Source(now, std::move(m_file), excludeString)); + m_promise.set_value(source); + } else { + releaseFile = std::move(m_file); + edm::Exception ex(edm::errors::FileOpenError); + ex << "XrdCl::File::Open(name='" << manager->m_name << "', flags=0x" << std::hex << manager->m_flags + << ", permissions=0" << std::oct << manager->m_perms << std::dec << ") => error '" << status->ToStr() + << "' (errno=" << status->errNo << ", code=" << status->code << ")"; + ex.addContext("In XrdAdaptor::RequestManager::OpenHandler::HandleResponseWithHosts()"); + manager->addConnections(ex); + + m_promise.set_exception(std::make_exception_ptr(ex)); } } manager->handleOpen(*status, source); } -std::string -XrdAdaptor::RequestManager::OpenHandler::current_source() -{ - std::lock_guard sentry(m_mutex); +std::string XrdAdaptor::RequestManager::OpenHandler::current_source() { + std::lock_guard sentry(m_mutex); - if (!m_file.get()) - { - return "(no open in progress)"; - } - std::string dataServer; - m_file->GetProperty("DataServer", dataServer); - if (dataServer.empty()) { return "(unknown source)"; } - return dataServer; + if (!m_file.get()) { + return "(no open in progress)"; + } + std::string dataServer; + m_file->GetProperty("DataServer", dataServer); + if (dataServer.empty()) { + return "(unknown source)"; + } + return dataServer; } -std::shared_future > -XrdAdaptor::RequestManager::OpenHandler::open() -{ - auto manager_ptr = m_manager.lock(); - if (!manager_ptr) - { - edm::Exception ex(edm::errors::LogicError); - ex << "XrdCl::File::Open() =>" - << " error: OpenHandler called within an invalid RequestManager context." - << " This is a logic error and should be reported to the CMSSW developers."; - ex.addContext("Calling XrdAdaptor::RequestManager::OpenHandler::open()"); - throw ex; - } - RequestManager &manager = *manager_ptr; - auto self_ptr = m_self_weak.lock(); - if (!self_ptr) - { - edm::Exception ex(edm::errors::LogicError); - ex << "XrdCl::File::Open() => error: " - << "OpenHandler called after it was deleted. This is a logic error " - << "and should be reported to the CMSSW developers."; - ex.addContext("Calling XrdAdapter::RequestManager::OpenHandler::open()"); - throw ex; - } +std::shared_future> XrdAdaptor::RequestManager::OpenHandler::open() { + auto manager_ptr = m_manager.lock(); + if (!manager_ptr) { + edm::Exception ex(edm::errors::LogicError); + ex << "XrdCl::File::Open() =>" + << " error: OpenHandler called within an invalid RequestManager context." + << " This is a logic error and should be reported to the CMSSW developers."; + ex.addContext("Calling XrdAdaptor::RequestManager::OpenHandler::open()"); + throw ex; + } + RequestManager &manager = *manager_ptr; + auto self_ptr = m_self_weak.lock(); + if (!self_ptr) { + edm::Exception ex(edm::errors::LogicError); + ex << "XrdCl::File::Open() => error: " + << "OpenHandler called after it was deleted. This is a logic error " + << "and should be reported to the CMSSW developers."; + ex.addContext("Calling XrdAdapter::RequestManager::OpenHandler::open()"); + throw ex; + } - // NOTE NOTE: we look at this variable *without* the lock. This means the method - // is not thread-safe; the caller is responsible to verify it is not called from - // multiple threads simultaneously. - // - // This is done because ::open may be called from a Xrootd callback; if we - // tried to hold m_mutex here, this object's callback may also be active, hold m_mutex, - // and make a call into xrootd (when it invokes m_file.reset()). Hence, our callback - // holds our mutex and attempts to grab an Xrootd mutex; RequestManager::requestFailure holds - // an Xrootd mutex and tries to hold m_mutex. This is a classic deadlock. - if (m_outstanding_open) - { - return m_shared_future; - } - std::lock_guard sentry(m_mutex); - std::promise > new_promise; - m_promise.swap(new_promise); - m_shared_future = m_promise.get_future().share(); - - auto opaque = manager.prepareOpaqueString(); - std::string new_name = manager.m_name + ((manager.m_name.find("?") == manager.m_name.npos) ? "?" : "&") + opaque; - edm::LogVerbatim("XrdAdaptorInternal") << "Trying to open URL: " << new_name; - m_file.reset(new XrdCl::File()); - m_outstanding_open = true; - - // Always make sure we release m_file and set m_outstanding_open to false on error. - std::unique_ptr> exit_guard(this, [&](OpenHandler*){m_outstanding_open = false; m_file.reset();}); - - XrdCl::XRootDStatus status; - if (!(status = m_file->Open(new_name, manager.m_flags, manager.m_perms, this)).IsOK()) - { - edm::Exception ex(edm::errors::FileOpenError); - ex << "XrdCl::File::Open(name='" << new_name - << "', flags=0x" << std::hex << manager.m_flags - << ", permissions=0" << std::oct << manager.m_perms << std::dec - << ") => error '" << status.ToStr() - << "' (errno=" << status.errNo << ", code=" << status.code << ")"; - ex.addContext("Calling XrdAdaptor::RequestManager::OpenHandler::open()"); - manager.addConnections(ex); - throw ex; - } - exit_guard.release(); - // Have a strong self-reference for as long as the callback is in-progress. - m_self = self_ptr; + // NOTE NOTE: we look at this variable *without* the lock. This means the method + // is not thread-safe; the caller is responsible to verify it is not called from + // multiple threads simultaneously. + // + // This is done because ::open may be called from a Xrootd callback; if we + // tried to hold m_mutex here, this object's callback may also be active, hold m_mutex, + // and make a call into xrootd (when it invokes m_file.reset()). Hence, our callback + // holds our mutex and attempts to grab an Xrootd mutex; RequestManager::requestFailure holds + // an Xrootd mutex and tries to hold m_mutex. This is a classic deadlock. + if (m_outstanding_open) { return m_shared_future; + } + std::lock_guard sentry(m_mutex); + std::promise> new_promise; + m_promise.swap(new_promise); + m_shared_future = m_promise.get_future().share(); + + auto opaque = manager.prepareOpaqueString(); + std::string new_name = manager.m_name + ((manager.m_name.find("?") == manager.m_name.npos) ? "?" : "&") + opaque; + edm::LogVerbatim("XrdAdaptorInternal") << "Trying to open URL: " << new_name; + m_file.reset(new XrdCl::File()); + m_outstanding_open = true; + + // Always make sure we release m_file and set m_outstanding_open to false on error. + std::unique_ptr> exit_guard(this, [&](OpenHandler *) { + m_outstanding_open = false; + m_file.reset(); + }); + + XrdCl::XRootDStatus status; + if (!(status = m_file->Open(new_name, manager.m_flags, manager.m_perms, this)).IsOK()) { + edm::Exception ex(edm::errors::FileOpenError); + ex << "XrdCl::File::Open(name='" << new_name << "', flags=0x" << std::hex << manager.m_flags << ", permissions=0" + << std::oct << manager.m_perms << std::dec << ") => error '" << status.ToStr() << "' (errno=" << status.errNo + << ", code=" << status.code << ")"; + ex.addContext("Calling XrdAdaptor::RequestManager::OpenHandler::open()"); + manager.addConnections(ex); + throw ex; + } + exit_guard.release(); + // Have a strong self-reference for as long as the callback is in-progress. + m_self = self_ptr; + return m_shared_future; } - diff --git a/Utilities/XrdAdaptor/src/XrdRequestManager.h b/Utilities/XrdAdaptor/src/XrdRequestManager.h index 01f1b520bd8c4..894e8409d071c 100644 --- a/Utilities/XrdAdaptor/src/XrdRequestManager.h +++ b/Utilities/XrdAdaptor/src/XrdRequestManager.h @@ -19,54 +19,44 @@ #include "XrdSource.h" namespace XrdCl { - class File; + class File; } - namespace XrdAdaptor { -struct SourceHash { - using Key =std::shared_ptr; - size_t operator()(const Key& iKey) const { - return tbb::tbb_hasher(iKey.get()); - } + struct SourceHash { + using Key = std::shared_ptr; + size_t operator()(const Key &iKey) const { return tbb::tbb_hasher(iKey.get()); } }; - -class XrootdException : public edm::Exception { - -public: - - XrootdException(XrdCl::Status & xrootd_status, edm::Exception::Code code) - : Exception(code), m_code(xrootd_status.code) - {} + class XrootdException : public edm::Exception { + public: + XrootdException(XrdCl::Status &xrootd_status, edm::Exception::Code code) + : Exception(code), m_code(xrootd_status.code) {} - ~XrootdException() noexcept override {}; + ~XrootdException() noexcept override{}; uint16_t getCode() { return m_code; } -private: - + private: uint16_t m_code; -}; - -class RequestManager : boost::noncopyable { + }; -public: - static const unsigned int XRD_DEFAULT_TIMEOUT = 3*60; + class RequestManager : boost::noncopyable { + public: + static const unsigned int XRD_DEFAULT_TIMEOUT = 3 * 60; virtual ~RequestManager() = default; /** * Interface for handling a client request. */ - std::future handle(void * into, IOSize size, IOOffset off) - { - auto c_ptr = std::make_shared(*this, into, size, off); - return handle(c_ptr); + std::future handle(void *into, IOSize size, IOOffset off) { + auto c_ptr = std::make_shared(*this, into, size, off); + return handle(c_ptr); } - std::future handle(std::shared_ptr > iolist); + std::future handle(std::shared_ptr> iolist); /** * Handle a client request. @@ -85,14 +75,14 @@ class RequestManager : boost::noncopyable { * Retrieve the names of the active sources * (primarily meant to enable meaningful log messages). */ - void getActiveSourceNames(std::vector & sources) const; - void getPrettyActiveSourceNames(std::vector & sources) const; + void getActiveSourceNames(std::vector &sources) const; + void getPrettyActiveSourceNames(std::vector &sources) const; /** * Retrieve the names of the disabled sources * (primarily meant to enable meaningful log messages). */ - void getDisabledSourceNames(std::vector & sources) const; + void getDisabledSourceNames(std::vector &sources) const; /** * Return a pointer to an active file. Useful for metadata @@ -108,7 +98,7 @@ class RequestManager : boost::noncopyable { /** * Return current filename */ - const std::string & getFilename() const {return m_name;} + const std::string &getFilename() const { return m_name; } /** * Some of the callback handlers need a weak_ptr reference to the RequestManager. @@ -117,17 +107,16 @@ class RequestManager : boost::noncopyable { * * Hence, all instances need to be created through this factory function. */ - static std::shared_ptr - getInstance(const std::string &filename, XrdCl::OpenFlags::Flags flags, XrdCl::Access::Mode perms) - { - std::shared_ptr instance(new RequestManager(filename, flags, perms)); - instance->initialize(instance); - return instance; + static std::shared_ptr getInstance(const std::string &filename, + XrdCl::OpenFlags::Flags flags, + XrdCl::Access::Mode perms) { + std::shared_ptr instance(new RequestManager(filename, flags, perms)); + instance->initialize(instance); + return instance; } -private: - - RequestManager(const std::string & filename, XrdCl::OpenFlags::Flags flags, XrdCl::Access::Mode perms); + private: + RequestManager(const std::string &filename, XrdCl::OpenFlags::Flags flags, XrdCl::Access::Mode perms); /** * Some of the callback handlers (particularly, file-open one) will want to call back into @@ -146,8 +135,9 @@ class RequestManager : boost::noncopyable { * Given a client request, split it into two requests lists. */ void splitClientRequest(const std::vector &iolist, - std::vector &req1, std::vector &req2, - std::vector> const& activeSources) const; + std::vector &req1, + std::vector &req2, + std::vector> const &activeSources) const; /** * Given a request, broadcast it to all sources. @@ -162,12 +152,14 @@ class RequestManager : boost::noncopyable { * The source check is somewhat expensive so it is only done once every * second. */ - void checkSources(timespec &now, IOSize requestSize, - std::vector>& activeSources, - std::vector>& inactiveSources); // TODO: inline - void checkSourcesImpl(timespec &now, IOSize requestSize, - std::vector>& activeSources, - std::vector>& inactiveSources); + void checkSources(timespec &now, + IOSize requestSize, + std::vector> &activeSources, + std::vector> &inactiveSources); // TODO: inline + void checkSourcesImpl(timespec &now, + IOSize requestSize, + std::vector> &activeSources, + std::vector> &inactiveSources); /** * Helper function for checkSources; compares the quality of source A * versus source B; if source A is significantly worse, remove it from @@ -176,17 +168,19 @@ class RequestManager : boost::noncopyable { * NOTE: assumes two sources are active and the caller must already hold * m_source_mutex */ - bool compareSources(const timespec &now, unsigned a, unsigned b, - std::vector>& activeSources, - std::vector>& inactiveSources) const; + bool compareSources(const timespec &now, + unsigned a, + unsigned b, + std::vector> &activeSources, + std::vector> &inactiveSources) const; /** * Anytime we potentially switch sources, update the internal site source list; * alert the user if necessary. */ - void reportSiteChange(std::vector > const& iOld, - std::vector > const& iNew, - std::string orig_site=std::string{}) const; + void reportSiteChange(std::vector> const &iOld, + std::vector> const &iNew, + std::string orig_site = std::string{}) const; /** * Update the StatisticsSenderService, if necessary, with the current server. @@ -209,16 +203,16 @@ class RequestManager : boost::noncopyable { * Note these member variables can only be accessed when the source mutex * is held. */ - std::vector > m_activeSources; - std::vector > m_inactiveSources; - + std::vector> m_activeSources; + std::vector> m_inactiveSources; + tbb::concurrent_unordered_set m_disabledSourceStrings; tbb::concurrent_unordered_set m_disabledExcludeStrings; tbb::concurrent_unordered_set, SourceHash> m_disabledSources; // StatisticsSenderService wants to know what our current server is; // this holds last-successfully-opened server name - std::atomic m_serverToAdvertise; + std::atomic m_serverToAdvertise; timespec m_lastSourceCheck; int m_timeout; @@ -239,25 +233,24 @@ class RequestManager : boost::noncopyable { std::atomic m_excluded_active_count; class OpenHandler : boost::noncopyable, public XrdCl::ResponseHandler { - public: + static std::shared_ptr getInstance(std::weak_ptr manager) { + OpenHandler *instance_ptr = new OpenHandler(manager); + std::shared_ptr instance(instance_ptr); + instance_ptr->m_self_weak = instance; + return instance; + } - static std::shared_ptr getInstance(std::weak_ptr manager) - { - OpenHandler *instance_ptr = new OpenHandler(manager); - std::shared_ptr instance(instance_ptr); - instance_ptr->m_self_weak = instance; - return instance; - } - - ~OpenHandler() override; + ~OpenHandler() override; - /** + /** * Handle the file-open response */ - void HandleResponseWithHosts(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList) override; + void HandleResponseWithHosts(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response, + XrdCl::HostList *hostList) override; - /** + /** * Future-based version of the handler * If called while a file-open is in progress, we will not start a new file-open. * Instead, the callback will be fired for the ongoing open. @@ -266,37 +259,36 @@ class RequestManager : boost::noncopyable { * The caller must ensure it is not called from multiple threads at once * for this object. */ - std::shared_future > open(); + std::shared_future> open(); - /** + /** * Returns the current source server name. Useful primarily for debugging. */ - std::string current_source(); + std::string current_source(); private: - - OpenHandler(std::weak_ptr manager); - std::shared_future > m_shared_future; - std::promise > m_promise; - // Set to true only when there is an outstanding open request; not - // protected by m_mutex, so the caller is required to know it is in a - // thread-safe context. - std::atomic m_outstanding_open {false}; - // Can only be touched when m_mutex is held. - std::unique_ptr m_file; - std::recursive_mutex m_mutex; - std::shared_ptr m_self; - - // Always maintain a weak self-reference; when the open is in-progress, - // this is upgraded to a strong reference to prevent this object from - // deletion as long as XrdCl has not performed the callback. - std::weak_ptr m_self_weak; - std::weak_ptr m_manager; + OpenHandler(std::weak_ptr manager); + std::shared_future> m_shared_future; + std::promise> m_promise; + // Set to true only when there is an outstanding open request; not + // protected by m_mutex, so the caller is required to know it is in a + // thread-safe context. + std::atomic m_outstanding_open{false}; + // Can only be touched when m_mutex is held. + std::unique_ptr m_file; + std::recursive_mutex m_mutex; + std::shared_ptr m_self; + + // Always maintain a weak self-reference; when the open is in-progress, + // this is upgraded to a strong reference to prevent this object from + // deletion as long as XrdCl has not performed the callback. + std::weak_ptr m_self_weak; + std::weak_ptr m_manager; }; std::shared_ptr m_open_handler; -}; + }; -} +} // namespace XrdAdaptor #endif diff --git a/Utilities/XrdAdaptor/src/XrdSource.cc b/Utilities/XrdAdaptor/src/XrdSource.cc index 571d6b0bcb304..420ea64d8ff87 100644 --- a/Utilities/XrdAdaptor/src/XrdSource.cc +++ b/Utilities/XrdAdaptor/src/XrdSource.cc @@ -18,16 +18,16 @@ #include "QualityMetric.h" #include "XrdStatistics.h" -#define MAX_REQUEST 256*1024 -#define XRD_CL_MAX_CHUNK 512*1024 +#define MAX_REQUEST 256 * 1024 +#define XRD_CL_MAX_CHUNK 512 * 1024 #ifdef XRD_FAKE_SLOW //#define XRD_DELAY 5140 #define XRD_DELAY 1000 #define XRD_SLOW_RATE 2 -std::atomic g_delayCount {0}; +std::atomic g_delayCount{0}; #else -std::atomic g_delayCount {0}; +std::atomic g_delayCount{0}; #endif using namespace XrdAdaptor; @@ -36,157 +36,143 @@ using namespace XrdAdaptor; // inactive anyway!) can even timeout. Rather than wait around for // a few minutes in the main thread, this class asynchronously closes // and deletes the XrdCl::File -class DelayedClose : boost::noncopyable, public XrdCl::ResponseHandler -{ +class DelayedClose : boost::noncopyable, public XrdCl::ResponseHandler { public: - - DelayedClose(std::shared_ptr fh, const std::string &id, const std::string &site) - : m_fh(std::move(fh)), - m_id(id), - m_site(site) - { - if (m_fh && m_fh->IsOpen()) - { - if (!m_fh->Close(this).IsOK()) - { - delete this; - } - } + DelayedClose(std::shared_ptr fh, const std::string &id, const std::string &site) + : m_fh(std::move(fh)), m_id(id), m_site(site) { + if (m_fh && m_fh->IsOpen()) { + if (!m_fh->Close(this).IsOK()) { + delete this; + } } + } + ~DelayedClose() override = default; - ~DelayedClose() override = default; - - - void HandleResponseWithHosts(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList) override - { - if (status && !status->IsOK()) - { - - edm::LogWarning("XrdFileWarning") << "Source delayed close failed with error '" << status->ToStr() - << "' (errno=" << status->errNo << ", code=" << status->code << ", server=" << m_id << ", site=" << m_site << ")"; - } - delete status; - delete hostList; - // NOTE: we do not delete response (copying behavior from XrdCl). - delete this; + void HandleResponseWithHosts(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response, + XrdCl::HostList *hostList) override { + if (status && !status->IsOK()) { + edm::LogWarning("XrdFileWarning") << "Source delayed close failed with error '" << status->ToStr() + << "' (errno=" << status->errNo << ", code=" << status->code + << ", server=" << m_id << ", site=" << m_site << ")"; } - + delete status; + delete hostList; + // NOTE: we do not delete response (copying behavior from XrdCl). + delete this; + } private: - edm::propagate_const> m_fh; - std::string m_id; - std::string m_site; + edm::propagate_const> m_fh; + std::string m_id; + std::string m_site; }; - /** * A handler for querying a XrdCl::FileSystem object which is safe to be * invoked from an XrdCl callback (that is, we don't need an available callback * thread to timeout). */ -class QueryAttrHandler : public XrdCl::ResponseHandler -{ - friend std::unique_ptr std::make_unique(); +class QueryAttrHandler : public XrdCl::ResponseHandler { + friend std::unique_ptr std::make_unique(); public: + ~QueryAttrHandler() override = default; + QueryAttrHandler(const QueryAttrHandler &) = delete; + QueryAttrHandler &operator=(const QueryAttrHandler &) = delete; + + static XrdCl::XRootDStatus query(XrdCl::FileSystem &fs, + const std::string &attr, + std::chrono::milliseconds timeout, + std::string &result) { + auto handler = std::make_unique(); + auto l_state = std::make_shared(); + handler->m_state = l_state; + XrdCl::Buffer arg(attr.size()); + arg.FromString(attr); + + XrdCl::XRootDStatus st = fs.Query(XrdCl::QueryCode::Config, arg, handler.get()); + if (!st.IsOK()) { + return st; + } - ~QueryAttrHandler() override = default; - QueryAttrHandler(const QueryAttrHandler&) = delete; - QueryAttrHandler& operator=(const QueryAttrHandler&) = delete; - - - static XrdCl::XRootDStatus query(XrdCl::FileSystem &fs, const std::string &attr, std::chrono::milliseconds timeout, std::string &result) - { - auto handler = std::make_unique(); - auto l_state = std::make_shared(); - handler->m_state = l_state; - XrdCl::Buffer arg(attr.size()); - arg.FromString(attr); - - XrdCl::XRootDStatus st = fs.Query(XrdCl::QueryCode::Config, arg, handler.get()); - if (!st.IsOK()) - { - return st; - } - - // Successfully registered the callback; it will always delete itself, so we shouldn't. - handler.release(); + // Successfully registered the callback; it will always delete itself, so we shouldn't. + handler.release(); - std::unique_lock guard(l_state->m_mutex); - // Wait until some status is available or a timeout. - l_state->m_condvar.wait_for(guard, timeout, [&]{return l_state->m_status.get();}); + std::unique_lock guard(l_state->m_mutex); + // Wait until some status is available or a timeout. + l_state->m_condvar.wait_for(guard, timeout, [&] { return l_state->m_status.get(); }); - if (l_state->m_status) - { - if (l_state->m_status->IsOK()) - { - result = l_state->m_response->ToString(); - } - return *(l_state->m_status); - } - else - { // We had a timeout; construct a reasonable message. - return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errSocketTimeout, 1, "Timeout when waiting for query callback."); - } + if (l_state->m_status) { + if (l_state->m_status->IsOK()) { + result = l_state->m_response->ToString(); + } + return *(l_state->m_status); + } else { // We had a timeout; construct a reasonable message. + return XrdCl::XRootDStatus( + XrdCl::stError, XrdCl::errSocketTimeout, 1, "Timeout when waiting for query callback."); } - + } private: + QueryAttrHandler() {} + + void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override { + // NOTE: we own the status and response pointers. + std::unique_ptr response_mgr; + response_mgr.reset(response); + + // Lock our state information then dispose of our object. + auto l_state = m_state.lock(); + delete this; + if (!l_state) { + return; + } - QueryAttrHandler() {} - + // On function exit, notify any waiting threads. + std::unique_ptr> notify_guard(nullptr, + [&](char *) { l_state->m_condvar.notify_all(); }); - void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) override { - // NOTE: we own the status and response pointers. - std::unique_ptr response_mgr; - response_mgr.reset(response); - - // Lock our state information then dispose of our object. - auto l_state = m_state.lock(); - delete this; - if (!l_state) {return;} - - // On function exit, notify any waiting threads. - std::unique_ptr> notify_guard(nullptr, [&](char *) {l_state->m_condvar.notify_all();}); - - { - // On exit from the block, make sure m_status is set; it needs to be set before we notify threads. - std::unique_ptr> exit_guard(nullptr, [&](char *) {if (!l_state->m_status) l_state->m_status.reset(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInternal));}); - if (!status) {return;} - if (status->IsOK()) - { - if (!response) {return;} - XrdCl::Buffer *buf_ptr; - response->Get(buf_ptr); - // AnyObject::Set lacks specialization for nullptr - response->Set(static_cast(nullptr)); - l_state->m_response.reset(buf_ptr); - } - l_state->m_status.reset(status); + // On exit from the block, make sure m_status is set; it needs to be set before we notify threads. + std::unique_ptr> exit_guard(nullptr, [&](char *) { + if (!l_state->m_status) + l_state->m_status.reset(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInternal)); + }); + if (!status) { + return; + } + if (status->IsOK()) { + if (!response) { + return; } + XrdCl::Buffer *buf_ptr; + response->Get(buf_ptr); + // AnyObject::Set lacks specialization for nullptr + response->Set(static_cast(nullptr)); + l_state->m_response.reset(buf_ptr); + } + l_state->m_status.reset(status); } - - - // Represents the current state of the callback. The parent class only manages a weak_ptr - // to the state. If the asynchronous callback cannot lock the weak_ptr, then it assumes the - // main thread has given up and doesn't touch any of the state variables. - struct QueryAttrState { - - // Synchronize between the callback thread and the main thread; condvar predicate - // is having m_status set. m_mutex protects m_status. - std::mutex m_mutex; - std::condition_variable m_condvar; - - // Results from the server - std::unique_ptr m_status; - std::unique_ptr m_response; - }; - std::weak_ptr m_state; + } + + // Represents the current state of the callback. The parent class only manages a weak_ptr + // to the state. If the asynchronous callback cannot lock the weak_ptr, then it assumes the + // main thread has given up and doesn't touch any of the state variables. + struct QueryAttrState { + // Synchronize between the callback thread and the main thread; condvar predicate + // is having m_status set. m_mutex protects m_status. + std::mutex m_mutex; + std::condition_variable m_condvar; + + // Results from the server + std::unique_ptr m_status; + std::unique_ptr m_response; + }; + std::weak_ptr m_state; }; - Source::Source(timespec now, std::unique_ptr fh, const std::string &exclude) : m_lastDowngrade({0, 0}), m_id("(unknown)"), @@ -194,269 +180,233 @@ Source::Source(timespec now, std::unique_ptr fh, const std::string m_fh(std::move(fh)), m_stats(nullptr) #ifdef XRD_FAKE_SLOW - , m_slow(++g_delayCount % XRD_SLOW_RATE == 0) - //, m_slow(++g_delayCount >= XRD_SLOW_RATE) - //, m_slow(true) + , + m_slow(++g_delayCount % XRD_SLOW_RATE == 0) +//, m_slow(++g_delayCount >= XRD_SLOW_RATE) +//, m_slow(true) #endif { - if (m_fh.get()) - { - if (!m_fh->GetProperty("DataServer", m_id)) - { - edm::LogWarning("XrdFileWarning") - << "Source::Source() failed to determine data server name.'"; - } - if (m_exclude.empty()) {m_exclude = m_id;} + if (m_fh.get()) { + if (!m_fh->GetProperty("DataServer", m_id)) { + edm::LogWarning("XrdFileWarning") << "Source::Source() failed to determine data server name.'"; } - m_qm = QualityMetricFactory::get(now, m_id); - m_prettyid = m_id + " (unknown site)"; - std::string domain_id; - if (getDomain(m_id, domain_id)) {m_site = domain_id;} - else {m_site = "Unknown (" + m_id + ")";} - setXrootdSite(); - assert(m_qm.get()); - assert(m_fh.get()); - XrdSiteStatisticsInformation *statsService = XrdSiteStatisticsInformation::getInstance(); - if (statsService) - { - m_stats = statsService->getStatisticsForSite(m_site); + if (m_exclude.empty()) { + m_exclude = m_id; } + } + m_qm = QualityMetricFactory::get(now, m_id); + m_prettyid = m_id + " (unknown site)"; + std::string domain_id; + if (getDomain(m_id, domain_id)) { + m_site = domain_id; + } else { + m_site = "Unknown (" + m_id + ")"; + } + setXrootdSite(); + assert(m_qm.get()); + assert(m_fh.get()); + XrdSiteStatisticsInformation *statsService = XrdSiteStatisticsInformation::getInstance(); + if (statsService) { + m_stats = statsService->getStatisticsForSite(m_site); + } } - -bool Source::getHostname(const std::string &id, std::string &hostname) -{ - size_t pos = id.find(":"); - hostname = id; - if ((pos != std::string::npos) && (pos > 0)) {hostname = id.substr(0, pos);} - - bool retval = true; - if (!hostname.empty() && ((hostname[0] == '[') || isdigit(hostname[0]))) - { - retval = false; - struct addrinfo hints; memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; - struct addrinfo *result; - if (!getaddrinfo(hostname.c_str(), nullptr, &hints, &result)) - { - std::vector host; host.reserve(256); - if (!getnameinfo(result->ai_addr, result->ai_addrlen, &host[0], 255, nullptr, 0, NI_NAMEREQD)) - { - hostname = &host[0]; - retval = true; - } - freeaddrinfo(result); - } +bool Source::getHostname(const std::string &id, std::string &hostname) { + size_t pos = id.find(":"); + hostname = id; + if ((pos != std::string::npos) && (pos > 0)) { + hostname = id.substr(0, pos); + } + + bool retval = true; + if (!hostname.empty() && ((hostname[0] == '[') || isdigit(hostname[0]))) { + retval = false; + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + struct addrinfo *result; + if (!getaddrinfo(hostname.c_str(), nullptr, &hints, &result)) { + std::vector host; + host.reserve(256); + if (!getnameinfo(result->ai_addr, result->ai_addrlen, &host[0], 255, nullptr, 0, NI_NAMEREQD)) { + hostname = &host[0]; + retval = true; + } + freeaddrinfo(result); } - return retval; + } + return retval; } +bool Source::getDomain(const std::string &host, std::string &domain) { + getHostname(host, domain); + size_t pos = domain.find("."); + if (pos != std::string::npos && (pos < domain.size())) { + domain = domain.substr(pos + 1); + } -bool Source::getDomain(const std::string &host, std::string &domain) -{ - getHostname(host, domain); - size_t pos = domain.find("."); - if (pos != std::string::npos && (pos < domain.size())) {domain = domain.substr(pos+1);} - - return !domain.empty(); + return !domain.empty(); } - -bool -Source::isDCachePool(XrdCl::File &file, const XrdCl::HostList *hostList) -{ - // WORKAROUND: On open-file recovery in the Xrootd client, it'll carry around the - // dCache opaque information to other sites, causing isDCachePool to erroneously return - // true. We are working with the upstream developers to solve this. - // - // For now, we see if the previous server also looks like a dCache pool - something that - // wouldn't happen at a real site, as the previous server should look like a dCache door. - std::string lastUrl; - file.GetProperty("LastURL", lastUrl); - if (!lastUrl.empty()) - { - bool result = isDCachePool(lastUrl); - if (result && hostList && (hostList->size() > 1)) - { - if (isDCachePool((*hostList)[hostList->size()-2].url.GetURL())) - { - return false; - } - return true; - } - return result; - } - return false; -} - -bool -Source::isDCachePool(const std::string &lastUrl) -{ - XrdCl::URL url(lastUrl); - XrdCl::URL::ParamsMap map = url.GetParams(); - // dCache pools always utilize this opaque identifier. - if (map.find("org.dcache.uuid") != map.end()) - { - return true; +bool Source::isDCachePool(XrdCl::File &file, const XrdCl::HostList *hostList) { + // WORKAROUND: On open-file recovery in the Xrootd client, it'll carry around the + // dCache opaque information to other sites, causing isDCachePool to erroneously return + // true. We are working with the upstream developers to solve this. + // + // For now, we see if the previous server also looks like a dCache pool - something that + // wouldn't happen at a real site, as the previous server should look like a dCache door. + std::string lastUrl; + file.GetProperty("LastURL", lastUrl); + if (!lastUrl.empty()) { + bool result = isDCachePool(lastUrl); + if (result && hostList && (hostList->size() > 1)) { + if (isDCachePool((*hostList)[hostList->size() - 2].url.GetURL())) { + return false; + } + return true; } - return false; + return result; + } + return false; } - -void -Source::determineHostExcludeString(XrdCl::File &file, const XrdCl::HostList *hostList, std::string &exclude) -{ - // Detect a dCache pool and, if we are in the federation context, give a custom - // exclude parameter. - // We assume this is a federation context if there's at least a regional, dCache door, - // and dCache pool server (so, more than 2 servers!). - - exclude = ""; - if (hostList && (hostList->size() > 3) && isDCachePool(file, hostList)) - { - const XrdCl::HostInfo &info = (*hostList)[hostList->size()-3]; - exclude = info.url.GetHostName(); - std::string lastUrl; file.GetProperty("LastURL", lastUrl); - edm::LogVerbatim("XrdAdaptorInternal") << "Changing exclude list for URL " << lastUrl << " to " << exclude; - } +bool Source::isDCachePool(const std::string &lastUrl) { + XrdCl::URL url(lastUrl); + XrdCl::URL::ParamsMap map = url.GetParams(); + // dCache pools always utilize this opaque identifier. + if (map.find("org.dcache.uuid") != map.end()) { + return true; + } + return false; } +void Source::determineHostExcludeString(XrdCl::File &file, const XrdCl::HostList *hostList, std::string &exclude) { + // Detect a dCache pool and, if we are in the federation context, give a custom + // exclude parameter. + // We assume this is a federation context if there's at least a regional, dCache door, + // and dCache pool server (so, more than 2 servers!). -bool -Source::getXrootdSite(XrdCl::File &fh, std::string &site) -{ + exclude = ""; + if (hostList && (hostList->size() > 3) && isDCachePool(file, hostList)) { + const XrdCl::HostInfo &info = (*hostList)[hostList->size() - 3]; + exclude = info.url.GetHostName(); std::string lastUrl; - fh.GetProperty("LastURL", lastUrl); - if (lastUrl.empty() || isDCachePool(lastUrl)) - { - std::string server, id; - if (!fh.GetProperty("DataServer", server)) {id = "(unknown)";} - else {id = server;} - if (lastUrl.empty()) {edm::LogWarning("XrdFileWarning") << "Unable to determine the URL associated with server " << id;} - site = "Unknown"; - if (!server.empty()) {getDomain(server, site);} - return false; - } - return getXrootdSiteFromURL(lastUrl, site); + file.GetProperty("LastURL", lastUrl); + edm::LogVerbatim("XrdAdaptorInternal") << "Changing exclude list for URL " << lastUrl << " to " << exclude; + } } -bool -Source::getXrootdSiteFromURL(std::string url, std::string &site) -{ - const std::string attr = "sitename"; - XrdCl::Buffer *response = nullptr; - XrdCl::Buffer arg( attr.size() ); - arg.FromString( attr ); - - XrdCl::FileSystem fs(url); - std::string rsite; - XrdCl::XRootDStatus st = QueryAttrHandler::query(fs, "sitename", std::chrono::seconds(1), rsite); - if (!st.IsOK()) - { - XrdCl::URL xurl(url); - getDomain(xurl.GetHostName(), site); - delete response; - return false; +bool Source::getXrootdSite(XrdCl::File &fh, std::string &site) { + std::string lastUrl; + fh.GetProperty("LastURL", lastUrl); + if (lastUrl.empty() || isDCachePool(lastUrl)) { + std::string server, id; + if (!fh.GetProperty("DataServer", server)) { + id = "(unknown)"; + } else { + id = server; } - if (!rsite.empty() && (rsite[rsite.size()-1] == '\n')) - { - rsite = rsite.substr(0, rsite.size()-1); + if (lastUrl.empty()) { + edm::LogWarning("XrdFileWarning") << "Unable to determine the URL associated with server " << id; } - if (rsite == "sitename") - { - XrdCl::URL xurl(url); - getDomain(xurl.GetHostName(), site); - return false; + site = "Unknown"; + if (!server.empty()) { + getDomain(server, site); } - site = rsite; - return true; + return false; + } + return getXrootdSiteFromURL(lastUrl, site); } -void -Source::setXrootdSite() -{ - std::string site; - bool goodSitename = getXrootdSite(*m_fh, site); - if (!goodSitename) - { - edm::LogInfo("XrdAdaptorInternal") - << "Xrootd server at " << m_id << " did not provide a sitename. Monitoring may be incomplete."; - } - else - { - m_site = site; - m_prettyid = m_id + " (site " + m_site + ")"; - } - edm::LogInfo("XrdAdaptorInternal") << "Reading from new server " << m_id << " at site " << m_site; +bool Source::getXrootdSiteFromURL(std::string url, std::string &site) { + const std::string attr = "sitename"; + XrdCl::Buffer *response = nullptr; + XrdCl::Buffer arg(attr.size()); + arg.FromString(attr); + + XrdCl::FileSystem fs(url); + std::string rsite; + XrdCl::XRootDStatus st = QueryAttrHandler::query(fs, "sitename", std::chrono::seconds(1), rsite); + if (!st.IsOK()) { + XrdCl::URL xurl(url); + getDomain(xurl.GetHostName(), site); + delete response; + return false; + } + if (!rsite.empty() && (rsite[rsite.size() - 1] == '\n')) { + rsite = rsite.substr(0, rsite.size() - 1); + } + if (rsite == "sitename") { + XrdCl::URL xurl(url); + getDomain(xurl.GetHostName(), site); + return false; + } + site = rsite; + return true; } -Source::~Source() -{ - new DelayedClose(fh(), m_id, m_site); +void Source::setXrootdSite() { + std::string site; + bool goodSitename = getXrootdSite(*m_fh, site); + if (!goodSitename) { + edm::LogInfo("XrdAdaptorInternal") << "Xrootd server at " << m_id + << " did not provide a sitename. Monitoring may be incomplete."; + } else { + m_site = site; + m_prettyid = m_id + " (site " + m_site + ")"; + } + edm::LogInfo("XrdAdaptorInternal") << "Reading from new server " << m_id << " at site " << m_site; } -std::shared_ptr -Source::getFileHandle() -{ - return fh(); -} +Source::~Source() { new DelayedClose(fh(), m_id, m_site); } -static void -validateList(const XrdCl::ChunkList& cl) -{ - off_t last_offset = -1; - for (const auto & ci : cl) - { - assert(static_cast(ci.offset) > last_offset); - last_offset = ci.offset; - assert(ci.length <= XRD_CL_MAX_CHUNK); - assert(ci.offset < 0x1ffffffffff); - assert(ci.offset > 0); - } - assert(cl.size() <= 1024); +std::shared_ptr Source::getFileHandle() { return fh(); } + +static void validateList(const XrdCl::ChunkList &cl) { + off_t last_offset = -1; + for (const auto &ci : cl) { + assert(static_cast(ci.offset) > last_offset); + last_offset = ci.offset; + assert(ci.length <= XRD_CL_MAX_CHUNK); + assert(ci.offset < 0x1ffffffffff); + assert(ci.offset > 0); + } + assert(cl.size() <= 1024); } -void -Source::handle(std::shared_ptr c) -{ - edm::LogVerbatim("XrdAdaptorInternal") << "Reading from " << ID() << ", quality " << m_qm->get() << std::endl; - c->m_source = shared_from_this(); - c->m_self_reference = c; - m_qm->startWatch(c->m_qmw); - if (m_stats) - { - std::shared_ptr readStats = XrdSiteStatistics::startRead(stats(), c); - c->setStatistics(readStats); - } +void Source::handle(std::shared_ptr c) { + edm::LogVerbatim("XrdAdaptorInternal") << "Reading from " << ID() << ", quality " << m_qm->get() << std::endl; + c->m_source = shared_from_this(); + c->m_self_reference = c; + m_qm->startWatch(c->m_qmw); + if (m_stats) { + std::shared_ptr readStats = XrdSiteStatistics::startRead(stats(), c); + c->setStatistics(readStats); + } #ifdef XRD_FAKE_SLOW - if (m_slow) std::this_thread::sleep_for(std::chrono::milliseconds(XRD_DELAY)); + if (m_slow) + std::this_thread::sleep_for(std::chrono::milliseconds(XRD_DELAY)); #endif - XrdCl::XRootDStatus status; - if (c->m_into) - { - // See notes in ClientRequest definition to understand this voodoo. - status = m_fh->Read(c->m_off, c->m_size, c->m_into, c.get()); - } - else - { - XrdCl::ChunkList cl; - cl.reserve(c->m_iolist->size()); - for (const auto & it : *c->m_iolist) - { - cl.emplace_back(it.offset(), it.size(), it.data()); - } - validateList(cl); - status = m_fh->VectorRead(cl, nullptr, c.get()); - } - - if (!status.IsOK()) - { - edm::Exception ex(edm::errors::FileReadError); - ex << "XrdFile::Read or XrdFile::VectorRead failed with error: '" - << status.ToStr() << "' (errNo = " << status.errNo << ")"; - ex.addContext("Calling Source::handle"); - throw ex; + XrdCl::XRootDStatus status; + if (c->m_into) { + // See notes in ClientRequest definition to understand this voodoo. + status = m_fh->Read(c->m_off, c->m_size, c->m_into, c.get()); + } else { + XrdCl::ChunkList cl; + cl.reserve(c->m_iolist->size()); + for (const auto &it : *c->m_iolist) { + cl.emplace_back(it.offset(), it.size(), it.data()); } + validateList(cl); + status = m_fh->VectorRead(cl, nullptr, c.get()); + } + + if (!status.IsOK()) { + edm::Exception ex(edm::errors::FileReadError); + ex << "XrdFile::Read or XrdFile::VectorRead failed with error: '" << status.ToStr() << "' (errNo = " << status.errNo + << ")"; + ex.addContext("Calling Source::handle"); + throw ex; + } } - diff --git a/Utilities/XrdAdaptor/src/XrdSource.h b/Utilities/XrdAdaptor/src/XrdSource.h index 63e3cc186d7d7..1dc7b9325f508 100644 --- a/Utilities/XrdAdaptor/src/XrdSource.h +++ b/Utilities/XrdAdaptor/src/XrdSource.h @@ -12,19 +12,18 @@ #include "QualityMetric.h" namespace XrdCl { - class File; + class File; } namespace XrdAdaptor { -class RequestList; -class ClientRequest; -class XrdSiteStatistics; -class XrdStatisticsService; + class RequestList; + class ClientRequest; + class XrdSiteStatistics; + class XrdStatisticsService; -class Source : public std::enable_shared_from_this, boost::noncopyable { - -public: + class Source : public std::enable_shared_from_this, boost::noncopyable { + public: Source(timespec now, std::unique_ptr fileHandle, const std::string &exclude); ~Source(); @@ -35,15 +34,17 @@ class Source : public std::enable_shared_from_this, boost::noncopyable { std::shared_ptr getFileHandle(); - const std::string & ID() const {return m_id;} - const std::string & Site() const {return m_site;} - const std::string & PrettyID() const {return m_prettyid;} - const std::string & ExcludeID() const {return m_exclude;} + const std::string &ID() const { return m_id; } + const std::string &Site() const { return m_site; } + const std::string &PrettyID() const { return m_prettyid; } + const std::string &ExcludeID() const { return m_exclude; } - unsigned getQuality() {return m_qm->get();} + unsigned getQuality() { return m_qm->get(); } - struct timespec getLastDowngrade() const {return m_lastDowngrade;} - void setLastDowngrade(struct timespec now) {m_lastDowngrade = now;} + struct timespec getLastDowngrade() const { + return m_lastDowngrade; + } + void setLastDowngrade(struct timespec now) { m_lastDowngrade = now; } static bool getDomain(const std::string &host, std::string &domain); static bool getXrootdSite(XrdCl::File &file, std::string &site); @@ -52,24 +53,24 @@ class Source : public std::enable_shared_from_this, boost::noncopyable { // Given a file and (possibly) a host list, determine the exclude string. static void determineHostExcludeString(XrdCl::File &file, const XrdCl::HostList *hostList, std::string &exclude); - // Given a connected File object, determine whether we believe this to be a + // Given a connected File object, determine whether we believe this to be a // dCache pool (dCache is a separate implementation and sometimes benefits from // implementation-specific behaviors. - static bool isDCachePool(XrdCl::File &file, const XrdCl::HostList *hostList=nullptr); + static bool isDCachePool(XrdCl::File &file, const XrdCl::HostList *hostList = nullptr); static bool isDCachePool(const std::string &url); // Given an Xrootd server ID, determine the hostname to the best of our ability. - static bool getHostname(const std::string & id, std::string &hostname); + static bool getHostname(const std::string &id, std::string &hostname); -private: + private: void requestCallback(/* TODO: type? */); void setXrootdSite(); - std::shared_ptr fh() const {return get_underlying_safe(m_fh);} - std::shared_ptr& fh() {return get_underlying_safe(m_fh);} - std::shared_ptr stats() const {return get_underlying_safe(m_stats);} - std::shared_ptr& stats() {return get_underlying_safe(m_stats);} + std::shared_ptr fh() const { return get_underlying_safe(m_fh); } + std::shared_ptr &fh() { return get_underlying_safe(m_fh); } + std::shared_ptr stats() const { return get_underlying_safe(m_stats); } + std::shared_ptr &stats() { return get_underlying_safe(m_stats); } struct timespec m_lastDowngrade; std::string m_id; @@ -84,8 +85,8 @@ class Source : public std::enable_shared_from_this, boost::noncopyable { #ifdef XRD_FAKE_SLOW bool m_slow; #endif -}; + }; -} +} // namespace XrdAdaptor #endif diff --git a/Utilities/XrdAdaptor/src/XrdStatistics.cc b/Utilities/XrdAdaptor/src/XrdStatistics.cc index f985ef08cec57..862c26015911f 100644 --- a/Utilities/XrdAdaptor/src/XrdStatistics.cc +++ b/Utilities/XrdAdaptor/src/XrdStatistics.cc @@ -13,189 +13,148 @@ using namespace XrdAdaptor; +std::atomic XrdSiteStatisticsInformation::m_instance; -std::atomic XrdSiteStatisticsInformation::m_instance; +XrdStatisticsService::XrdStatisticsService(const edm::ParameterSet &iPS, edm::ActivityRegistry &iRegistry) { + XrdSiteStatisticsInformation::createInstance(); - -XrdStatisticsService::XrdStatisticsService(const edm::ParameterSet &iPS, edm::ActivityRegistry &iRegistry) -{ - XrdSiteStatisticsInformation::createInstance(); - - if (iPS.getUntrackedParameter("reportToFJR", false)) - { - iRegistry.watchPostEndJob(this, &XrdStatisticsService::postEndJob); - } + if (iPS.getUntrackedParameter("reportToFJR", false)) { + iRegistry.watchPostEndJob(this, &XrdStatisticsService::postEndJob); + } } - -void XrdStatisticsService::postEndJob() -{ - edm::Service reportSvc; - if (!reportSvc.isAvailable()) {return;} - - XrdSiteStatisticsInformation *instance = XrdSiteStatisticsInformation::getInstance(); - if (!instance) {return;} - - std::map props; - for (auto& stats : instance->m_sites) - { - stats->recomputeProperties(props); - reportSvc->reportPerformanceForModule(stats->site(), "XrdSiteStatistics", props); - } +void XrdStatisticsService::postEndJob() { + edm::Service reportSvc; + if (!reportSvc.isAvailable()) { + return; + } + + XrdSiteStatisticsInformation *instance = XrdSiteStatisticsInformation::getInstance(); + if (!instance) { + return; + } + + std::map props; + for (auto &stats : instance->m_sites) { + stats->recomputeProperties(props); + reportSvc->reportPerformanceForModule(stats->site(), "XrdSiteStatistics", props); + } } -std::vector> -XrdStatisticsService::condorUpdate() -{ - std::vector> result; - XrdSiteStatisticsInformation *instance = XrdSiteStatisticsInformation::getInstance(); - if (!instance) {return result;} - - std::lock_guard lock(instance->m_mutex); - result.reserve(instance->m_sites.size()); - for (auto& stats : instance->m_sites) - { - CondorIOStats cs; - std::shared_ptr ss = get_underlying_safe(stats); - if (!ss) continue; - cs.bytesRead = ss->getTotalBytes(); - cs.transferTime = ss->getTotalReadTime(); - result.emplace_back(ss->site(), cs); - } +std::vector> XrdStatisticsService::condorUpdate() { + std::vector> result; + XrdSiteStatisticsInformation *instance = XrdSiteStatisticsInformation::getInstance(); + if (!instance) { return result; + } + + std::lock_guard lock(instance->m_mutex); + result.reserve(instance->m_sites.size()); + for (auto &stats : instance->m_sites) { + CondorIOStats cs; + std::shared_ptr ss = get_underlying_safe(stats); + if (!ss) + continue; + cs.bytesRead = ss->getTotalBytes(); + cs.transferTime = ss->getTotalReadTime(); + result.emplace_back(ss->site(), cs); + } + return result; } - -std::shared_ptr -XrdSiteStatisticsInformation::getStatisticsForSite(std::string const &site) -{ - std::lock_guard lock(m_mutex); - for (auto& stats : m_sites) - { - if (stats->site() == site) {return get_underlying_safe(stats);} +std::shared_ptr XrdSiteStatisticsInformation::getStatisticsForSite(std::string const &site) { + std::lock_guard lock(m_mutex); + for (auto &stats : m_sites) { + if (stats->site() == site) { + return get_underlying_safe(stats); } - m_sites.emplace_back(new XrdSiteStatistics(site)); - return get_underlying_safe(m_sites.back()); + } + m_sites.emplace_back(new XrdSiteStatistics(site)); + return get_underlying_safe(m_sites.back()); } - -void -XrdSiteStatisticsInformation::createInstance() -{ - if (!m_instance) - { - std::unique_ptr tmp { new XrdSiteStatisticsInformation() }; - XrdSiteStatisticsInformation* expected = nullptr; - if (m_instance.compare_exchange_strong(expected,tmp.get())) - { - tmp.release(); - } +void XrdSiteStatisticsInformation::createInstance() { + if (!m_instance) { + std::unique_ptr tmp{new XrdSiteStatisticsInformation()}; + XrdSiteStatisticsInformation *expected = nullptr; + if (m_instance.compare_exchange_strong(expected, tmp.get())) { + tmp.release(); } + } } -XrdSiteStatisticsInformation * -XrdSiteStatisticsInformation::getInstance() -{ - return m_instance.load(std::memory_order_relaxed); +XrdSiteStatisticsInformation *XrdSiteStatisticsInformation::getInstance() { + return m_instance.load(std::memory_order_relaxed); } -void XrdStatisticsService::fillDescriptions(edm::ConfigurationDescriptions &descriptions) -{ - edm::ParameterSetDescription desc; - desc.setComment("Report Xrootd-related statistics centrally."); - desc.addUntracked("reportToFJR", true) - ->setComment("True: Add per-site Xrootd statistics to the framework job report.\n" - "False: Collect no site-specific statistics.\n"); - descriptions.add("XrdAdaptor::XrdStatisticsService", desc); +void XrdStatisticsService::fillDescriptions(edm::ConfigurationDescriptions &descriptions) { + edm::ParameterSetDescription desc; + desc.setComment("Report Xrootd-related statistics centrally."); + desc.addUntracked("reportToFJR", true) + ->setComment( + "True: Add per-site Xrootd statistics to the framework job report.\n" + "False: Collect no site-specific statistics.\n"); + descriptions.add("XrdAdaptor::XrdStatisticsService", desc); } - -XrdSiteStatistics::XrdSiteStatistics(std::string const &site) : - m_site(site), - m_readvCount(0), - m_chunkCount(0), - m_readvSize(0), - m_readvNS(0.0), - m_readCount(0), - m_readSize(0), - m_readNS(0) -{ +XrdSiteStatistics::XrdSiteStatistics(std::string const &site) + : m_site(site), + m_readvCount(0), + m_chunkCount(0), + m_readvSize(0), + m_readvNS(0.0), + m_readCount(0), + m_readSize(0), + m_readNS(0) {} + +std::shared_ptr XrdSiteStatistics::startRead(std::shared_ptr parent, + std::shared_ptr req) { + std::shared_ptr readStats(new XrdReadStatistics(parent, req->getSize(), req->getCount())); + return readStats; } -std::shared_ptr -XrdSiteStatistics::startRead(std::shared_ptr parent, std::shared_ptr req) -{ - std::shared_ptr readStats(new XrdReadStatistics(parent, req->getSize(), req->getCount())); - return readStats; +static std::string i2str(int input) { + std::ostringstream formatter; + formatter << input; + return formatter.str(); } - -static std::string -i2str(int input) -{ - std::ostringstream formatter; - formatter << input; - return formatter.str(); +static std::string d2str(double input) { + std::ostringstream formatter; + formatter << std::setw(4) << input; + return formatter.str(); } +void XrdSiteStatistics::recomputeProperties(std::map &props) { + props.clear(); -static std::string -d2str(double input) -{ - std::ostringstream formatter; - formatter << std::setw(4) << input; - return formatter.str(); -} - - -void -XrdSiteStatistics::recomputeProperties(std::map &props) -{ - props.clear(); + props["readv-numOperations"] = i2str(m_readvCount); + props["readv-numChunks"] = i2str(m_chunkCount); + props["readv-totalMegabytes"] = d2str(static_cast(m_readvSize) / (1024.0 * 1024.0)); + props["readv-totalMsecs"] = d2str(m_readvNS / 1e6); - props["readv-numOperations"] = i2str(m_readvCount); - props["readv-numChunks"] = i2str(m_chunkCount); - props["readv-totalMegabytes"] = d2str(static_cast(m_readvSize)/(1024.0*1024.0)); - props["readv-totalMsecs"] = d2str(m_readvNS/1e6); - - props["read-numOperations"] = i2str(m_readCount); - props["read-totalMegabytes"] = d2str(static_cast(m_readSize)/(1024.0*1024.0)); - props["read-totalMsecs"] = d2str(static_cast(m_readNS)/1e6); + props["read-numOperations"] = i2str(m_readCount); + props["read-totalMegabytes"] = d2str(static_cast(m_readSize) / (1024.0 * 1024.0)); + props["read-totalMsecs"] = d2str(static_cast(m_readNS) / 1e6); } - -void -XrdSiteStatistics::finishRead(XrdReadStatistics const &readStats) -{ - if (readStats.readCount() > 1) - { - m_readvCount ++; - m_chunkCount += readStats.readCount(); - m_readvSize += readStats.size(); - m_readvNS += readStats.elapsedNS(); - } - else - { - m_readCount ++; - m_readSize += readStats.size(); - m_readNS += readStats.elapsedNS(); - } +void XrdSiteStatistics::finishRead(XrdReadStatistics const &readStats) { + if (readStats.readCount() > 1) { + m_readvCount++; + m_chunkCount += readStats.readCount(); + m_readvSize += readStats.size(); + m_readvNS += readStats.elapsedNS(); + } else { + m_readCount++; + m_readSize += readStats.size(); + m_readNS += readStats.elapsedNS(); + } } +XrdReadStatistics::XrdReadStatistics(std::shared_ptr parent, IOSize size, size_t count) + : m_size(size), m_count(count), m_parent(parent), m_start(std::chrono::high_resolution_clock::now()) {} -XrdReadStatistics::XrdReadStatistics(std::shared_ptr parent, IOSize size, size_t count) : - m_size(size), - m_count(count), - m_parent(parent), - m_start(std::chrono::high_resolution_clock::now()) -{ +uint64_t XrdReadStatistics::elapsedNS() const { + std::chrono::time_point end = std::chrono::high_resolution_clock::now(); + return std::chrono::duration_cast(end - m_start).count(); } - - -uint64_t -XrdReadStatistics::elapsedNS() const -{ - std::chrono::time_point end = std::chrono::high_resolution_clock::now(); - return std::chrono::duration_cast(end-m_start).count(); -} - diff --git a/Utilities/XrdAdaptor/src/XrdStatistics.h b/Utilities/XrdAdaptor/src/XrdStatistics.h index 58e1595a3fb03..688fac32b31f3 100644 --- a/Utilities/XrdAdaptor/src/XrdStatistics.h +++ b/Utilities/XrdAdaptor/src/XrdStatistics.h @@ -11,36 +11,30 @@ #include #include -namespace edm -{ - class ParameterSet; - class ActivityRegistry; - class ConfigurationDescriptions; - - namespace service - { - class CondorStatusService; - } -} +namespace edm { + class ParameterSet; + class ActivityRegistry; + class ConfigurationDescriptions; -namespace XrdAdaptor -{ + namespace service { + class CondorStatusService; + } +} // namespace edm -class ClientRequest; -class XrdReadStatistics; -class XrdSiteStatistics; +namespace XrdAdaptor { + class ClientRequest; + class XrdReadStatistics; + class XrdSiteStatistics; -/* NOTE: All member information is kept in the XrdSiteStatisticsInformation singleton, + /* NOTE: All member information is kept in the XrdSiteStatisticsInformation singleton, * _not_ within the service itself. This is because we need to be able to use the * singleton on non-CMSSW-created threads. Services are only available to threads * created by CMSSW. */ -class XrdStatisticsService -{ -public: - - XrdStatisticsService(const edm::ParameterSet& iPS, edm::ActivityRegistry &iRegistry); + class XrdStatisticsService { + public: + XrdStatisticsService(const edm::ParameterSet &iPS, edm::ActivityRegistry &iRegistry); void postEndJob(); @@ -56,48 +50,49 @@ class XrdStatisticsService // as self-identified by the Xrootd host; may not necessarily match up with the // "CMS site name". std::vector> condorUpdate(); -}; + }; -class XrdSiteStatisticsInformation -{ -friend class XrdStatisticsService; + class XrdSiteStatisticsInformation { + friend class XrdStatisticsService; -public: + public: static XrdSiteStatisticsInformation *getInstance(); std::shared_ptr getStatisticsForSite(std::string const &site); -private: + private: static void createInstance(); - static std::atomic m_instance; + static std::atomic m_instance; std::mutex m_mutex; std::vector>> m_sites; -}; + }; -class XrdSiteStatistics -{ -friend class XrdReadStatistics; + class XrdSiteStatistics { + friend class XrdReadStatistics; -public: + public: XrdSiteStatistics(std::string const &site); - XrdSiteStatistics(const XrdSiteStatistics&) = delete; - XrdSiteStatistics &operator=(const XrdSiteStatistics&) = delete; + XrdSiteStatistics(const XrdSiteStatistics &) = delete; + XrdSiteStatistics &operator=(const XrdSiteStatistics &) = delete; - std::string const &site() const {return m_site;} + std::string const &site() const { return m_site; } // Note that, while this function is thread-safe, the numbers are only consistent if no other // thread is reading data. void recomputeProperties(std::map &props); - static std::shared_ptr startRead(std::shared_ptr parent, std::shared_ptr req); + static std::shared_ptr startRead(std::shared_ptr parent, + std::shared_ptr req); void finishRead(XrdReadStatistics const &); - uint64_t getTotalBytes() const {return m_readvSize + m_readSize;} - std::chrono::nanoseconds getTotalReadTime() {return std::chrono::nanoseconds(m_readvNS) + std::chrono::nanoseconds(m_readNS);} + uint64_t getTotalBytes() const { return m_readvSize + m_readSize; } + std::chrono::nanoseconds getTotalReadTime() { + return std::chrono::nanoseconds(m_readvNS) + std::chrono::nanoseconds(m_readNS); + } -private: + private: const std::string m_site = "Unknown"; std::atomic m_readvCount; @@ -107,30 +102,29 @@ friend class XrdReadStatistics; std::atomic m_readCount; std::atomic m_readSize; std::atomic m_readNS; -}; + }; -class XrdReadStatistics -{ -friend class XrdSiteStatistics; + class XrdReadStatistics { + friend class XrdSiteStatistics; -public: - ~XrdReadStatistics() {m_parent->finishRead(*this);} - XrdReadStatistics(const XrdReadStatistics&) = delete; - XrdReadStatistics &operator=(const XrdReadStatistics&) = delete; + public: + ~XrdReadStatistics() { m_parent->finishRead(*this); } + XrdReadStatistics(const XrdReadStatistics &) = delete; + XrdReadStatistics &operator=(const XrdReadStatistics &) = delete; -private: + private: XrdReadStatistics(std::shared_ptr parent, IOSize size, size_t count); uint64_t elapsedNS() const; - int readCount() const {return m_count;} - int size() const {return m_size;} + int readCount() const { return m_count; } + int size() const { return m_size; } size_t m_size; IOSize m_count; edm::propagate_const> m_parent; std::chrono::time_point m_start; -}; + }; -} +} // namespace XrdAdaptor #endif