Merge pull request cms-sw#72 from MT2Analysis/bruno_747_jet30

crab fixes to run on data

CMGTools/RootTools/python/samples/samples_13TeV_DATA2015.py

@@ -11,7 +11,8 @@
 dataDir = "$CMSSW_BASE/src/CMGTools/TTHAnalysis/data"  # use environmental variable, useful for instance to run on CRAB
 #json=dataDir+'/json/Cert_246908-248005_13TeV_PromptReco_Collisions15_ZeroTesla_JSON.txt'
 #lumi: delivered= 4.430 (/nb) recorded= 4.013 (/nb)
-json=dataDir+'/json/DCSONLY_Run2015B.json'
+json=dataDir+'/json/json_DCSONLY_Run2015B.txt'
+#json=dataDir+'/json/Cert_246908-251252_13TeV_PromptReco_Collisions15_JSON.txt' # golden json 5.6/pb
 
 #jetHT_0T = cfg.DataComponent(
 #    name = 'jetHT_0T',

CMGTools/TTHAnalysis/cfg/crab/heppy_crab_config.py

@@ -14,7 +14,7 @@
 os.system("tar czf cmgdataset.tar.gz --directory $HOME .cmgdataset")
 os.system("tar czf cafpython.tar.gz --directory /afs/cern.ch/cms/caf/ python")
 config.JobType.inputFiles = ['FrameworkJobReport.xml','heppy_config.py','heppy_crab_script.py','cmgdataset.tar.gz', 'python.tar.gz', 'cafpython.tar.gz']
-config.JobType.outputFiles = ['output.log.tgz'] # mt2.root is automatically send because of the pset file
+config.JobType.outputFiles = ['RLTInfo.root', 'output.log.tgz'] # mt2.root is automatically send because of the pset file
 
 config.section_("Data")
 config.Data.inputDBS = 'global'
@@ -24,7 +24,7 @@
 
 config.section_("Site")
 #config.Site.whitelist = ["T2_CH_CSCS"]
-config.Site.whitelist = ["T2_CH_CSCS"] #, "T2_IT_Legnaro", "T2_UK_London_IC", "T2_UK_SGrid_Bristol", "T2_DE_DESY", "T2_ES_CIEMAT", "T2_IT_Rome", "T2_AT_Vienna","T2_DE_RWTH","T2_FR_GRIF_IRFU", "T2_HU_Budapest", "T2_FR_IPHC", "T2_BE_IIHE", "T2_IT_Pisa", "T2_ES_IFCA", "T2_UK_London_Brunel", "T2_US_Purdue", "T2_UA_KIPT", "T2_US_MIT", "T2_US_Wisconsin", "T2_US_UCSD", "T2_US_Vanderbilt", "T2_US_Caltech"]
+config.Site.whitelist = ["T2_CH_CSCS", "T2_IT_Legnaro", "T2_UK_London_IC", "T2_UK_SGrid_Bristol", "T2_DE_DESY", "T2_ES_CIEMAT", "T2_IT_Rome", "T2_AT_Vienna","T2_DE_RWTH","T2_FR_GRIF_IRFU", "T2_HU_Budapest", "T2_FR_IPHC", "T2_BE_IIHE", "T2_IT_Pisa", "T2_ES_IFCA", "T2_UK_London_Brunel", "T2_US_Purdue", "T2_UA_KIPT", "T2_US_MIT", "T2_US_Wisconsin", "T2_US_UCSD", "T2_US_Vanderbilt", "T2_US_Caltech"]
 config.Site.storageSite = "T3_CH_PSI"
 #config.Data.ignoreLocality = True
 

CMGTools/TTHAnalysis/cfg/crab/heppy_crab_config_env.py

@@ -14,6 +14,8 @@
 cmg_version = os.environ["CMG_VERSION"]
 debug  = os.environ["DEBUG"] == 'True'
 useAAA = os.environ["USEAAA"] == 'True'
+if "JSON" in os.environ:
+    JSON = os.environ["JSON"]
 
 if debug:
     NJOBS = 4
@@ -45,3 +47,10 @@
     pass
 else:
     config.JobType.scriptArgs += ["nevents="+str(NEVENTS)]
+
+try: 
+    JSON
+except NameError:
+    pass
+else:
+    config.JobType.scriptArgs += ["json="+JSON]

CMGTools/TTHAnalysis/cfg/crab/heppy_crab_script.py

@@ -13,6 +13,13 @@
 nevents = None # this means run all events
 nprint  = 0 # quiet printout, change if you want to print the first nprint events
 useAAA = True # use xrootd by default
+json = None
+
+def XrootdRedirector():
+    americas     = ["CO", "MX","US"]
+    oldcontinent = ["AT", "BE", "CH", "DE", "EE", "ES", "FR", "GR", "HU", "IT", "RU", "UK"]
+    region = os.environ["GLIDEIN_CMSSite"].split("_")[1] if "GLIDEIN_CMSSite" in os.environ else ""
+    return  "xrootd-cms.infn.it/" if region in oldcontinent else "cmsxrootd.fnal.gov/" if region in americas else "cms-xrd-global.cern.ch/" 
 
 # arguments of scriptExe
 print "ARGV:",sys.argv
@@ -31,6 +38,9 @@
     elif arg.split("=")[0] == "useAAA":
         useAAA = not (arg.split("=")[1] == 'False') # 'True' by default
         if useAAA: print "chosen to run via xrootd"
+    elif arg.split("=")[0] == "json":
+        json = arg.split("=")[1]
+        print "run on json:", json
 
 print "dataset:", dataset
 print "job", job , " out of", total
@@ -47,10 +57,12 @@
 selectedComponents = []
 for comp in config.components:
     if comp.name == dataset:
+        if comp.isData and json != None:
+            comp.json = json
         # this selects the files and events and changes the name to _ChunkX according to fineSplitFactor and splitFactor
         newComp = split([comp])[job-1] # first job number is 1
         if useAAA:
-            newComp.files = [x.replace("root://eoscms.cern.ch//eos/cms","root://cms-xrd-global.cern.ch/") for x in newComp.files]
+            newComp.files = [x.replace("root://eoscms.cern.ch//eos/cms","root://" + XrootdRedirector()) for x in newComp.files]
         selectedComponents.append(newComp)
 
 # check selectedComponents
@@ -80,6 +92,7 @@
 
 # assign the right name
 os.rename("Output/mt2.root", "mt2.root")
+os.rename("Output/RLTInfo.root", "RLTInfo.root")
 
 # print in crab log file the content of the job log files, so one can see it from 'crab getlog'
 print "-"*25

CMGTools/TTHAnalysis/cfg/crab/launchall.py

@@ -3,11 +3,21 @@
 # datasets to run as defined from run_susyMT2.cfg
 # number of jobs to run per dataset decided based on splitFactor and fineSplitFactor from cfg file
 # in principle one only needs to modify the following two lines:
-production_label = "prod74Xdata_noJSON"
-cmg_version = 'MT2_CMGTools-from-CMSSW_7_4_3'
+production_label = "prod747data_test"
+cmg_version = 'MT2_CMGTools-from-CMSSW_7_4_7'
+
+debug  = True
+useAAA = True
+
+JSON = "$CMSSW_BASE/src/CMGTools/TTHAnalysis/data/json/json_DCSONLY_Run2015B.txt"
+#JSON = "$CMSSW_BASE/src/CMGTools/TTHAnalysis/data/json/Cert_246908-251252_13TeV_PromptReco_Collisions15_JSON.txt"
+
+#recreate cached datasets for data that keep evolving (remove)
+os.system("rm ~/.cmgdataset/CMS*PromptReco*")
+
+# update most recent DCS-only json
+os.system("cp -f /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions15/13TeV/DCSOnly/json_DCSONLY_Run2015B.txt ../../data/json/")
 
-debug  = False
-useAAA = False
 
 handle = open("heppy_config.py", 'r')
 cfo = imp.load_source("heppy_config", "heppy_config.py", handle)
@@ -28,6 +38,8 @@
     NJOBS = len(split([comp]))
     os.environ["NJOBS"] = str(NJOBS)
     os.environ["DATASET"] = str(comp.name)
+    if comp.isData and 'JSON' in vars():
+        os.environ["JSON"] = JSON
     os.system("crab submit -c heppy_crab_config_env.py")
 
 os.system("rm -f python.tar.gz")

CMGTools/TTHAnalysis/data/json/Cert_246908-251252_13TeV_PromptReco_Collisions15_JSON.txt

@@ -0,0 +1 @@
+{"251244": [[85, 86], [88, 93], [96, 121], [123, 156], [158, 428], [430, 442]], "251251": [[1, 31], [33, 97], [99, 167]], "251252": [[1, 283], [285, 505], [507, 554]]}