Package madgraph :: Package interface :: Module madweight_interface
[hide private]
[frames] | no frames]

Source Code for Module madgraph.interface.madweight_interface

   1  ################################################################################ 
   2  # 
   3  # Copyright (c) 2011 The MadGraph Development team and Contributors 
   4  # 
   5  # This file is a part of the MadGraph 5 project, an application which  
   6  # automatically generates Feynman diagrams and matrix elements for arbitrary 
   7  # high-energy processes in the Standard Model and beyond. 
   8  # 
   9  # It is subject to the MadGraph license which should accompany this  
  10  # distribution. 
  11  # 
  12  # For more information, please visit: http://madgraph.phys.ucl.ac.be 
  13  # 
  14  ################################################################################ 
  15  """ 
  16  A user friendly interface to access all the function associated to MadWeight  
  17  """ 
  18   
  19  from __future__ import absolute_import 
  20  import logging 
  21  import os 
  22  import subprocess 
  23  import time 
  24  import glob 
  25  import math 
  26  import xml.sax.handler 
  27  import shutil 
  28  from six import StringIO 
  29  from six.moves import map 
  30  from six.moves import range 
  31   
  32  if __name__ == '__main__': 
  33      import sys 
  34      sys.path.append('/Users/omatt/Documents/eclipse/madweight/') 
  35   
  36  logger = logging.getLogger('cmdprint') 
  37   
  38  pjoin = os.path.join 
  39  try: 
  40      from madgraph import InvalidCmd, MadGraph5Error, MG5DIR 
  41      import madgraph.interface.extended_cmd as cmd 
  42      import madgraph.interface.common_run_interface as common_run 
  43       
  44       
  45      import madgraph.madweight.MW_info as MW_info 
  46      import madgraph.madweight.change_tf as change_tf 
  47      import madgraph.madweight.create_param as create_param 
  48      import madgraph.madweight.create_run as create_run 
  49      import madgraph.madweight.Cards as Cards 
  50      import madgraph.madweight.write_MadWeight as write_MadWeight 
  51      import madgraph.madweight.verif_event as verif_event 
  52      import madgraph.madweight.MW_driver as MW_driver 
  53       
  54      import madgraph.various.misc as misc 
  55      import madgraph.various.banner as banner 
  56      import madgraph.iolibs.files as files 
  57      MADEVENT = False 
  58  except ImportError as error: 
  59      raise  
  60      logger.debug(error) 
  61      from internal import InvalidCmd, MadGraph5Error 
  62      import internal.extended_cmd as cmd 
  63      import internal.common_run_interface as common_run 
  64      import internal.madweight.MW_info as MW_info 
  65      import internal.madweight.change_tf as change_tf 
  66      import internal.madweight.create_param as create_param 
  67      import internal.madweight.create_run as create_run 
  68      import internal.madweight.Cards as Cards 
  69      import internal.madweight.write_MadWeight as write_MadWeight 
  70      import internal.madweight.verif_event as verif_event 
  71      import internal.madweight.MW_driver as MW_driver 
  72       
  73       
  74      import internal.misc as misc  
  75      import internal.banner as banner 
  76      import internal.files as files 
  77      MADEVENT = True 
  78   
  79   
  80  AlreadyRunning = common_run.AlreadyRunning 
  81   
  82  #=============================================================================== 
  83  # CmdExtended 
  84  #=============================================================================== 
85 -class CmdExtended(cmd.Cmd):
86 """Particularisation of the cmd command for MadEvent""" 87 88 #suggested list of command 89 next_possibility = { 90 'start': [], 91 } 92 93 debug_output = 'MW5_debug' 94 error_debug = 'Please report this bug on https://bugs.launchpad.net/mg5amcnlo\n' 95 error_debug += 'with MadWeight in the title of the bug report.' 96 error_debug += 'More information is found in \'%(debug)s\'.\n' 97 error_debug += 'Please attach this file to your report.' 98 99 config_debug = 'If you need help with this issue please contact us on https://answers.launchpad.net/mg5amcnlo\n' 100 101 102 keyboard_stop_msg = """stopping all operation 103 in order to quit madweight please enter exit""" 104 105 # Define the Error 106 InvalidCmd = InvalidCmd 107 ConfigurationError = MadGraph5Error 108
109 - def __init__(self, *arg, **opt):
110 """Init history and line continuation""" 111 112 # Tag allowing/forbiding question 113 self.force = False 114 115 # If possible, build an info line with current version number 116 # and date, from the VERSION text file 117 info = misc.get_pkg_info() 118 info_line = "" 119 if info and 'version' in info and 'date' in info: 120 len_version = len(info['version']) 121 len_date = len(info['date']) 122 if len_version + len_date < 30: 123 info_line = "#* VERSION %s %s %s *\n" % \ 124 (info['version'], 125 (30 - len_version - len_date) * ' ', 126 info['date']) 127 else: 128 root_path = pjoin(os.path.dirname(__file__), os.path.pardir,os.path.pardir) 129 version = open(pjoin(root_path,'MGMEVersion.txt')).readline().strip() 130 info_line = "#* VERSION %s %s *\n" % \ 131 (version, (24 - len(version)) * ' ') 132 133 # Create a header for the history file. 134 # Remember to fill in time at writeout time! 135 self.history_header = \ 136 '#************************************************************\n' + \ 137 '#* MadWeight 5 *\n' + \ 138 '#* *\n' + \ 139 "#* * * *\n" + \ 140 "#* * * * * *\n" + \ 141 "#* * * * * 5 * * * * *\n" + \ 142 "#* * * * * *\n" + \ 143 "#* * * *\n" + \ 144 "#* *\n" + \ 145 "#* *\n" + \ 146 info_line + \ 147 "#* *\n" + \ 148 "#* The MadGraph Development Team - Please visit us at *\n" + \ 149 "#* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \ 150 '#* *\n' + \ 151 '#************************************************************\n' + \ 152 '#* *\n' + \ 153 '#* Command File for MadWeight *\n' + \ 154 '#* *\n' + \ 155 '#* run as ./bin/madweight.py FILENAME *\n' + \ 156 '#* *\n' + \ 157 '#************************************************************\n' 158 159 if info_line: 160 info_line = info_line[1:] 161 162 logger.info(\ 163 "************************************************************\n" + \ 164 "* *\n" + \ 165 "* W E L C O M E to M A D G R A P H 5 *\n" + \ 166 "* M A D W E I G H T *\n" + \ 167 "* *\n" + \ 168 "* * * *\n" + \ 169 "* * * * * *\n" + \ 170 "* * * * * 5 * * * * *\n" + \ 171 "* * * * * *\n" + \ 172 "* * * *\n" + \ 173 "* *\n" + \ 174 info_line + \ 175 "* *\n" + \ 176 "* The MadGraph Development Team - Please visit us at *\n" + \ 177 "* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \ 178 "* *\n" + \ 179 "* Type 'help' for in-line help. *\n" + \ 180 "* *\n" + \ 181 "************************************************************") 182 183 cmd.Cmd.__init__(self, *arg, **opt)
184
185 -class HelpToCmd(object):
186
187 - def help_collect(self):
188 logger.info('collect [-refine]') 189 logger.info(' combine the results of the jobs launched on the cluster.') 190 logger.info(' This creates three type of output files:') 191 logger.info(' - weights.out [weight for event/card]') 192 logger.info(' - unnormalize-likelihood.out [-\sum ln(Weight)]') 193 logger.info(' - output.xml [additional information]') 194 logger.info('') 195 logger.info(' The option \'-refine\' is to be added if this is not the first') 196 logger.info(' cluster submission. Otherwise previous run submission will be lost.')
197
198 - def help_define_transfer_fct(self):
199 """help for define transfer_fct""" 200 logger.info(' Modify the current transfer functions') 201 logger.info(' If no argument provided a question showing the list of possibilities.') 202 logger.info(' will be ask. If the TF is provided as argument, no question is asked.')
203 204
205 -class CompleteForCmd(object):
206 207
208 - def complete_collect(self,text, line, begidx, endidx):
209 """ complete the collect command""" 210 args = self.split_arg(line[0:begidx]) 211 212 return self.list_completion(text,['-refine','-f','--refine'], line)
213
214 - def complete_define_transfer_fct(self, text, line, begidx, endidx):
215 """ complete the define_transfer_fct """ 216 217 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data') 218 listdir=os.listdir(path) 219 args = self.split_arg(line[0:begidx]) 220 if len(args) == 1: 221 222 possibilities = [content[3:-4] for content in listdir \ 223 if (content.startswith('TF') and content.endswith('dat'))] 224 return self.list_completion(text, possibilities, line)
225 226 #=============================================================================== 227 # MadWeightCmd 228 #===============================================================================
229 -class MadWeightCmd(CmdExtended, HelpToCmd, CompleteForCmd, common_run.CommonRunCmd):
230 231 _set_options = [] 232 prompt = 'MadWeight5>' 233 helporder = ['MadWeight Function', 'Documented commands', 'Advanced commands'] 234
235 - def remove_fct(self):
236 """Not in help: remove fct""" 237 return None
238 239 do_decay_events = remove_fct 240 do_delphes = remove_fct 241 do_pgs = remove_fct 242 _True = [1,True,'1','T','t','.true.','True'] 243 244 ############################################################################
245 - def __init__(self, me_dir = None, options={}, *completekey, **stdin):
246 """ add information to the cmd """ 247 248 CmdExtended.__init__(self, *completekey, **stdin) 249 common_run.CommonRunCmd.__init__(self, me_dir, options) 250 self.configured = 0 # time at which the last option configuration occur
251
252 - def do_quit(self, *args, **opts):
253 common_run.CommonRunCmd.do_quit(self, *args, **opts) 254 CmdExtended.do_quit(self, *args, **opts) 255 return True
256
257 - def configure(self):
258 os.chdir(pjoin(self.me_dir)) 259 self.__CMD__initpos = self.me_dir 260 261 time_mod = max([os.path.getctime(pjoin(self.me_dir,'Cards','run_card.dat')), 262 os.path.getctime(pjoin(self.me_dir,'Cards','MadWeight_card.dat'))]) 263 264 265 if self.configured > time_mod and \ 266 hasattr(self,'MWparam') and hasattr(self,'run_card'): 267 return 268 self.configured = time_mod 269 270 271 272 self.MWparam = MW_info.MW_info(pjoin(self.me_dir,'Cards','MadWeight_card.dat')) 273 run_card = pjoin(self.me_dir, 'Cards','run_card.dat') 274 self.run_card = banner.RunCard(run_card) 275 276 if self.options['run_mode'] == 0: 277 self.exec_cmd('set run_mode 2 --no_save') 278 self.exec_cmd('set nb_core 1 --no_save') 279 if not self.options['cluster_temp_path']: 280 if self.options['run_mode'] == 2: 281 logger.info('Options cluster_temp_path is required for MW run. Trying to run with /tmp', 282 '$MG:BOLD') 283 self.exec_cmd('set cluster_temp_path /tmp --no_save') 284 elif self.options['cluster_type'] != 'condor': 285 raise Exception('cluster_temp_path needs to be define for MW. Please retry.')
286
287 - def do_define_transfer_fct(self, line):
288 """MadWeight Function:Define the current transfer function""" 289 290 with misc.chdir(self.me_dir): 291 self.configure() 292 args = self.split_arg(line) 293 294 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data') 295 listdir=os.listdir(path) 296 question = 'Please choose your transfer_function between\n' 297 possibilities = [content[3:-4] for content in listdir \ 298 if (content.startswith('TF') and content.endswith('dat'))] 299 for i, tfname in enumerate(possibilities): 300 question += ' %s / %s\n' % (i, tfname) 301 possibilities += list(range(len(possibilities))) 302 303 if args and args[0] in possibilities: 304 tfname = args[0] 305 else: 306 tfname = self.ask(question, 'dbl_gauss_pt_jet', possibilities) 307 if tfname.isdigit(): 308 tfname = possibilities[int(tfname)] 309 310 P_dir, MW_dir = MW_info.detect_SubProcess(P_mode=1) 311 os.chdir('./Source/MadWeight/transfer_function') 312 change_tf.create_TF_main(tfname,0, MW_dir)
313 314
315 - def do_treatcards(self, line):
316 """MadWeight Function:create the various param_card // compile input for the run_card""" 317 self.configure() 318 args = self.split_arg(line) 319 320 create_param.Param_card(run_name=self.MWparam) 321 self.MWparam.update_nb_card() 322 Cards.create_include_file(self.MWparam) 323 create_run.update_cuts_status(self.MWparam)
324
325 - def do_get_integration_channel(self, line):
326 """MadWeight Function:analyze the cards/diagram to find an way to integrate efficiently""" 327 self.configure() 328 args = self.split_arg(line) 329 330 write_MadWeight.create_all_fortran_code(self.MWparam)
331
332 - def do_compile(self, line, refine=False):
333 """MadWeight Function:compile the code""" 334 self.configure() 335 336 misc.compile(arg=["../lib/libtools.a"], cwd=pjoin(self.me_dir,'Source')) 337 misc.compile(arg=["../lib/libblocks.a"], cwd=pjoin(self.me_dir,'Source')) 338 misc.compile(arg=["../lib/libTF.a"], cwd=pjoin(self.me_dir,'Source')) 339 misc.compile(arg=["../lib/libpdf.a"], cwd=pjoin(self.me_dir,'Source')) 340 misc.compile(arg=["../lib/libdhelas.a"], cwd=pjoin(self.me_dir,'Source')) 341 misc.compile(arg=["../lib/libmodel.a"], cwd=pjoin(self.me_dir,'Source')) 342 misc.compile(arg=["../lib/libgeneric.a"], cwd=pjoin(self.me_dir,'Source')) 343 misc.compile(arg=["../lib/libcernlib.a"], cwd=pjoin(self.me_dir,'Source')) 344 345 # 346 # here check validity of some parameters 347 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['montecarlo_perm']=='t': 348 raise Exception('Cannot use mint if monte carlo over permutations') 349 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['use_sobol']=='t': 350 raise Exception('sobol generator with mint not implemented') 351 352 353 354 for MW_dir in self.MWparam.MW_listdir: 355 logger.info('compile %s' %MW_dir) 356 pdir = pjoin(self.me_dir,'SubProcesses',MW_dir) 357 if refine and os.path.exists(pjoin(pdir, 'initialization.o')): 358 os.remove(pjoin(pdir, 'initialization.o')) 359 if refine and os.path.exists(pjoin(pdir, 'comp_madweight')): 360 os.remove(pjoin(pdir, 'comp_madweight')) 361 misc.compile(cwd=pdir) 362 if not os.path.exists(pjoin(pdir, 'comp_madweight')): 363 raise Exception('compilation fails') 364 logger.info('MadWeight code has been compiled.')
365 366
367 - def do_check_events(self, line):
368 """MadWeight Function: check that the events are valid 369 and write the events to MG mapping""" 370 self.configure() 371 evt_file = pjoin(self.me_dir,'Events','input.lhco') 372 if not os.path.exists(evt_file): 373 question = 'Which LHCO file do you want to use?' 374 default = '' 375 if os.path.exists('%s.gz' % evt_file): 376 input_file = '%s.gz' % evt_file 377 else: 378 input_file = self.ask(question, default, path_msg='valid path') 379 380 if not input_file: 381 raise self.InvalidCmd('Please specify a valid LHCO File') 382 383 if input_file.endswith('.gz'): 384 misc.gunzip(input_file, keep=True, stdout=evt_file) 385 else: 386 files.cp(input_file, evt_file) 387 388 verif_event.verif_event(self.MWparam)
389 390 391 392 393 394
395 - def check_launch_jobs(self, args):
396 """format the argument to retrun a list with two argument, 397 The first corresponding to the fact if we need te create the output dir 398 The second if we can launch the job on the cluster.""" 399 400 if not args: 401 #use default 402 args[:] = [True, True] 403 return 404 else: 405 create_dir = True 406 launch = True 407 for arg in args: 408 if arg.count('=') !=1 : 409 logger.warning('command launch_jobs does not recognized argument %s. This argument is ignored' % arg) 410 restrict, value = arg.split('=') 411 if restrict == '--create_dir=': 412 if value in self._True: 413 create_dir = True 414 else: 415 create_dir = False 416 elif restrict == '--submit=': 417 if value in self._True: 418 launch = True 419 else: 420 launch = False 421 args[:] = [create_dir, launch] 422 return
423
424 - def do_submit_jobs(self, line):
425 """MadWeight Function:Submitting the jobs to the cluster""" 426 427 428 self.configure() 429 self.clean_old_run(keep_event=True) 430 args = self.split_arg(line) 431 self.check_launch_jobs(args) 432 # now args is of the type [True True] 433 create_dir, launch_jobs = args[0], args[1] 434 435 for nb_card in self.MWparam.actif_param: 436 for dirname in self.MWparam.MW_listdir: 437 nb_job = self.MWparam.nb_event_MW[dirname] 438 if self.MWparam['mw_run']['nb_event_by_node'] > 1: 439 nb_job = 1+ (nb_job-1) // self.MWparam['mw_run']['nb_event_by_node'] 440 441 for event_sample in range(nb_job): 442 self.submit_job(dirname, nb_card, event_sample) 443 444 starttime = time.time() 445 #logger.info(' Waiting for submitted jobs to complete') 446 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'), 447 starttime=starttime, level='madweight', update_results=False) 448 449 try: 450 self.cluster.wait(self.me_dir, update_status) 451 except Exception: 452 self.cluster.remove() 453 raise 454 except KeyboardInterrupt: 455 if not self.force: 456 ans = self.ask('Error detected. Do you want to clean the queue?', 457 default = 'y', choices=['y','n']) 458 else: 459 ans = 'y' 460 if ans == 'y': 461 self.cluster.remove() 462 raise
463
464 - def submit_job(self, dirname, nb_card, sample_nb, evt_file=None, restrict_evt=[]):
465 """launch on the cluster the job which creates the computation""" 466 467 input_files = [pjoin(self.me_dir, 'SubProcesses', dirname, 'comp_madweight'), 468 pjoin(self.me_dir, 'Cards', 'param_card_%i.dat' % nb_card), 469 self.get_pdf_input_filename(), 470 pjoin(self.me_dir, 'Cards', 'ident_card.dat'), 471 pjoin(self.me_dir, 'Cards', 'run_card.dat') 472 ] 473 474 # add event_file: 475 if not evt_file: 476 evt_file = (sample_nb // self.MWparam['mw_run']['event_packing']) 477 evt = 'verif_%i.lhco' % evt_file 478 first_event = (sample_nb % self.MWparam['mw_run']['event_packing']) * self.MWparam['mw_run']['nb_event_by_node'] 479 name = self.MWparam.name 480 input_files.append(pjoin(self.me_dir, 'SubProcesses', dirname, name, evt)) 481 482 if restrict_evt: 483 restrict_path = pjoin(self.me_dir, 'SubProcesses', dirname, name, 484 'restrict%i_%i.dat' % (nb_card,evt_file)) 485 input_files.append(restrict_path) 486 #open(restrict_path, 'w').write(' '.join(map(str, restrict_evt))) 487 488 # Need to add PDF (maybe also symfact, ...) ? 489 490 output_file = ['output_%s_%s.xml' % (nb_card, sample_nb)] 491 exe = pjoin(self.me_dir, 'bin', 'internal', 'madweight', 'MW_driver.py') 492 493 # expected args: card_nb, first_event, nb_event, evt, mw_int_points, log_level 494 args = [str(nb_card), str(first_event), 495 str(self.MWparam['mw_run']['nb_event_by_node']) ,evt, 496 str(self.MWparam['mw_run']['mw_int_points']), 497 self.MWparam['mw_run']['log_level'], str(sample_nb)] 498 cwd = pjoin(self.me_dir, 'SubProcesses', dirname, name) 499 # Ensure that the code is working ONLY if TEMP_CLUSTER_PATH is define 500 if self.options['run_mode'] == 0: 501 raise Exception('need to check the validity') 502 else: 503 # ensure that this is running with NO central disk !!! 504 if not self.options['cluster_temp_path'] and not self.options['cluster_type'] == 'condor': 505 raise self.ConfigurationError('MadWeight requires temp_cluster_path options to be define') 506 self.cluster.submit2(exe, args, cwd, input_files=input_files, output_files=output_file)
507 508 509
510 - def check_collect(self, args):
511 """ """ 512 513 if len(args) >1: 514 self.help_collect() 515 raise self.InvalidCmd('Invalid Command format') 516 elif len(args) == 1: 517 if args not in ['-refine', '--refine']: 518 args[0] = '-refine' 519 else: 520 self.help_collect() 521 raise self.InvalidCmd('Invalid Command format')
522
523 - def do_collect(self, line):
524 """MadWeight Function: making the collect of the results""" 525 526 self.configure() 527 args = self.split_arg(line) 528 self.check_collect(args) 529 xml_reader = MWParserXML() 530 531 name = self.MWparam.name 532 # 1. Concatanate the file. ############################################# 533 out_dir = pjoin(self.me_dir, 'Events', name) 534 if '-refine' in args: 535 out_path = pjoin(out_dir, 'refine.xml') 536 else: 537 out_path = pjoin(out_dir, 'output.xml') 538 if os.path.exists(out_path): 539 logger.warning('Output file already exists. Current one will be tagged with _old suffix') 540 logger.warning('Run "collect -refine to instead update your current results."') 541 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 542 files.mv(pjoin(out_dir, 'weights.out'), pjoin(out_dir, 'weights.out')) 543 for MWdir in self.MWparam.MW_listdir: 544 out_dir = pjoin(self.me_dir, 'Events', name, MWdir) 545 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 546 out_dir = pjoin(self.me_dir, 'Events', name) 547 548 fsock = open(out_path, 'w') 549 fsock.write('<madweight>\n<banner>\n') 550 # BANNER 551 for card in ['proc_card_mg5.dat','MadWeight_card.dat','transfer_card.dat','param_card.dat','run_card.dat']: 552 cname = card[:-4] 553 fsock.write('<%s>\n' % cname) 554 fsock.write(open(pjoin(self.me_dir,'Cards',card)).read().replace('<','!>')) 555 fsock.write('</%s>\n' % cname) 556 fsock.write('</banner>\n') 557 at_least_one = False 558 for MWdir in self.MWparam.MW_listdir: 559 out_dir = pjoin(self.me_dir, 'Events', name, MWdir) 560 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, name) 561 if not os.path.exists(out_dir): 562 os.mkdir(out_dir) 563 if '-refine' in args: 564 out_path = pjoin(out_dir, 'refine.xml') 565 else: 566 out_path = pjoin(out_dir, 'output.xml') 567 fsock2 = open(out_path,'w') 568 fsock.write('<subprocess id=\'%s\'>\n' % MWdir) 569 fsock2.write('<subprocess id=\'%s\'>\n' % MWdir) 570 for output in misc.glob('output_*_*.xml', input_dir): 571 at_least_one = True 572 text = open(output).read() 573 fsock2.write(text) 574 fsock.write(text) 575 os.remove(output) 576 fsock.write('</subprocess>\n') 577 fsock2.write('</subprocess>\n') 578 fsock2.close() 579 fsock.write('\n</madweight>\n') 580 fsock.close() 581 # 2. Special treatment for refine mode 582 if '-refine' in args: 583 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) 584 for MWdir in self.MWparam.MW_listdir: 585 out_dir = pjoin(self.me_dir, 'Events',name, MWdir) 586 ref_output = xml_reader2.read_file(pjoin(out_dir, 'refine.xml')) 587 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) 588 base_output = xml_reader2.read_file(pjoin(out_dir, 'output.xml')) 589 590 base_output.refine(ref_output) 591 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 592 base_output.write(pjoin(out_dir, 'output.xml'), MWdir) 593 elif not at_least_one: 594 logger.warning("Nothing to collect restore _old file as current.") 595 out_dir = pjoin(self.me_dir, 'Events', name) 596 files.mv(pjoin(out_dir, 'output_old.xml'), pjoin(out_dir, 'output.xml')) 597 files.mv(pjoin(out_dir, 'weights_old.out'), pjoin(out_dir, 'weights.out')) 598 for MWdir in self.MWparam.MW_listdir: 599 out_dir = pjoin(self.me_dir, 'Events', name, MWdir) 600 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 601 602 603 604 # 3. Read the (final) log file for extracting data 605 total = {} 606 likelihood = {} 607 err_likelihood = {} 608 cards = set() 609 events = set() 610 tf_sets = set() 611 for MW_dir in self.MWparam.MW_listdir: 612 out_dir = pjoin(self.me_dir, 'Events', name, MW_dir) 613 xml_reader = MWParserXML() 614 data = xml_reader.read_file(pjoin(out_dir, 'output.xml')) 615 # 616 log_level = self.MWparam['mw_run']['log_level'] 617 generator = ((int(i),j,int(k),data[i][j][k]) for i in data 618 for j in data[i] 619 for k in data[i][j]) 620 for card, event, tf_set, obj in generator: 621 # update the full list of events/cards 622 cards.add(card) 623 events.add(event) 624 tf_sets.add(tf_set) 625 # now compute the associate value, error[square] 626 if (card,event, tf_set) in total: 627 value, error = total[(card, event, tf_set)] 628 else: 629 value, error = 0, 0 630 obj.calculate_total() 631 value, error = (value + obj.value, error + obj.error**2) 632 total[(card, event, tf_set)] = (value, error) 633 if tf_set == 1: 634 if value: 635 if card not in likelihood: 636 likelihood[card], err_likelihood[card] = 0, 0 637 likelihood[card] -= math.log(value) 638 err_likelihood[card] += error / value 639 else: 640 likelihood[card] = float('Inf') 641 err_likelihood[card] = float('nan') 642 643 644 # write the weights file: 645 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'w') 646 logger.info('Write output file with weight information: %s' % fsock.name) 647 fsock.write('# Weight (un-normalize) for each card/event/set of transfer fct\n') 648 fsock.write('# format: LHCO_event_number card_id tf_set_id value integration_error\n') 649 events = list(events) 650 events.sort() 651 cards = list(cards) 652 cards.sort() 653 tf_sets = list(tf_sets) 654 tf_sets.sort() 655 for event in events: 656 for card in cards: 657 for tf_set in tf_sets: 658 try: 659 value, error = total[(card, event,tf_set)] 660 except KeyError: 661 continue 662 error = math.sqrt(error) 663 fsock.write('%s %s %s %s %s \n' % (event.replace('@', ' '), card, tf_set, value, error)) 664 665 # write the likelihood file: 666 fsock = open(pjoin(self.me_dir, 'Events', name, 'un-normalized_likelihood.out'), 'w') 667 fsock.write('# Warning: this Likelihood needs a bin by bin normalization !\n') 668 fsock.write('# IF more than one set of transfer function are define. ONLY the first one is ') 669 fsock.write('# include in this file.') 670 fsock.write('# format: card_id value integration_error\n') 671 for card in cards: 672 value, error = likelihood[card], err_likelihood[card] 673 error = math.sqrt(error) 674 fsock.write('%s %s %s \n' % (card, value, error))
675 676
677 - def do_clean(self, line):
678 """MadWeight Function: syntax: clean [XXX] 679 clean the previous run XXX (the last one by default)""" 680 681 args = self.split_arg(line) 682 self.configure() 683 if len(args) == 0: 684 name = self.MWparam.name 685 else: 686 name = args[0] 687 688 ans = self.ask('Do you want to remove Events/%s directory?', 'y',['y','n']) 689 if ans == 'y': 690 try: 691 shutil.rmtree(pjoin(self.me_dir, 'Events', name)) 692 except Exception as error: 693 logger.warning(error) 694 for Pdir in self.MWparam.MW_listdir: 695 try: 696 shutil.rmtree(pjoin(self.me_dir, 'SubProcesses', Pdir, name)) 697 except Exception as error: 698 logger.warning(error)
699
700 - def ask_edit_cards(self, cards, *arg, **opts):
701 super(MadWeightCmd, self).ask_edit_cards(cards, *arg, **opts) 702 self.configured = 0 703 self.configure()
704
705 - def do_launch(self, line):
706 """MadWeight Function:run the full suite of commands""" 707 708 args = self.split_arg(line) 709 710 #if not os.path.exists(pjoin(self.me_dir, 'Cards','transfer_card.dat')): 711 # self.exec_cmd('define_transfer_fct') 712 713 cards = ['param_card.dat', 'run_card.dat', 'madweight_card.dat', 714 'transfer_card.dat', 'input.lhco'] 715 if not self.force: 716 self.ask_edit_cards(cards, mode='fixed', plot=False) 717 else: 718 self.configured = 0 719 self.configure() 720 with misc.chdir(self.me_dir): 721 if not os.path.exists(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile'])): 722 raise self.InvalidCmd('Please specify a valid LHCO File') 723 if pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']) not in \ 724 [pjoin(self.me_dir, 'Events', 'input.lhco'), pjoin(self.me_dir, 'Events', 'input.lhco.gz')]: 725 zipped = self.MWparam['mw_run']['inputfile'].endswith('.gz') 726 if zipped: 727 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']), 728 pjoin(self.me_dir, 'Events', 'input.lhco.gz')) 729 if os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')): 730 os.remove(pjoin(self.me_dir, 'Events', 'input.lhco')) 731 else: 732 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']), 733 pjoin(self.me_dir, 'Events', 'input.lhco')) 734 735 if not (os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')) or \ 736 os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco.gz'))): 737 raise self.InvalidCmd('Please specify a valid LHCO File') 738 739 self.exec_cmd('treatcards') 740 self.exec_cmd('get_integration_channel') 741 self.exec_cmd('compile') 742 self.exec_cmd('check_events') 743 self.exec_cmd('submit_jobs') 744 self.exec_cmd('collect')
745 746
747 - def check_refine(self, args):
748 """check the argument validity""" 749 750 if len(args) != 1: 751 raise self.InvalidCmd('refine requires a single argument') 752 753 try: 754 args[0] = float(args[0]) 755 except Exception: 756 raise self.InvalidCmd('First argument of refine command should be a number') 757 758 if args[0] < 0 or args[0]>1: 759 raise self.InvalidCmd('The first argument should be a number between 0 and 1.')
760
761 - def clean_old_run(self, keep_event=False):
762 763 for MWdir in self.MWparam.MW_listdir: 764 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, self.MWparam.name) 765 if not os.path.exists(input_dir): 766 continue 767 for filename in os.listdir(input_dir): 768 if keep_event and filename.startswith('verif'): 769 continue 770 os.remove(pjoin(input_dir, filename))
771 772 773
774 - def do_refine(self, line):
775 """MadWeight Function:syntax: refine X 776 relaunch the computation of the weight which have a precision lower than X""" 777 args = self.split_arg(line) 778 self.check_refine(args) 779 self.configure() 780 781 self.clean_old_run(keep_event=True) 782 if not self.force: 783 cards = ['madweight_card.dat'] 784 self.ask_edit_cards(cards, mode='fixed', plot=False) 785 self.exec_cmd("treatcards") 786 self.do_compile('', refine=True) # force re-compilation 787 788 nb_events_by_file = self.MWparam['mw_run']['nb_event_by_node'] * self.MWparam['mw_run']['event_packing'] 789 asked_events = self.MWparam['mw_run']['nb_exp_events'] 790 791 precision = args[0] 792 name = self.MWparam.name 793 allow_refine = [] 794 # events/cards to refine 795 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'r') 796 for line in fsock: 797 if '#' in line: 798 line = line.split('#')[0] 799 line = line.split() 800 if len(line) == 5: 801 lhco_nb, card_nb, tf_set, value, error = line 802 else: 803 continue 804 if tf_set != '1': 805 continue 806 if float(value) * precision < float(error): 807 allow_refine.append((int(card_nb), lhco_nb)) 808 logger.info("%s selected jobs for the refinment." % len(allow_refine)) 809 for MWdir in self.MWparam.MW_listdir: 810 # We need to know in which file are written all the relevant event 811 event_to_file = {} 812 for evt_nb in range(asked_events//nb_events_by_file +1): 813 evt = 'verif_%i.lhco' % evt_nb 814 for line in open(pjoin(self.me_dir, 'SubProcesses', MWdir, name, evt)): 815 split = line.split() 816 if len(split) == 3: 817 event_to_file[split[1]] = evt_nb 818 to_refine = {} 819 out_dir = pjoin(self.me_dir, 'Events',name, MWdir) 820 xml_reader = MWParserXML(keep_level='weight') 821 data = xml_reader.read_file(pjoin(out_dir, 'output.xml')) 822 generator = ((int(i),j,int(k),data[i][j][k]) for i in data 823 for j in data[i] 824 for k in data[i][j]) 825 for card, event, tf_set, obj in generator: 826 if tf_set != 1: 827 continue 828 value, error = obj.value, obj.error 829 if value * precision < error: 830 if card not in to_refine: 831 to_refine[card] = [] 832 if (card,event) in allow_refine: 833 to_refine[card].append(event) 834 if to_refine: 835 self.resubmit(MWdir, to_refine, event_to_file) 836 837 # control 838 starttime = time.time() 839 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'), 840 starttime=starttime, level='madweight', update_results=False) 841 try: 842 self.cluster.wait(self.me_dir, update_status) 843 except Exception: 844 self.cluster.remove() 845 raise 846 except KeyboardInterrupt: 847 if not self.force: 848 ans = self.ask('Error detected. Do you want to clean the queue?', 849 default = 'y', choices=['y','n']) 850 else: 851 ans = 'y' 852 if ans == 'y': 853 self.cluster.remove() 854 raise 855 856 self.do_collect('-refine')
857
858 - def resubmit(self, M_path, to_refine, event_to_file):
859 """resubmit various jobs""" 860 861 for card, event_list in to_refine.items(): 862 packets = {} 863 for event in event_list: 864 evt_nb_file = event_to_file[event] 865 if evt_nb_file in packets: 866 packets[evt_nb_file].append(event) 867 else: 868 packets[evt_nb_file] = [event] 869 870 # evt_file = (sample_nb // self.MWparam['mw_run']['event_packing']) 871 # open(restrict_path, 'w').write(' '.join(map(str, restrict_evt))) 872 873 max_evts = self.MWparam['mw_run']['nb_event_by_node'] 874 for evt_nb, evt_list in packets.items(): 875 restrict_path = pjoin(self.me_dir, 'SubProcesses', M_path, self.MWparam.name, 876 'restrict%i_%i.dat' % (card,evt_nb_file)) 877 open(restrict_path, 'w').write(' '.join(map(str, evt_list))) 878 879 nb_weights = len(evt_list) 880 for i in range(1+ (nb_weights-1)//max_evts): 881 #sub_list = evt_list[max_evts * i: max_evts * (i+1)] 882 self.submit_job(M_path, card, i, evt_file=evt_nb, 883 restrict_evt=True)
884 885 886 887 #=============================================================================== 888 # MadEventCmd 889 #===============================================================================
890 -class MadWeightCmdShell(MadWeightCmd, cmd.CmdShell):
891 """The command line processor of MadGraph""" 892 pass
893 894
895 -class CollectObj(dict):
896 pass 897 898 #2 #############################################################################
899 - def refine(self, other):
900 901 for card, CardDATA in other.items(): 902 if card not in self: 903 self[card] = other[card] 904 continue 905 906 for event, obj2 in CardDATA.items(): 907 self[card][event] = obj2
908 909 910
911 - def write(self, out_path, MWdir):
912 """ """ 913 fsock = open(out_path, 'w') 914 fsock.write('<subprocess id=\'%s\'>\n' % MWdir) 915 for card in self: 916 fsock.write('<card id=\'%s\'>\n' % card) 917 for event in self[card].values(): 918 event.write(fsock) 919 fsock.write('</card>\n') 920 921 fsock.write('</subprocess>') 922 fsock.close()
923 924 925 #1 #################################################################################
926 -class MWParserXML(xml.sax.handler.ContentHandler):
927 """ This class will organize in python obect the TF_param.dat file 928 (written in xml) 929 """ 930 931 #2 #############################################################################
932 - def __init__(self, keep_level='weight'):
933 self.in_el = {'process': '', 'card':'', 'event':'', 'tfset':'' , 934 'permutation':'', 'channel':'','full':''} 935 if keep_level in ['weight', 'event', 'debug']: 936 keep_level = 'tfset' 937 self.all_event = {} 938 self.keep_level = keep_level 939 self.buffer='' 940 self.output = CollectObj()
941 942 943 #2 #############################################################################
944 - def startElement(self, name, attributes):
945 946 # if in lower level than needed skip the collection of data 947 if self.in_el[self.keep_level] != '' or name == 'log': 948 return 949 950 obj_class = {'event': MW_driver.Weight, 'permutation':MW_driver.Permutation, 951 'channel': MW_driver.Channel, 'tfset':MW_driver.TFsets} 952 953 954 if name == "process": 955 pass 956 elif name == 'card': 957 id = attributes['id'] 958 if id not in self.output: 959 self.output[id] = {} 960 self.in_el[name] = self.output[id] 961 self.curr_card_id = id 962 elif name == 'event': 963 id = attributes['id'] 964 value = float(attributes['value']) 965 error = float(attributes['error']) 966 card = self.curr_card_id 967 if ((card,id) in self.all_event): 968 data = self.all_event[(card,id)] 969 else: 970 data = MW_driver.Weight(id, self.keep_level) 971 self.all_event[(card,id)] = data 972 data.value = value 973 data.error = error 974 self.in_el['event'] = data 975 # assign it in the mother: 976 card = self.in_el['card'] 977 card[id] = data 978 elif name == 'tfset': 979 id = attributes['id'] 980 value = float(attributes['value']) 981 error = float(attributes['error']) 982 data = MW_driver.TFsets(id) 983 data.value = value 984 data.error = error 985 data.add('0', '', value, error, '') 986 #assign it to the mother: 987 event = self.in_el['event'] 988 event[id] = data 989 self.in_el['tfset'] = data 990 elif name == 'permutation': 991 tfset = self.in_el['tfset'] 992 993 id = attributes['id'] 994 value = float(attributes['value']) 995 error = float(attributes['error']) 996 data = MW_driver.Permutation(id, '') 997 data.value = value 998 data.error = error 999 self.in_el['permutation'] = data 1000 # assign it in the mother: 1001 tfset[id] = data 1002 elif name == 'channel': 1003 id = attributes['id'] 1004 value = float(attributes['value']) 1005 error = float(attributes['error']) 1006 data = MW_driver.Channel(id, value, error) 1007 self.in_el['channel'] = data 1008 # assign it in the mother: 1009 permutation = self.in_el['permutation'] 1010 permutation[id] = data 1011 elif name in ['log','subprocess','br']: 1012 pass 1013 else: 1014 raise Exception(name) 1015 if name != 'br': 1016 self.text = StringIO()
1017
1018 - def characters(self, content):
1019 self.text.write(content)
1020
1021 - def endElement(self, name):
1022 if name == 'log': 1023 data = self.in_el['event'] 1024 data.log = self.text.getvalue() 1025 1026 self.in_el[name] = ''
1027 1028 #2 #############################################################################
1029 - def read_file(self,filepos):
1030 """ parse the file and fulfill the object """ 1031 self.output = CollectObj() 1032 parser = xml.sax.make_parser( ) 1033 parser.setContentHandler(self) 1034 parser.parse(filepos) 1035 return self.output
1036