1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """ Command interface for MadSpin """
16 from __future__ import division
17 import difflib
18 import logging
19 import math
20 import os
21 import re
22 import shutil
23 import time
24 import subprocess
25 from subprocess import Popen, PIPE, STDOUT
26
27
28 pjoin = os.path.join
29
30 import madgraph.interface.extended_cmd as extended_cmd
31 import madgraph.interface.madgraph_interface as mg_interface
32 import madgraph.interface.master_interface as master_interface
33 import madgraph.interface.common_run_interface as common_run_interface
34 import madgraph.iolibs.files as files
35 import MadSpin.interface_madspin as madspin_interface
36 import madgraph.various.misc as misc
37 import madgraph.various.banner as banner
38 import madgraph.various.lhe_parser as lhe_parser
39 import madgraph.various.combine_plots as combine_plots
40
41 import models.import_ufo as import_ufo
42 import models.check_param_card as check_param_card
43 import MadSpin.decay as madspin
44
45
46 logger = logging.getLogger('decay.stdout')
47 logger_stderr = logging.getLogger('decay.stderr')
48 cmd_logger = logging.getLogger('cmdprint2')
53 """Basic interface for reweighting operation"""
54
55 prompt = 'Reweight>'
56 debug_output = 'Reweight_debug'
57
58 @misc.mute_logger()
59 - def __init__(self, event_path=None, *completekey, **stdin):
60 """initialize the interface with potentially an event_path"""
61
62 if not event_path:
63 cmd_logger.info('************************************************************')
64 cmd_logger.info('* *')
65 cmd_logger.info('* Welcome to Reweight Module *')
66 cmd_logger.info('* *')
67 cmd_logger.info('************************************************************')
68 extended_cmd.Cmd.__init__(self, *completekey, **stdin)
69
70 self.model = None
71 self.has_standalone_dir = False
72
73 self.options = {'curr_dir': os.path.realpath(os.getcwd())}
74
75 self.events_file = None
76 self.processes = {}
77 self.mg5cmd = master_interface.MasterCmd()
78 self.seed = None
79
80 if event_path:
81 logger.info("Extracting the banner ...")
82 self.do_import(event_path)
83
84
85 self.calculator = {}
86 self.calculator_nbcall = {}
87
88
89 self.all_cross_section = {}
90
92 """import the event file"""
93
94
95 self.options['curr_dir'] = os.path.realpath(os.path.dirname(inputfile))
96 if os.path.basename(os.path.dirname(os.path.dirname(inputfile))) == 'Events':
97 self.options['curr_dir'] = pjoin(self.options['curr_dir'],
98 os.path.pardir, os.pardir)
99
100
101 if not os.path.exists(inputfile):
102 if inputfile.endswith('.gz'):
103 if not os.path.exists(inputfile[:-3]):
104 raise self.InvalidCmd('No such file or directory : %s' % inputfile)
105 else:
106 inputfile = inputfile[:-3]
107 elif os.path.exists(inputfile + '.gz'):
108 inputfile = inputfile + '.gz'
109 else:
110 raise self.InvalidCmd('No such file or directory : %s' % inputfile)
111
112 if inputfile.endswith('.gz'):
113 misc.gunzip(inputfile)
114 inputfile = inputfile[:-3]
115
116
117 self.lhe_input = lhe_parser.EventFile(os.path.realpath(inputfile))
118 if not self.lhe_input.banner:
119 value = self.ask("What is the path to banner", 0, [0], "please enter a path", timeout=0)
120 self.lhe_input.banner = open(value).read()
121 self.banner = self.lhe_input.get_banner()
122
123
124 if 'slha' not in self.banner:
125 self.events_file = None
126 raise self.InvalidCmd('Event file does not contain model information')
127 elif 'mg5proccard' not in self.banner:
128 self.events_file = None
129 raise self.InvalidCmd('Event file does not contain generation information')
130
131 if 'madspin' in self.banner:
132 raise self.InvalidCmd('Reweight should be done before running MadSpin')
133
134
135
136 process = self.banner.get_detail('proc_card', 'generate')
137 if '[' in process:
138 msg = 'Reweighting options is valid only for LO events'
139 raise Exception, msg
140 if not process:
141 msg = 'Invalid proc_card information in the file (no generate line):\n %s' % self.banner['mg5proccard']
142 raise Exception, msg
143 process, option = mg_interface.MadGraphCmd.split_process_line(process)
144 self.proc_option = option
145
146 logger.info("process: %s" % process)
147 logger.info("options: %s" % option)
148
149
151 """Check some basic property of the events file"""
152
153 sum_of_weight = 0
154 sum_of_abs_weight = 0
155 negative_event = 0
156 positive_event = 0
157
158 start = time.time()
159 for event_nb,event in enumerate(self.lhe_input):
160
161 if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),10)==0):
162 running_time = misc.format_timer(time.time()-start)
163 logger.info('Event nb %s %s' % (event_nb, running_time))
164 if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events')
165
166 event.check()
167
168 sum_of_weight += event.wgt
169 sum_of_abs_weight += abs(event.wgt)
170 if event.wgt < 0 :
171 negative_event +=1
172 else:
173 positive_event +=1
174
175 logger.info("total cross-section: %s" % sum_of_weight)
176 logger.info("total abs cross-section: %s" % sum_of_abs_weight)
177 logger.info("fraction of negative event %s", negative_event/(negative_event+positive_event))
178 logger.info("total number of events %s", (negative_event+positive_event))
179 logger.info("negative event %s", negative_event)
180
181
182
183
184 @extended_cmd.debug()
186 "Complete the import command"
187
188 args=self.split_arg(line[0:begidx])
189
190 if len(args) == 1:
191 base_dir = '.'
192 else:
193 base_dir = args[1]
194
195 return self.path_completion(text, base_dir)
196
197
198 if os.path.sep in args[-1] + text:
199 return self.path_completion(text,
200 pjoin(*[a for a in args if \
201 a.endswith(os.path.sep)]))
202
204 """checking the validity of the set command"""
205
206 if len(args) < 2:
207 raise self.InvalidCmd('set command requires at least two argument.')
208
209 valid = ['max_weight','seed','curr_dir']
210 if args[0] not in self.options and args[0] not in valid:
211 raise self.InvalidCmd('Unknown options %s' % args[0])
212
213 if args[0] == 'max_weight':
214 try:
215 args[1] = float(args[1].replace('d','e'))
216 except ValueError:
217 raise self.InvalidCmd('second argument should be a real number.')
218
219 elif args[0] == 'BW_effect':
220 if args[1] in [0, False,'.false.', 'F', 'f', 'False', 'no']:
221 args[1] = 0
222 elif args[1] in [1, True,'.true.', 'T', 't', 'True', 'yes']:
223 args[1] = 1
224 else:
225 raise self.InvalidCmd('second argument should be either T or F.')
226
227 elif args[0] == 'curr_dir':
228 if not os.path.isdir(args[1]):
229 raise self.InvalidCmd('second argument should be a path to a existing directory')
230
231
233 """check the validity of the launch command"""
234
235 if not self.lhe_input:
236 raise self.InvalidCmd("No events files defined.")
237
239 """help for the launch command"""
240
241 logger.info('''Add to the loaded events a weight associated to a
242 new param_card (to be define). The weight returned is the ratio of the
243 square matrix element by the squared matrix element of production.
244 All scale are kept fix for this re-weighting.''')
245
246
248 """end of the configuration launched the code"""
249
250 args = self.split_arg(line)
251 self.check_launch(args)
252
253 model_line = self.banner.get('proc_card', 'full_model_line')
254
255 if not self.has_standalone_dir:
256 self.create_standalone_directory()
257
258 ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card.dat'), 'w')
259 ff.write(self.banner['slha'])
260 ff.close()
261 ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card_orig.dat'), 'w')
262 ff.write(self.banner['slha'])
263 ff.close()
264 cmd = common_run_interface.CommonRunCmd.ask_edit_card_static(cards=['param_card.dat'],
265 ask=self.ask, pwd=pjoin(self.me_dir,'rw_me'))
266 new_card = open(pjoin(self.me_dir, 'rw_me', 'Cards', 'param_card.dat')).read()
267
268
269
270
271 if 'initrwgt' in self.banner:
272 if 'type=\'mg_reweighting\'' in self.banner['initrwgt']:
273 blockpat = re.compile(r'''<weightgroup type=\'mg_reweighting\'\s*>(?P<text>.*?)</weightgroup>''', re.I+re.M+re.S)
274 before, content, after = blockpat.split(self.banner['initrwgt'])
275 header_rwgt_other = before + after
276 pattern = re.compile('<weight id=\'mg_reweight_(?P<id>\d+)\'>(?P<info>[^<>]*)</weight>', re.S+re.I+re.M)
277 mg_rwgt_info = pattern.findall(content)
278 maxid = 0
279 for i, diff in mg_rwgt_info:
280 if int(i) > maxid:
281 maxid = int(i)
282 maxid += 1
283 rewgtid = maxid
284 else:
285 header_rwgt_other = self.banner['initrwgt']
286 mg_rwgt_info = []
287 rewgtid = 1
288 else:
289 self.banner['initrwgt'] = ''
290 header_rwgt_other = ''
291 mg_rwgt_info = []
292 rewgtid = 1
293
294
295
296 s_orig = self.banner['slha']
297 s_new = new_card
298 old_param = check_param_card.ParamCard(s_orig.splitlines())
299 new_param = check_param_card.ParamCard(s_new.splitlines())
300 card_diff = old_param.create_diff(new_param)
301 if card_diff == '':
302 raise self.InvalidCmd, 'original card and new card are identical'
303 mg_rwgt_info.append((str(rewgtid), card_diff))
304
305
306 self.banner['initrwgt'] = header_rwgt_other
307 self.banner['initrwgt'] += '\n<weightgroup type=\'mg_reweighting\'>\n'
308 for tag, diff in mg_rwgt_info:
309 self.banner['initrwgt'] += '<weight id=\'mg_reweight_%s\'>%s</weight>\n' % \
310 (tag, diff)
311 self.banner['initrwgt'] += '\n</weightgroup>\n'
312 self.banner['initrwgt'] = self.banner['initrwgt'].replace('\n\n', '\n')
313
314 output = open( self.lhe_input.name +'rw', 'w')
315
316
317 logger.info('starts to compute weight for events with the following modification to the param_card:')
318 logger.info(card_diff)
319
320
321 self.banner.write(output, close_tag=False)
322
323 if self.mother:
324 out_path = pjoin(self.mother.me_dir, 'Events', 'reweight.lhe')
325 output2 = open(out_path, 'w')
326 self.banner.write(output2, close_tag=False)
327 new_banner = banner.Banner(self.banner)
328 if not hasattr(self, 'run_card'):
329 self.run_card = new_banner.charge_card('run_card')
330 self.run_card['run_tag'] = 'reweight_%s' % rewgtid
331 new_banner['slha'] = s_new
332 del new_banner['initrwgt']
333
334 assert new_banner['slha'] != self.banner['slha']
335 assert 'initrwgt' in self.banner
336 ff = open(pjoin(self.mother.me_dir,'Events',self.mother.run_name, '%s_%s_banner.txt' % \
337 (self.mother.run_name, self.run_card['run_tag'])),'w')
338 new_banner.write(ff)
339 ff.close()
340
341
342 tag_name = 'mg_reweight_%s' % rewgtid
343 start = time.time()
344 cross = 0
345
346 os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'y'
347 if self.lhe_input.closed:
348 self.lhe_input = lhe_parser.EventFile(self.lhe_input.name)
349
350 for event_nb,event in enumerate(self.lhe_input):
351
352 if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),1000)==0):
353 running_time = misc.format_timer(time.time()-start)
354 logger.info('Event nb %s %s' % (event_nb, running_time))
355 if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events')
356
357
358 weight = self.calculate_weight(event)
359 cross += weight
360 event.reweight_data[tag_name] = weight
361
362 output.write(str(event))
363 if self.mother:
364 event.wgt = weight
365 event.reweight_data = {}
366 output2.write(str(event))
367
368 running_time = misc.format_timer(time.time()-start)
369 logger.info('All event done (nb_event: %s) %s' % (event_nb+1, running_time))
370
371 output.write('</LesHouchesEvents>\n')
372 output.close()
373 os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'n'
374 if self.mother:
375 output2.write('</LesHouchesEvents>\n')
376 output2.close()
377
378 if hasattr(self.mother, 'results'):
379 run_name = self.mother.run_name
380 results = self.mother.results
381 results.add_run(run_name, self.run_card, current=True)
382 results.add_detail('nb_event', event_nb+1)
383 results.add_detail('cross', cross)
384 results.add_detail('error', 'nan')
385 self.mother.create_plot(mode='reweight', event_path=output2.name,
386 tag=self.run_card['run_tag'])
387
388 if 'plot' in results.current.reweight:
389 html_dir = pjoin(self.mother.me_dir, 'HTML', run_name)
390 td = pjoin(self.mother.options['td_path'], 'td')
391 MA = pjoin(self.mother.options['madanalysis_path'])
392 path1 = pjoin(html_dir, 'plots_parton')
393 path2 = pjoin(html_dir, 'plots_%s' % self.run_card['run_tag'])
394 outputplot = path2
395 combine_plots.merge_all_plots(path2, path1, outputplot, td, MA)
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410 self.lhe_input.close()
411 files.mv(output.name, self.lhe_input.name)
412 logger.info('Event %s have now the additional weight' % self.lhe_input.name)
413 logger.info('new cross-section is : %g pb' % cross)
414 self.terminate_fortran_executables(new_card_only=True)
415
416 self.all_cross_section[rewgtid] = cross
417
418
419
420
428
429
431 """routine to return the matrix element"""
432
433 tag, order = event.get_tag_and_order()
434 orig_order, Pdir = self.id_to_path[tag]
435
436 run_id = (tag, hypp_id)
437
438 if run_id in self.calculator:
439 external = self.calculator[run_id]
440 self.calculator_nbcall[run_id] += 1
441 else:
442
443
444 tmpdir = pjoin(self.me_dir,'rw_me', 'SubProcesses', Pdir)
445 executable_prod="./check"
446 if not os.path.exists(pjoin(tmpdir, 'check')):
447 misc.compile( cwd=tmpdir)
448 external = Popen(executable_prod, stdout=PIPE, stdin=PIPE,
449 stderr=STDOUT, cwd=tmpdir)
450 self.calculator[run_id] = external
451 self.calculator_nbcall[run_id] = 1
452
453 if hypp_id == 1:
454 external.stdin.write('param_card.dat\n')
455 elif hypp_id == 0:
456 external.stdin.write('param_card_orig.dat\n')
457
458 external.stdin.write('%g\n' % event.aqcd)
459 stdin_text = event.get_momenta_str(orig_order)
460 external.stdin.write(stdin_text)
461 me_value = external.stdout.readline()
462 try:
463 me_value = float(me_value)
464 except Exception:
465 print 'ZERO DETECTED'
466 print stdin_text
467 print me_value
468 os.system('lsof -p %s' % external.pid)
469 me_value = 0
470
471 if len(self.calculator) > 100:
472 logger.debug('more than 100 calculator. Perform cleaning')
473 nb_calls = self.calculator_nbcall.values()
474 nb_calls.sort()
475 cut = max([nb_calls[len(nb_calls)//2], 0.001 * nb_calls[-1]])
476 for key, external in list(self.calculator.items()):
477 nb = self.calculator_nbcall[key]
478 if nb < cut:
479 external.stdin.close()
480 external.stdout.close()
481 external.terminate()
482 del self.calculator[key]
483 del self.calculator_nbcall[key]
484 else:
485 self.calculator_nbcall[key] = self.calculator_nbcall[key] //10
486
487 return me_value
488
490 """routine to terminate all fortran executables"""
491
492 for (mode, production) in dict(self.calculator):
493
494 if new_card_only and production == 0:
495 continue
496 external = self.calculator[(mode, production)]
497 external.stdin.close()
498 external.stdout.close()
499 external.terminate()
500 del self.calculator[(mode, production)]
501
519
522
523
525 """Adding one element to the list based on the matrix element"""
526
527
528
529
530
531
532 @misc.mute_logger()
534 """generate the various directory for the weight evaluation"""
535
536
537 path_me = self.me_dir
538 try:
539 shutil.rmtree(pjoin(path_me,'rw_me'))
540 except Exception:
541 pass
542
543
544 complex_mass = False
545 has_cms = re.compile(r'''set\s+complex_mass_scheme\s*(True|T|1|true|$|;)''')
546 for line in self.banner.proc_card:
547 if line.startswith('set'):
548 self.mg5cmd.exec_cmd(line, printcmd=False, precmd=False, postcmd=False)
549 if has_cms.search(line):
550 complex_mass = True
551 elif line.startswith('define'):
552 try:
553 self.mg5cmd.exec_cmd(line, printcmd=False, precmd=False, postcmd=False)
554 except Exception:
555 pass
556
557 info = self.banner.get('proc_card', 'full_model_line')
558 if '-modelname' in info:
559 mg_names = False
560 else:
561 mg_names = True
562 model_name = self.banner.get('proc_card', 'model')
563 if model_name:
564 self.load_model(model_name, mg_names, complex_mass)
565 else:
566 raise self.InvalidCmd('Only UFO model can be loaded in this module.')
567
568 mgcmd = self.mg5cmd
569 modelpath = self.model.get('modelpath')
570 if os.path.basename(modelpath) != mgcmd._curr_model['name']:
571 name, restrict = mgcmd._curr_model['name'].rsplit('-',1)
572 if os.path.exists(pjoin(os.path.dirname(modelpath),name, 'restrict_%s.dat' % restrict)):
573 modelpath = pjoin(os.path.dirname(modelpath), mgcmd._curr_model['name'])
574
575 commandline="import model %s " % modelpath
576 mgcmd.exec_cmd(commandline)
577
578
579 processes = [line[9:].strip() for line in self.banner.proc_card
580 if line.startswith('generate')]
581 processes += [' '.join(line.split()[2:]) for line in self.banner.proc_card
582 if re.search('^\s*add\s+process', line)]
583 mgcmd.exec_cmd("set group_subprocesses False")
584
585 logger.info('generating the square matrix element for reweighting')
586 start = time.time()
587 commandline=''
588 for proc in processes:
589 if '[' not in proc:
590 commandline+="add process %s ;" % proc
591 else:
592 raise self.InvalidCmd('NLO processes can\'t be reweight')
593
594 commandline = commandline.replace('add process', 'generate',1)
595 logger.info(commandline)
596 mgcmd.exec_cmd(commandline, precmd=True)
597 commandline = 'output standalone_rw %s' % pjoin(path_me,'rw_me')
598 mgcmd.exec_cmd(commandline, precmd=True)
599 logger.info('Done %.4g' % (time.time()-start))
600 self.has_standalone_dir = True
601
602
603
604 matrix_elements = mgcmd._curr_matrix_elements.get_matrix_elements()
605
606 self.id_to_path = {}
607 for me in matrix_elements:
608 for proc in me.get('processes'):
609 initial = []
610 final = [l.get('id') for l in proc.get('legs')\
611 if l.get('state') or initial.append(l.get('id'))]
612 order = (initial, final)
613 tag = proc.get_initial_final_ids()
614 decay_finals = proc.get_final_ids_after_decay()
615
616 if tag[1] != decay_finals:
617 order = (initial, list(decay_finals))
618 decay_finals.sort()
619 tag = (tag[0], tuple(decay_finals))
620 Pdir = pjoin(path_me, 'rw_me', 'SubProcesses',
621 'P%s' % me.get('processes')[0].shell_string())
622 assert os.path.exists(Pdir), "Pdir %s do not exists" % Pdir
623 if tag in self.id_to_path:
624 if not Pdir == self.id_to_path[tag][1]:
625 misc.sprint(tag, Pdir, self.id_to_path[tag][1])
626 raise self.InvalidCmd, '2 different process have the same final states. This module can not handle such situation'
627 else:
628 continue
629 self.id_to_path[tag] = [order, Pdir]
630
631
632 - def load_model(self, name, use_mg_default, complex_mass=False):
652