1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 from __future__ import division
16 import os
17 import math
18 import logging
19 import re
20 import xml.dom.minidom as minidom
21
22 logger = logging.getLogger('madevent.stdout')
23
24 pjoin = os.path.join
25 try:
26 import madgraph
27 except ImportError:
28 import internal.cluster as cluster
29 import internal.misc as misc
30 from internal import MadGraph5Error
31 else:
32 import madgraph.various.cluster as cluster
33 import madgraph.various.misc as misc
34 from madgraph import MadGraph5Error
35
37 """ A class to store statistics about a MadEvent run. """
38
40 """ Initialize the run dictionary. For now, the same as a regular
41 dictionary, except that we specify some default statistics. """
42
43 madloop_statistics = {
44 'unknown_stability' : 0,
45 'stable_points' : 0,
46 'unstable_points' : 0,
47 'exceptional_points' : 0,
48 'DP_usage' : 0,
49 'QP_usage' : 0,
50 'DP_init_usage' : 0,
51 'QP_init_usage' : 0,
52 'CutTools_DP_usage' : 0,
53 'CutTools_QP_usage' : 0,
54 'PJFry_usage' : 0,
55 'Golem_usage' : 0,
56 'IREGI_usage' : 0,
57 'Samurai_usage' : 0,
58 'Ninja_usage' : 0,
59 'Ninja_QP_usage' : 0,
60 'max_precision' : 1.0e99,
61 'min_precision' : 0.0,
62 'averaged_timing' : 0.0,
63 'n_madloop_calls' : 0,
64 'cumulative_timing' : 0.0,
65 'skipped_subchannel' : 0
66
67 }
68
69 for key, value in madloop_statistics.items():
70 self[key] = value
71
72 super(dict,self).__init__(*args, **opts)
73
75 """ Update the current statitistics with the new_stats specified."""
76
77 if isinstance(new_stats,RunStatistics):
78 new_stats = [new_stats, ]
79 elif isinstance(new_stats,list):
80 if any(not isinstance(_,RunStatistics) for _ in new_stats):
81 raise MadGraph5Error, "The 'new_stats' argument of the function "+\
82 "'updtate_statistics' must be a (possibly list of) "+\
83 "RunStatistics instance."
84
85 keys = set([])
86 for stat in [self,]+new_stats:
87 keys |= set(stat.keys())
88
89 new_stats = new_stats+[self,]
90 for key in keys:
91
92 if key=='max_precision':
93
94 self[key] = min( _[key] for _ in new_stats if key in _)
95 elif key=='min_precision':
96
97 self[key] = max( _[key] for _ in new_stats if key in _)
98 elif key=='averaged_timing':
99 n_madloop_calls = sum(_['n_madloop_calls'] for _ in new_stats if
100 'n_madloop_calls' in _)
101 if n_madloop_calls > 0 :
102 self[key] = sum(_[key]*_['n_madloop_calls'] for _ in
103 new_stats if (key in _ and 'n_madloop_calls' in _) )/n_madloop_calls
104 else:
105
106 self[key] = sum(_[key] for _ in new_stats if key in _)
107
109 """ Load the statistics from an xml node. """
110
111 def getData(Node):
112 return Node.childNodes[0].data
113
114 u_return_code = xml_node.getElementsByTagName('u_return_code')
115 u_codes = [int(_) for _ in getData(u_return_code[0]).split(',')]
116 self['CutTools_DP_usage'] = u_codes[1]
117 self['PJFry_usage'] = u_codes[2]
118 self['IREGI_usage'] = u_codes[3]
119 self['Golem_usage'] = u_codes[4]
120 self['Samurai_usage'] = u_codes[5]
121 self['Ninja_usage'] = u_codes[6]
122 self['Ninja_QP_usage'] = u_codes[8]
123 self['CutTools_QP_usage'] = u_codes[9]
124 t_return_code = xml_node.getElementsByTagName('t_return_code')
125 t_codes = [int(_) for _ in getData(t_return_code[0]).split(',')]
126 self['DP_usage'] = t_codes[1]
127 self['QP_usage'] = t_codes[2]
128 self['DP_init_usage'] = t_codes[3]
129 self['DP_init_usage'] = t_codes[4]
130 h_return_code = xml_node.getElementsByTagName('h_return_code')
131 h_codes = [int(_) for _ in getData(h_return_code[0]).split(',')]
132 self['unknown_stability'] = h_codes[1]
133 self['stable_points'] = h_codes[2]
134 self['unstable_points'] = h_codes[3]
135 self['exceptional_points'] = h_codes[4]
136 average_time = xml_node.getElementsByTagName('average_time')
137 avg_time = float(getData(average_time[0]))
138 self['averaged_timing'] = avg_time
139 cumulated_time = xml_node.getElementsByTagName('cumulated_time')
140 cumul_time = float(getData(cumulated_time[0]))
141 self['cumulative_timing'] = cumul_time
142 max_prec = xml_node.getElementsByTagName('max_prec')
143 max_prec = float(getData(max_prec[0]))
144
145 self['min_precision'] = max_prec
146 min_prec = xml_node.getElementsByTagName('min_prec')
147 min_prec = float(getData(min_prec[0]))
148
149 self['max_precision'] = min_prec
150 n_evals = xml_node.getElementsByTagName('n_evals')
151 n_evals = int(getData(n_evals[0]))
152 self['n_madloop_calls'] = n_evals
153
155 """Returns a one-line string summarizing the run statistics
156 gathered for the channel G."""
157
158
159
160 if self['n_madloop_calls']==0:
161 return ''
162
163 stability = [
164 ('tot#',self['n_madloop_calls']),
165 ('unkwn#',self['unknown_stability']),
166 ('UPS%',float(self['unstable_points'])/self['n_madloop_calls']),
167 ('EPS#',self['exceptional_points'])]
168
169 stability = [_ for _ in stability if _[1] > 0 or _[0] in ['UPS%','EPS#']]
170 stability = [(_[0],'%i'%_[1]) if isinstance(_[1], int) else
171 (_[0],'%.3g'%(100.0*_[1])) for _ in stability]
172
173 tools_used = [
174 ('CT_DP',float(self['CutTools_DP_usage'])/self['n_madloop_calls']),
175 ('CT_QP',float(self['CutTools_QP_usage'])/self['n_madloop_calls']),
176 ('PJFry',float(self['PJFry_usage'])/self['n_madloop_calls']),
177 ('Golem',float(self['Golem_usage'])/self['n_madloop_calls']),
178 ('IREGI',float(self['IREGI_usage'])/self['n_madloop_calls']),
179 ('Samurai',float(self['Samurai_usage'])/self['n_madloop_calls']),
180 ('Ninja_DP',float(self['Ninja_usage'])/self['n_madloop_calls']),
181 ('Ninja_QP',float(self['Ninja_QP_usage'])/self['n_madloop_calls'])]
182
183 tools_used = [(_[0],'%.3g'%(100.0*_[1])) for _ in tools_used if _[1] > 0.0 ]
184
185 to_print = [('%s statistics:'%(G if isinstance(G,str) else
186 str(os.path.join(list(G))))\
187 +(' %s,'%misc.format_time(int(self['cumulative_timing'])) if
188 int(self['cumulative_timing']) > 0 else '')
189 +((' Avg. ML timing = %i ms'%int(1.0e3*self['averaged_timing'])) if
190 self['averaged_timing'] > 0.001 else
191 (' Avg. ML timing = %i mus'%int(1.0e6*self['averaged_timing']))) \
192 +', Min precision = %.2e'%self['min_precision'])
193 ,' -> Stability %s'%dict(stability)
194 ,' -> Red. tools usage in %% %s'%dict(tools_used)
195
196
197
198
199
200 ]
201
202 if self['skipped_subchannel'] > 0 and not no_warning:
203 to_print.append("WARNING: Some event with large weight have been "+\
204 "discarded. This happened %s times." % self['skipped_subchannel'])
205
206 return ('\n'.join(to_print)).replace("'"," ")
207
209 """return if any stat needs to be reported as a warning
210 When this is True, the print_warning doit retourner un warning
211 """
212
213 if self['n_madloop_calls'] > 0:
214 fraction = self['exceptional_points']/float(self['n_madloop_calls'])
215 else:
216 fraction = 0.0
217
218 if self['skipped_subchannel'] > 0:
219 return True
220 elif fraction > 1.0e-4:
221 return True
222 else:
223 return False
224
226 """get a string with all the identified warning"""
227
228 to_print = []
229 if self['skipped_subchannel'] > 0:
230 to_print.append("Some event with large weight have been discarded."+\
231 " This happens %s times." % self['skipped_subchannel'])
232 if self['n_madloop_calls'] > 0:
233 fraction = self['exceptional_points']/float(self['n_madloop_calls'])
234 if fraction > 1.0e-4:
235 to_print.append("Some PS with numerical instability have been set "+\
236 "to a zero matrix-element (%.3g%%)" % (100.0*fraction))
237
238 return ('\n'.join(to_print)).replace("'"," ")
239
241
243 """Initialize all data """
244
245 self.run_statistics = RunStatistics()
246 self.name = name
247 self.parent_name = ''
248 self.axsec = 0
249 self.xsec = 0
250 self.xerru = 0
251 self.xerrc = 0
252 self.nevents = 0
253 self.nw = 0
254 self.maxit = 0
255 self.nunwgt = 0
256 self.luminosity = 0
257 self.mfactor = 1
258 self.ysec_iter = []
259 self.yerr_iter = []
260 self.yasec_iter = []
261 self.eff_iter = []
262 self.maxwgt_iter = []
263 self.maxwgt = 0
264 self.th_maxwgt= 0
265
266 self.th_nunwgt = 0
267
268
269 return
270
271
273 """read results.dat and fullfill information"""
274
275 if isinstance(filepath, str):
276 finput = open(filepath)
277 elif isinstance(filepath, file):
278 finput = filepath
279 else:
280 raise Exception, "filepath should be a path or a file descriptor"
281
282 i=0
283 found_xsec_line = False
284 for line in finput:
285
286
287 if '<' in line:
288 break
289 i+=1
290 if i == 1:
291 def secure_float(d):
292 try:
293 return float(d)
294 except ValueError:
295 m=re.search(r'''([+-]?[\d.]*)([+-]\d*)''', d)
296 if m:
297 return float(m.group(1))*10**(float(m.group(2)))
298 return
299
300 data = [secure_float(d) for d in line.split()]
301 self.axsec, self.xerru, self.xerrc, self.nevents, self.nw,\
302 self.maxit, self.nunwgt, self.luminosity, self.wgt, \
303 self.xsec = data[:10]
304 if len(data) > 10:
305 self.maxwgt = data[10]
306 if len(data) >12:
307 self.th_maxwgt, self.th_nunwgt = data[11:13]
308 if self.mfactor > 1:
309 self.luminosity /= self.mfactor
310 continue
311 try:
312 l, sec, err, eff, maxwgt, asec = line.split()
313 found_xsec_line = True
314 except:
315 break
316 self.ysec_iter.append(secure_float(sec))
317 self.yerr_iter.append(secure_float(err))
318 self.yasec_iter.append(secure_float(asec))
319 self.eff_iter.append(secure_float(eff))
320 self.maxwgt_iter.append(secure_float(maxwgt))
321
322 finput.seek(0)
323 xml = []
324 for line in finput:
325 if re.match('^.*<.*>',line):
326 xml.append(line)
327 break
328 for line in finput:
329 xml.append(line)
330
331 if xml:
332 self.parse_xml_results('\n'.join(xml))
333
334
335 if self.nevents == 0 and self.nunwgt == 0 and isinstance(filepath, str) and \
336 os.path.exists(pjoin(os.path.split(filepath)[0], 'nevts')):
337 nevts = int(open(pjoin(os.path.split(filepath)[0], 'nevts')).read())
338 self.nevents = nevts
339 self.nunwgt = nevts
340
342 """ Parse the xml part of the results.dat file."""
343
344 dom = minidom.parseString(xml)
345
346 statistics_node = dom.getElementsByTagName("run_statistics")
347
348 if statistics_node:
349 try:
350 self.run_statistics.load_statistics(statistics_node[0])
351 except ValueError, IndexError:
352 logger.warning('Fail to read run statistics from results.dat')
353
355 self.mfactor = int(value)
356
358 """Change the number of iterations for this process"""
359
360 if len(self.ysec_iter) <= nb_iter:
361 return
362
363
364 nb_to_rm = len(self.ysec_iter) - nb_iter
365 ysec = [0]
366 yerr = [0]
367 for i in range(nb_to_rm):
368 ysec[0] += self.ysec_iter[i]
369 yerr[0] += self.yerr_iter[i]**2
370 ysec[0] /= (nb_to_rm+1)
371 yerr[0] = math.sqrt(yerr[0]) / (nb_to_rm + 1)
372
373 for i in range(1, nb_iter):
374 ysec[i] = self.ysec_iter[nb_to_rm + i]
375 yerr[i] = self.yerr_iter[nb_to_rm + i]
376
377 self.ysec_iter = ysec
378 self.yerr_iter = yerr
379
380 - def get(self, name):
381
382 if name in ['xsec', 'xerru','xerrc']:
383 return getattr(self, name) * self.mfactor
384 elif name in ['luminosity']:
385
386
387 return getattr(self, name)
388 elif (name == 'eff'):
389 return self.xerr*math.sqrt(self.nevents/(self.xsec+1e-99))
390 elif name == 'xerr':
391 return math.sqrt(self.xerru**2+self.xerrc**2)
392 elif name == 'name':
393 return pjoin(self.parent_name, self.name)
394 else:
395 return getattr(self, name)
396
398
403
405 """read the data in the file"""
406 try:
407 oneresult = OneResult(name)
408 oneresult.set_mfactor(mfactor)
409 oneresult.read_results(filepath)
410 oneresult.parent_name = self.name
411 self.append(oneresult)
412 return oneresult
413 except Exception:
414 logger.critical("Error when reading %s" % filepath)
415 raise
416
417
419 """compute the value associate to this combination"""
420
421 self.compute_iterations()
422 self.axsec = sum([one.axsec for one in self])
423 self.xsec = sum([one.xsec for one in self])
424 self.xerrc = sum([one.xerrc for one in self])
425 self.xerru = math.sqrt(sum([one.xerru**2 for one in self]))
426
427 self.nevents = sum([one.nevents for one in self])
428 self.nw = sum([one.nw for one in self])
429 self.maxit = len(self.yerr_iter)
430 self.nunwgt = sum([one.nunwgt for one in self])
431 self.wgt = 0
432 self.luminosity = min([0]+[one.luminosity for one in self])
433 if update_statistics:
434 self.run_statistics.aggregate_statistics([_.run_statistics for _ in self])
435
437 """compute the value associate to this combination"""
438
439 nbjobs = len(self)
440 if not nbjobs:
441 return
442 self.axsec = sum([one.axsec for one in self]) / nbjobs
443 self.xsec = sum([one.xsec for one in self]) /nbjobs
444 self.xerrc = sum([one.xerrc for one in self]) /nbjobs
445 self.xerru = math.sqrt(sum([one.xerru**2 for one in self])) /nbjobs
446
447 self.nevents = sum([one.nevents for one in self])
448 self.nw = 0
449 self.maxit = 0
450 self.nunwgt = sum([one.nunwgt for one in self])
451 self.wgt = 0
452 self.luminosity = sum([one.luminosity for one in self])
453 self.ysec_iter = []
454 self.yerr_iter = []
455 self.th_maxwgt = 0.0
456 self.th_nunwgt = 0
457 for result in self:
458 self.ysec_iter+=result.ysec_iter
459 self.yerr_iter+=result.yerr_iter
460 self.yasec_iter += result.yasec_iter
461 self.eff_iter += result.eff_iter
462 self.maxwgt_iter += result.maxwgt_iter
463
464
465
467 """Compute iterations to have a chi-square on the stability of the
468 integral"""
469
470 nb_iter = min([len(a.ysec_iter) for a in self], 0)
471
472 for oneresult in self:
473 oneresult.change_iterations_number(nb_iter)
474
475
476 for i in range(nb_iter):
477 value = [one.ysec_iter[i] for one in self]
478 error = [one.yerr_iter[i]**2 for one in self]
479
480
481 self.ysec_iter.append(sum(value))
482 self.yerr_iter.append(math.sqrt(sum(error)))
483
484
485 template_file = \
486 """
487 %(diagram_link)s
488 <BR>
489 <b>s= %(cross).5g ± %(error).3g (%(unit)s)</b><br><br>
490 <table class="sortable" id='tablesort'>
491 <tr><th>Graph</th>
492 <th> %(result_type)s</th>
493 <th>Error</th>
494 <th>Events (K)</th>
495 <th>Unwgt</th>
496 <th>Luminosity</th>
497 </tr>
498 %(table_lines)s
499 </table>
500 </center>
501 <br><br><br>
502 """
503 table_line_template = \
504 """
505 <tr><td align=right>%(P_title)s</td>
506 <td align=right><a id="%(P_link)s" href=%(P_link)s onClick="check_link('%(P_link)s','%(mod_P_link)s','%(P_link)s')"> %(cross)s </a> </td>
507 <td align=right> %(error)s</td>
508 <td align=right> %(events)s</td>
509 <td align=right> %(unweighted)s</td>
510 <td align=right> %(luminosity)s</td>
511 </tr>
512 """
513
514 - def get_html(self,run, unit, me_dir = []):
515 """write html output"""
516
517
518 P_grouping = {}
519
520 tables_line = ''
521 for oneresult in self:
522 if oneresult.name.startswith('P'):
523 title = '<a href=../../SubProcesses/%(P)s/diagrams.html>%(P)s</a>' \
524 % {'P':oneresult.name}
525 P = oneresult.name.split('_',1)[0]
526 if P in P_grouping:
527 P_grouping[P] += float(oneresult.xsec)
528 else:
529 P_grouping[P] = float(oneresult.xsec)
530 else:
531 title = oneresult.name
532
533 if not isinstance(oneresult, Combine_results):
534
535 if os.path.exists(pjoin(me_dir, 'Events', run, 'alllogs_1.html')):
536 link = '../../Events/%(R)s/alllogs_1.html#/%(P)s/%(G)s' % \
537 {'P': self.name,
538 'G': oneresult.name,
539 'R': run}
540 mod_link = link
541 elif os.path.exists(pjoin(me_dir, 'Events', run, 'alllogs_0.html')):
542 link = '../../Events/%(R)s/alllogs_0.html#/%(P)s/%(G)s' % \
543 {'P': self.name,
544 'G': oneresult.name,
545 'R': run}
546 mod_link = link
547 else:
548
549 link = '../../SubProcesses/%(P)s/%(G)s/%(R)s_log.txt' % \
550 {'P': self.name,
551 'G': oneresult.name,
552 'R': run}
553 mod_link = '../../SubProcesses/%(P)s/%(G)s/log.txt' % \
554 {'P': self.name,
555 'G': oneresult.name}
556 else:
557 link = '#%s' % oneresult.name
558 mod_link = link
559
560 dico = {'P_title': title,
561 'P_link': link,
562 'mod_P_link': mod_link,
563 'cross': '%.4g' % oneresult.xsec,
564 'error': '%.3g' % oneresult.xerru,
565 'events': oneresult.nevents/1000.0,
566 'unweighted': oneresult.nunwgt,
567 'luminosity': '%.3g' % oneresult.luminosity
568 }
569
570 tables_line += self.table_line_template % dico
571
572 for P_name, cross in P_grouping.items():
573 dico = {'P_title': '%s sum' % P_name,
574 'P_link': './results.html',
575 'mod_P_link':'',
576 'cross': cross,
577 'error': '',
578 'events': '',
579 'unweighted': '',
580 'luminosity': ''
581 }
582 tables_line += self.table_line_template % dico
583
584 if self.name.startswith('P'):
585 title = '<dt><a name=%(P)s href=../../SubProcesses/%(P)s/diagrams.html>%(P)s</a></dt><dd>' \
586 % {'P':self.name}
587 else:
588 title = ''
589
590 dico = {'cross': self.xsec,
591 'abscross': self.axsec,
592 'error': self.xerru,
593 'unit': unit,
594 'result_type': 'Cross-Section',
595 'table_lines': tables_line,
596 'diagram_link': title
597 }
598
599 html_text = self.template_file % dico
600 return html_text
601
603 """write a correctly formatted results.dat"""
604
605 def fstr(nb):
606 data = '%E' % nb
607 if data == 'NAN':
608 nb, power = 0,0
609 else:
610 nb, power = data.split('E')
611 nb = float(nb) /10
612 power = int(power) + 1
613 return '%.5fE%+03i' %(nb,power)
614
615 line = '%s %s %s %i %i %i %i %s %s %s %s %s %i\n' % (fstr(self.axsec), fstr(self.xerru),
616 fstr(self.xerrc), self.nevents, self.nw, self.maxit, self.nunwgt,
617 fstr(self.luminosity), fstr(self.wgt), fstr(self.xsec), fstr(self.maxwgt),
618 fstr(self.th_maxwgt), self.th_nunwgt)
619 fsock = open(output_path,'w')
620 fsock.writelines(line)
621 for i in range(len(self.ysec_iter)):
622 line = '%s %s %s %s %s %s\n' % (i+1, self.ysec_iter[i], self.yerr_iter[i],
623 self.eff_iter[i], self.maxwgt_iter[i], self.yasec_iter[i])
624 fsock.writelines(line)
625
626
627
628 results_header = """
629 <head>
630 <title>Process results</title>
631 <script type="text/javascript" src="../sortable.js"></script>
632 <link rel=stylesheet href="../mgstyle.css" type="text/css">
633 </head>
634 <body>
635 <script type="text/javascript">
636 function UrlExists(url) {
637 var http = new XMLHttpRequest();
638 http.open('HEAD', url, false);
639 try{
640 http.send()
641 }
642 catch(err){
643 return 1==2;
644 }
645 return http.status!=404;
646 }
647 function check_link(url,alt, id){
648 var obj = document.getElementById(id);
649 if ( ! UrlExists(url)){
650 if ( ! UrlExists(alt)){
651 obj.href = alt;
652 return true;
653 }
654 obj.href = alt;
655 return false;
656 }
657 obj.href = url;
658 return 1==1;
659 }
660 </script>
661 """
662
664 """ """
665
666 run = cmd.results.current['run_name']
667 all = Combine_results(run)
668
669 for Pdir in open(pjoin(cmd.me_dir, 'SubProcesses','subproc.mg')):
670 Pdir = Pdir.strip()
671 P_comb = Combine_results(Pdir)
672
673 P_path = pjoin(cmd.me_dir, 'SubProcesses', Pdir)
674 G_dir = [G for G in os.listdir(P_path) if G.startswith('G') and
675 os.path.isdir(pjoin(P_path,G))]
676
677 try:
678 for line in open(pjoin(P_path, 'symfact.dat')):
679 name, mfactor = line.split()
680 if float(mfactor) < 0:
681 continue
682 if os.path.exists(pjoin(P_path, 'ajob.no_ps.log')):
683 continue
684
685 if not folder_names:
686 name = 'G' + name
687 P_comb.add_results(name, pjoin(P_path,name,'results.dat'), mfactor)
688 else:
689 for folder in folder_names:
690 if 'G' in folder:
691 dir = folder.replace('*', name)
692 else:
693 dir = folder.replace('*', '_G' + name)
694 P_comb.add_results(dir, pjoin(P_path,dir,'results.dat'), mfactor)
695 except IOError:
696 continue
697 P_comb.compute_values()
698 all.append(P_comb)
699 all.compute_values()
700 return all
701
702
704 """ folder_names has been added for the amcatnlo runs """
705 run = cmd.results.current['run_name']
706 if not os.path.exists(pjoin(cmd.me_dir, 'HTML', run)):
707 os.mkdir(pjoin(cmd.me_dir, 'HTML', run))
708
709 unit = cmd.results.unit
710 P_text = ""
711 Presults = collect_result(cmd, folder_names=folder_names)
712
713
714 for P_comb in Presults:
715 P_text += P_comb.get_html(run, unit, cmd.me_dir)
716 P_comb.compute_values()
717 P_comb.write_results_dat(pjoin(cmd.me_dir, 'SubProcesses', P_comb.name,
718 '%s_results.dat' % run))
719
720
721 Presults.write_results_dat(pjoin(cmd.me_dir,'SubProcesses', 'results.dat'))
722
723 fsock = open(pjoin(cmd.me_dir, 'HTML', run, 'results.html'),'w')
724 fsock.write(results_header)
725 fsock.write('%s <dl>' % Presults.get_html(run, unit, cmd.me_dir))
726 fsock.write('%s </dl></body>' % P_text)
727
728 return Presults.xsec, Presults.xerru
729