1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 """A set of functions performing routine administrative I/O tasks."""
17
18 import logging
19 import os
20 import re
21 import signal
22 import subprocess
23 import sys
24 import StringIO
25 import sys
26 import optparse
27 import time
28 import shutil
29 import gzip as ziplib
30
31 try:
32
33 import madgraph
34 from madgraph import MadGraph5Error, InvalidCmd
35 import madgraph.iolibs.files as files
36 except Exception, error:
37 if __debug__:
38 print error
39
40 import internal as madgraph
41 from internal import MadGraph5Error, InvalidCmd
42 import internal.files as files
43
44 logger = logging.getLogger('cmdprint.ext_program')
45 logger_stderr = logging.getLogger('madevent.misc')
46 pjoin = os.path.join
52 """Parse a newline separated list of "param=value" as a dictionnary
53 """
54
55 info_dict = {}
56 pattern = re.compile("(?P<name>\w*)\s*=\s*(?P<value>.*)",
57 re.IGNORECASE | re.VERBOSE)
58 for entry in fsock:
59 entry = entry.strip()
60 if len(entry) == 0: continue
61 m = pattern.match(entry)
62 if m is not None:
63 info_dict[m.group('name')] = m.group('value')
64 else:
65 raise IOError, "String %s is not a valid info string" % entry
66
67 return info_dict
68
69
70
71
72
73 -def mute_logger(names=['madgraph','ALOHA','cmdprint','madevent'], levels=[50,50,50,50]):
74 """change the logger level and restore those at their initial value at the
75 end of the function decorated."""
76 def control_logger(f):
77 def restore_old_levels(names, levels):
78 for name, level in zip(names, levels):
79 log_module = logging.getLogger(name)
80 log_module.setLevel(level)
81
82 def f_with_no_logger(self, *args, **opt):
83 old_levels = []
84 for name, level in zip(names, levels):
85 log_module = logging.getLogger(name)
86 old_levels.append(log_module.level)
87 log_module.setLevel(level)
88 try:
89 out = f(self, *args, **opt)
90 restore_old_levels(names, old_levels)
91 return out
92 except:
93 restore_old_levels(names, old_levels)
94 raise
95
96 return f_with_no_logger
97 return control_logger
98
103 """Returns the current version information of the MadGraph5_aMC@NLO package,
104 as written in the VERSION text file. If the file cannot be found,
105 a dictionary with empty values is returned. As an option, an info
106 string can be passed to be read instead of the file content.
107 """
108
109 if info_str is None:
110 info_dict = files.read_from_file(os.path.join(madgraph.__path__[0],
111 "VERSION"),
112 parse_info_str,
113 print_error=False)
114 else:
115 info_dict = parse_info_str(StringIO.StringIO(info_str))
116
117 return info_dict
118
123 """Returns the present time info for use in MG5 command history header.
124 """
125
126 creation_time = time.asctime()
127 time_info = {'time': creation_time,
128 'fill': ' ' * (26 - len(creation_time))}
129
130 return time_info
131
136 """Browse the subdirectories of the path 'start_path' and returns the first
137 one found which contains at least one file ending with the string extension
138 given in argument."""
139
140 subdirs=[pjoin(start_path,dir) for dir in os.listdir(start_path)]
141 for subdir in subdirs:
142 if os.path.isfile(subdir):
143 if os.path.basename(subdir).endswith(extension):
144 return start_path
145 elif os.path.isdir(subdir):
146 return find_includes_path(subdir, extension)
147 return None
148
149
150
151
152 -def which(program):
153 def is_exe(fpath):
154 return os.path.exists(fpath) and os.access(\
155 os.path.realpath(fpath), os.X_OK)
156
157 if not program:
158 return None
159
160 fpath, fname = os.path.split(program)
161 if fpath:
162 if is_exe(program):
163 return program
164 else:
165 for path in os.environ["PATH"].split(os.pathsep):
166 exe_file = os.path.join(path, program)
167 if is_exe(exe_file):
168 return exe_file
169 return None
170
175 def is_lib(fpath):
176 return os.path.exists(fpath) and os.access(fpath, os.R_OK)
177
178 if not lib:
179 return None
180
181 fpath, fname = os.path.split(lib)
182 if fpath:
183 if is_lib(lib):
184 return lib
185 else:
186 locations = sum([os.environ[env_path].split(os.pathsep) for env_path in
187 ["DYLD_LIBRARY_PATH","LD_LIBRARY_PATH","LIBRARY_PATH","PATH"]
188 if env_path in os.environ],[])
189 for path in locations:
190 lib_file = os.path.join(path, lib)
191 if is_lib(lib_file):
192 return lib_file
193 return None
194
199 """ Return nice information on the current variable """
200
201
202 info = [('type',type(var)),('str', var)]
203 if hasattr(var, 'func_doc'):
204 info.append( ('DOC', var.func_doc) )
205 if hasattr(var, '__doc__'):
206 info.append( ('DOC', var.__doc__) )
207 if hasattr(var, '__dict__'):
208 info.append( ('ATTRIBUTE', var.__dict__.keys() ))
209
210 spaces = ' ' * nb_space
211
212 outstr=''
213 for name, value in info:
214 outstr += '%s%3s : %s\n' % (spaces,name, value)
215
216 return outstr
217
218
219
220
221 wait_once = False
223
224 def deco_retry(f):
225 def deco_f_retry(*args, **opt):
226 for i in range(nb_try):
227 try:
228 return f(*args, **opt)
229 except KeyboardInterrupt:
230 raise
231 except Exception, error:
232 global wait_once
233 if not wait_once:
234 text = """Start waiting for update. (more info in debug mode)"""
235 logger.info(text)
236 logger_stderr.debug('fail to do %s function with %s args. %s try on a max of %s (%s waiting time)' %
237 (str(f), ', '.join([str(a) for a in args]), i+1, nb_try, sleep * (i+1)))
238 logger_stderr.debug('error is %s' % str(error))
239 wait_once = True
240 time.sleep(sleep * (i+1))
241
242 raise error.__class__, '[Fail %i times] \n %s ' % (i+1, error)
243 return deco_f_retry
244 return deco_retry
245
246
247
248
249 -def compile(arg=[], cwd=None, mode='fortran', job_specs = True, nb_core=1 ,**opt):
250 """compile a given directory"""
251
252 cmd = ['make']
253 try:
254 if nb_core > 1:
255 cmd.append('-j%s' % nb_core)
256 cmd += arg
257 p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
258 stderr=subprocess.STDOUT, cwd=cwd, **opt)
259 (out, err) = p.communicate()
260 except OSError, error:
261 if cwd and not os.path.exists(cwd):
262 raise OSError, 'Directory %s doesn\'t exists. Impossible to run make' % cwd
263 else:
264 error_text = "Impossible to compile %s directory\n" % cwd
265 error_text += "Trying to launch make command returns:\n"
266 error_text += " " + str(error) + "\n"
267 error_text += "In general this means that your computer is not able to compile."
268 if sys.platform == "darwin":
269 error_text += "Note that MacOSX doesn\'t have gmake/gfortan install by default.\n"
270 error_text += "Xcode3 contains those required programs"
271 raise MadGraph5Error, error_text
272
273 if p.returncode:
274
275 if not cwd:
276 cwd = os.getcwd()
277 all_file = [f.lower() for f in os.listdir(cwd)]
278 if 'makefile' not in all_file:
279 raise OSError, 'no makefile present in %s' % os.path.realpath(cwd)
280
281 if mode == 'fortran' and not (which('g77') or which('gfortran')):
282 error_msg = 'A fortran compiler (g77 or gfortran) is required to create this output.\n'
283 error_msg += 'Please install g77 or gfortran on your computer and retry.'
284 raise MadGraph5Error, error_msg
285 elif mode == 'cpp' and not which('g++'):
286 error_msg ='A C++ compiler (g++) is required to create this output.\n'
287 error_msg += 'Please install g++ (which is part of the gcc package) on your computer and retry.'
288 raise MadGraph5Error, error_msg
289
290
291 if any(tag.upper() in out.upper() for tag in ['real(kind=16)','real*16',
292 'complex*32']) and mode == 'fortran' and not \
293 ''.join(get_gfortran_version().split('.')) >= '46':
294 if not which('gfortran'):
295 raise MadGraph5Error, 'The fortran compiler gfortran v4.6 or later '+\
296 'is required to compile %s.\nPlease install it and retry.'%cwd
297 else:
298 logger_stderr.error('ERROR, you could not compile %s because'%cwd+\
299 ' your version of gfortran is older than 4.6. MadGraph5_aMC@NLO will carry on,'+\
300 ' but will not be able to compile an executable.')
301 return p.returncode
302
303 error_text = 'A compilation Error occurs '
304 if cwd:
305 error_text += 'when trying to compile %s.\n' % cwd
306 error_text += 'The compilation fails with the following output message:\n'
307 error_text += ' '+out.replace('\n','\n ')+'\n'
308 error_text += 'Please try to fix this compilations issue and retry.\n'
309 error_text += 'Help might be found at https://answers.launchpad.net/madgraph5.\n'
310 error_text += 'If you think that this is a bug, you can report this at https://bugs.launchpad.net/madgraph5'
311 raise MadGraph5Error, error_text
312 return p.returncode
313
315 """ Returns the gfortran version as a string.
316 Returns '0' if it failed."""
317 try:
318 p = Popen([compiler, '-dumpversion'], stdout=subprocess.PIPE,
319 stderr=subprocess.PIPE)
320 output, error = p.communicate()
321 version_finder=re.compile(r"(?P<version>(\d.)*\d)")
322 version = version_finder.search(output).group('version')
323 return version
324 except Exception:
325 return '0'
326
327 -def mod_compilator(directory, new='gfortran', current=None, compiler_type='gfortran'):
328
329 if type(directory)!=list:
330 directory=[directory]
331
332
333 file_to_change=find_makefile_in_dir(directory)
334 if compiler_type == 'gfortran':
335 comp_re = re.compile('^(\s*)FC\s*=\s*(.+)\s*$')
336 var = 'FC'
337 elif compiler_type == 'cpp':
338 comp_re = re.compile('^(\s*)CXX\s*=\s*(.+)\s*$')
339 var = 'CXX'
340 else:
341 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type
342
343 mod = False
344 for name in file_to_change:
345 lines = open(name,'r').read().split('\n')
346 for iline, line in enumerate(lines):
347 result = comp_re.match(line)
348 if result:
349 if new != result.group(2):
350 mod = True
351 lines[iline] = result.group(1) + var + "=" + new
352 if mod:
353 open(name,'w').write('\n'.join(lines))
354
359 """mute_logger (designed to work as with statement),
360 files allow to redirect the output of the log to a given file.
361 """
362
363 - def __init__(self, names, levels, files=None, **opt):
364 assert isinstance(names, list)
365 assert isinstance(names, list)
366
367 self.names = names
368 self.levels = levels
369 if isinstance(files, list):
370 self.files = files
371 else:
372 self.files = [files] * len(names)
373 self.logger_saved_info = {}
374 self.opts = opt
375
377 old_levels = []
378 for name, level, path in zip(self.names, self.levels, self.files):
379 if path:
380 self.setup_logFile_for_logger(path, name, **self.opts)
381 log_module = logging.getLogger(name)
382 old_levels.append(log_module.level)
383 log_module = logging.getLogger(name)
384 log_module.setLevel(level)
385 self.levels = old_levels
386
387 - def __exit__(self, ctype, value, traceback ):
396
398 """ Setup the logger by redirecting them all to logfiles in tmp """
399
400 logs = full_logname.split('.')
401 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\
402 range(len(full_logname.split('.')))]
403 for logname in lognames:
404 try:
405 os.remove(path)
406 except Exception, error:
407 pass
408 my_logger = logging.getLogger(logname)
409 hdlr = logging.FileHandler(path)
410
411
412 self.logger_saved_info[logname] = [hdlr, my_logger.handlers]
413
414
415 for old_hdlr in list(my_logger.handlers):
416 my_logger.removeHandler(old_hdlr)
417 my_logger.addHandler(hdlr)
418
419 my_logger.debug('Log of %s' % logname)
420
422 """ Setup the logger by redirecting them all to logfiles in tmp """
423
424 logs = full_logname.split('.')
425 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\
426 range(len(full_logname.split('.')))]
427 for logname in lognames:
428 if path:
429 try:
430 os.remove(path)
431 except Exception, error:
432 pass
433 my_logger = logging.getLogger(logname)
434 if logname in self.logger_saved_info:
435 my_logger.removeHandler(self.logger_saved_info[logname][0])
436 for old_hdlr in self.logger_saved_info[logname][1]:
437 my_logger.addHandler(old_hdlr)
438 else:
439 my_logger.setLevel(level)
440
446 """find the current compiler for the current directory"""
447
448
449
450 if compiler_type == 'fortran':
451 comp = re.compile("^\s*FC\s*=\s*([\w\/\\.\-]+)\s*")
452 elif compiler_type == 'cpp':
453 comp = re.compile("^\s*CXX\s*=\s*([\w\/\\.\-]+)\s*")
454 else:
455 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type
456
457 for line in open(path):
458 if comp.search(line):
459 compiler = comp.search(line).groups()[0]
460 return compiler
461
463 """ return a list of all file starting with makefile in the given directory"""
464
465 out=[]
466
467 if type(directory)==list:
468 for name in directory:
469 out+=find_makefile_in_dir(name)
470 return out
471
472
473 for name in os.listdir(directory):
474 if os.path.isdir(directory+'/'+name):
475 out+=find_makefile_in_dir(directory+'/'+name)
476 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('makefile'):
477 out.append(directory+'/'+name)
478 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('make_opt'):
479 out.append(directory+'/'+name)
480 return out
481
483
484
485 os.path.walk('.', rm_file_extension, '.o')
486
487
488 libraries = ['libblocks.a', 'libgeneric_mw.a', 'libMWPS.a', 'libtools.a', 'libdhelas3.a',
489 'libdsample.a', 'libgeneric.a', 'libmodel.a', 'libpdf.a', 'libdhelas3.so', 'libTF.a',
490 'libdsample.so', 'libgeneric.so', 'libmodel.so', 'libpdf.so']
491 lib_pos='./lib'
492 [os.remove(os.path.join(lib_pos, lib)) for lib in libraries \
493 if os.path.exists(os.path.join(lib_pos, lib))]
494
499
503 replace_dict = dict(key_values)
504 replacement_function = lambda match: replace_dict[match.group(0)]
505 pattern = re.compile("|".join([re.escape(k) for k, v in key_values]), re.M)
506 return lambda string: pattern.sub(replacement_function, string)
507
510
513 def deco_check(f):
514 def deco_f(arg, *args, **opt):
515 try:
516 return f(arg, *args, **opt)
517 except OSError, error:
518 logger.debug('try to recover from %s' % error)
519 if isinstance(arg, list):
520 prog = arg[0]
521 else:
522 prog = arg[0]
523
524
525 if error.errno == 13:
526 if os.path.exists(prog):
527 os.system('chmod +x %s' % prog)
528 elif 'cwd' in opt and opt['cwd'] and \
529 os.path.isfile(pjoin(opt['cwd'],arg[0])):
530 os.system('chmod +x %s' % pjoin(opt['cwd'],arg[0]))
531 return f(arg, *args, **opt)
532
533 elif error.errno == 2:
534
535 raise Exception, '%s fails with no such file or directory' \
536 % arg
537 else:
538 raise
539 return deco_f
540 return deco_check
541
542
543 @check_system_error()
544 -def call(arg, *args, **opt):
545 """nice way to call an external program with nice error treatment"""
546 return subprocess.call(arg, *args, **opt)
547
548 @check_system_error()
549 -def Popen(arg, *args, **opt):
550 """nice way to call an external program with nice error treatment"""
551 return subprocess.Popen(arg, *args, **opt)
552
555 """try to open a file with multiple try to ensure that filesystem is sync"""
556 return open(filepath, *args, ** opt)
557
558
559
560
561
562 -def tail(f, n, offset=None):
563 """Reads a n lines from f with an offset of offset lines. The return
564 value is a tuple in the form ``lines``.
565 """
566 avg_line_length = 74
567 to_read = n + (offset or 0)
568
569 while 1:
570 try:
571 f.seek(-(avg_line_length * to_read), 2)
572 except IOError:
573
574
575 f.seek(0)
576 pos = f.tell()
577 lines = f.read().splitlines()
578 if len(lines) >= to_read or pos == 0:
579 return lines[-to_read:offset and -offset or None]
580 avg_line_length *= 1.3
581 avg_line_length = int(avg_line_length)
582
587 """return the last line of a file"""
588
589 return tail(fsock, 1)[0]
590
592 """read a file returning the lines in reverse order for each call of readline()
593 This actually just reads blocks (4096 bytes by default) of data from the end of
594 the file and returns last line in an internal buffer."""
595
596
598 """ readline in a backward way """
599
600 while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size):
601 self.blkcount = self.blkcount + 1
602 line = self.data[0]
603 try:
604 self.seek(-self.blksize * self.blkcount, 2)
605 self.data = (self.read(self.blksize) + line).split('\n')
606 except IOError:
607 self.seek(0)
608 data = self.read(self.size - (self.blksize * (self.blkcount-1))) + line
609 self.data = data.split('\n')
610
611 if len(self.data) == 0:
612 return ""
613
614 line = self.data.pop()
615 return line + '\n'
616
617 - def __init__(self, filepos, blksize=4096):
618 """initialize the internal structures"""
619
620
621 self.size = os.stat(filepos)[6]
622
623 self.blksize = blksize
624
625 self.blkcount = 1
626 file.__init__(self, filepos, 'rb')
627
628
629 if self.size > self.blksize:
630 self.seek(-self.blksize * self.blkcount, 2)
631 self.data = self.read(self.blksize).split('\n')
632
633
634 if not self.data[-1]:
635 self.data.pop()
636
638 line = self.readline()
639 if line:
640 return line
641 else:
642 raise StopIteration
643
659
673
679 """create a temporary directory and ensure this one to be cleaned.
680 """
681
682 - def __init__(self, suffix='', prefix='tmp', dir=None):
683 self.nb_try_remove = 0
684 import tempfile
685 self.path = tempfile.mkdtemp(suffix, prefix, dir)
686
687
688 - def __exit__(self, ctype, value, traceback ):
689 try:
690 shutil.rmtree(self.path)
691 except OSError:
692 self.nb_try_remove += 1
693 if self.nb_try_remove < 3:
694 time.sleep(10)
695 self.__exit__(ctype, value, traceback)
696 else:
697 logger.warning("Directory %s not completely cleaned. This directory can be removed manually" % self.path)
698
701
702
703
704 -def gunzip(path, keep=False, stdout=None):
705 """ a standard replacement for os.system('gunzip -f %s.gz ' % event_path)"""
706
707 if not path.endswith(".gz"):
708 if os.path.exists("%s.gz" % path):
709 path = "%s.gz" % path
710 else:
711 raise Exception, "%(path)s does not finish by .gz and the file %(path)s.gz does not exists" %\
712 {"path": path}
713 if not stdout:
714 stdout = path[:-3]
715 open(stdout,'w').write(ziplib.open(path, "r").read())
716 if not keep:
717 os.remove(path)
718
719 -def gzip(path, stdout=None, error=True):
720 """ a standard replacement for os.system('gzip %s ' % path)"""
721 if not stdout:
722 stdout = "%s.gz" % path
723 elif not stdout.endswith(".gz"):
724 stdout = "%s.gz" % stdout
725 try:
726 ziplib.open(stdout,"w").write(open(path).read())
727 except:
728 if error:
729 raise
730 else:
731 os.remove(path)
732
737 """ a convinient class to open a file """
738
739 web_browser = None
740 eps_viewer = None
741 text_editor = None
742 configured = False
743
745 """open a file"""
746
747
748 if not self.configured:
749 self.configure()
750
751 try:
752 extension = filename.rsplit('.',1)[1]
753 except IndexError:
754 extension = ''
755
756
757
758 if extension in ['html','htm','php']:
759 self.open_program(self.web_browser, filename, background=True)
760 elif extension in ['ps','eps']:
761 self.open_program(self.eps_viewer, filename, background=True)
762 else:
763 self.open_program(self.text_editor,filename, mac_check=False)
764
765
766 @classmethod
789
790 @classmethod
828
829
830 @staticmethod
832 """find a valid shell program in the list"""
833
834 for p in possibility:
835 if which(p):
836 logger.info('Using default %s \"%s\". ' % (program, p) + \
837 'Set another one in ./input/mg5_configuration.txt')
838 return p
839
840 logger.info('No valid %s found. ' % program + \
841 'Please set in ./input/mg5_configuration.txt')
842 return None
843
844
845 - def open_program(self, program, file_path, mac_check=True, background=False):
846 """ open a file with a given program """
847
848 if mac_check==True and sys.platform == 'darwin':
849 return self.open_mac_program(program, file_path)
850
851
852 if program:
853 arguments = program.split()
854 arguments.append(file_path)
855
856 if not background:
857 subprocess.call(arguments)
858 else:
859 import thread
860 thread.start_new_thread(subprocess.call,(arguments,))
861 else:
862 logger.warning('Not able to open file %s since no program configured.' % file_path + \
863 'Please set one in ./input/mg5_configuration.txt')
864
866 """ open a text with the text editor """
867
868 if not program:
869
870 os.system('open %s' % file_path)
871 elif which(program):
872
873 arguments = program.split()
874 arguments.append(file_path)
875 subprocess.call(arguments)
876 else:
877
878 os.system('open -a %s %s' % (program, file_path))
879
881 """ check if a path is executable"""
882 try:
883 return os.access(path, os.X_OK)
884 except Exception:
885 return False
886
888 """Option Peaser which raise an error instead as calling exit"""
889
890 - def exit(self, status=0, msg=None):
895
897 """Returns the current line number in our program."""
898 import inspect
899 if opt.has_key('log'):
900 log = opt['log']
901 else:
902 log = logging.getLogger('madgraph')
903 if opt.has_key('level'):
904 level = opt['level']
905 else:
906 level = logging.getLogger('madgraph').level
907 if level == 20:
908 level = 10
909 lineno = inspect.currentframe().f_back.f_lineno
910 fargs = inspect.getframeinfo(inspect.currentframe().f_back)
911 filename, lineno = fargs[:2]
912
913
914
915
916 log.log(level, ' '.join([str(a) for a in args]) + \
917 '\nraised at %s at line %s ' % (filename, lineno))
918
919 return
920
921
922
923
924 -def equal(a,b,sig_fig=6, zero_limit=True):
925 """function to check if two float are approximatively equal"""
926 import math
927
928 if not a or not b:
929 if zero_limit:
930 power = sig_fig + 1
931 else:
932 return a == b
933 else:
934 power = sig_fig - int(math.log10(abs(a))) + 1
935
936 return ( a==b or abs(int(a*10**power) - int(b*10**power)) < 10)
937
943 self.newPath = newPath
944
946 self.savedPath = os.getcwd()
947 os.chdir(self.newPath)
948
949 - def __exit__(self, etype, value, traceback):
950 os.chdir(self.savedPath)
951
952
953
954
955
956
957 -class digest:
958
960 try:
961 return self.test_hashlib()
962 except Exception:
963 pass
964 try:
965 return self.test_md5()
966 except Exception:
967 pass
968 try:
969 return self.test_zlib()
970 except Exception:
971 pass
972
974 import hashlib
975 def digest(text):
976 """using mg5 for the hash"""
977 t = hashlib.md5()
978 t.update(text)
979 return t.hexdigest()
980 return digest
981
983 import md5
984 def digest(text):
985 """using mg5 for the hash"""
986 t = md5.md5()
987 t.update(text)
988 return t.hexdigest()
989 return digest
990
992 import zlib
993 def digest(text):
994 return zlib.adler32(text)
995
996 digest = digest().test_all()
997