#!/usr/bin/env python # # iterCommands.py: execute a command for a set of files # Less complete than 'find -exec' or 'find|xargs' but easier to use # Also print a progress bar and can run jobs in parallele. # # # Copyright (C) 2010 Clement Creusot # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import sys import re import string import os.path import getopt import time import math from subprocess import call,check_call,CalledProcessError from multiprocessing import Pool,Value #------------------------------------------------------------------------------ def print_help(): print "Usage: "+os.path.basename(sys.argv[0])+" \"command\" [items]" print " Option; -f FILE extract list of items from FILE" print " -p print commands only" print " -q quiet" print " -c continue if command fails" print " -g INT group input element in fix size sets" print " -P run jobs in parallele " print " -s SET use sequence defined in SET ('1,5,7' or '0:10' or '0:0.1:1' or '0:5,95:100' ...)" print " Query/Replace:" print " {} : name of the file" print " {AAA/BBB} : complete name of the file where regexp pattern AAA have been replaced by BBB" print " {AAA/BBB/b} : basename of the file where regexp pattern AAA have been replaced by BBB" sys.exit() #------------------------------------------------------------------------------ def print_error(*str): print "\nERROR: ", for i in str: print i, print sys.exit() #------------------------------------------------------------------------------ # replace the curve brackets by name, do some BASH like query replace # add "/b" at the end to get the query replace only on the basename def CommandReplaceBrackets(command, name): p = re.compile( '{[^{}]*}') itemsMatch = [] for item in p.findall(command): if cmp(item,"{}")==0: itemsMatch.append(name) else: tmpName = name queryreplace = string.split(string.strip(item,"{/}"),"/") if len(queryreplace)>2: if cmp(queryreplace[2],"b")==0: tmpName = os.path.basename(name) if len(queryreplace) ==1: queryreplace.append("") itemsMatch.append(re.sub(queryreplace[0],queryreplace[1],tmpName)) itemsNonMatch = p.split(command) newcommand = itemsNonMatch[0] for i in range(len(itemsMatch)): newcommand += itemsMatch[i] newcommand += itemsNonMatch[i+1] return newcommand #------------------------------------------------------------------------------ #Global variable startTime = 0 lastTime = 0 discarded = 0 def PrintProgressionBar(reached, total, comments, bar_width=40, out=sys.stderr): global startTime, lastTime, discarded if(reached==0): startTime = 0 lastTime = 0 discarded = 0 rest = "" if cmp(comments[0:9],"remaining")==0: rest = comments[9:] if startTime ==0: startTime = time.time() lastTime = startTime comments = "unknown" else: #if the mean is too low to be reliable #we use the last period and start to compute a new mean if (reached == discarded): mean = (time.time() - startTime) else: mean = (time.time() - startTime)/(reached-discarded) last = (time.time() - lastTime) if last > 5*mean: seconds = last*(total-reached) startTime = time.time() discarded = reached else: seconds = mean *(total-reached) comments = time.strftime("%H:%M:%S", time.gmtime(seconds)) lastTime = time.time() run_symbol=['|','\\','-','/'] percent = float(reached)/float(total) * 100 done = int(math.floor(float(reached)/float(total) * bar_width)) percent_str = "%(p)3.2f" % {"p":percent} percent_str = percent_str.rjust(6, ' ') sentence = "|" + "="*done + " "*(bar_width-done) + run_symbol[(int(total-reached))%4] + " ("+percent_str+"%) " + comments sentence += " "+rest[-(79-len(sentence)):] out.write("\r"+sentence.ljust(79,' ')) if reached==total: out.write("\n") out.flush() #------------------------------------------------------------------------------ def main(): # Reading the parameters try: optlist, args = getopt.getopt(sys.argv[1:], 'hf:pqcs:g:P') except getopt.GetoptError, err: print str(err) print_help() if len(args) < 1: print_help() command = args[0] fromFile = "" options = {} options["printCommand"] = False options["quiet"] = False options["stopAtFaillure"]=True options["parallele"] = False groupSize = 1 L = [] # use options for opt,value in optlist: if opt in ("-h", "--help"): print_help() elif opt=='-f': fromFile = value elif opt=='-p': options["printCommand"] = True elif opt=='-P': options["parallele"] = True elif opt=='-g': groupSize = int(value) elif opt=='-q': options["quiet"] = True elif opt=='-c': options["stopAtFaillure"] = False elif opt=='-s': t = string.split(value,",") for el in t: def tonum(s): d = float(s) i = int(s) if i==d: return i else: return d v = map(tonum,string.split(el,":")) if len(v)==1: L.extend(v) elif len(v)==2: L.extend(range(v[0],v[1])) elif len(v)==3: L.extend(range(v[0],v[1],v[2])) else: print "Unrecognized sequence format" print_help() else: print "Unhandled option" print_help() # set up the list of items items = [] if fromFile != "": if os.path.exists(fromFile): file = open(fromFile, "r") elif cmp(fromFile,"-") == 0: file = sys.stdin else: print_error("File "+fromFile+" doesnt exist") for line in file.readlines(): t = string.split(line) if t!= []: if t[0][0] != '#': items.append(t[0]) if os.path.exists(fromFile): file.close() else: if len(L)==0: if len(args) < 2: print_help() else: for i in range(len(args)-1): items.append(args[i+1]) if len(items)!=0 and len(L)!=0: print "Can not use both item list and sequence" print_help() if len(L)!=0: items = L # group by set of groupSize if requested if groupSize != 1 : groupNb = int(math.ceil(len(items)/float(groupSize))) S = [] for i in range(groupNb-1): S.append(string.join(items[i*groupSize:(i+1)*groupSize]," ")) S.append(string.join(items[(groupNb-1)*groupSize:]," ")) items = S iterCommand(items,command, options) # global variable and call function for parallele processing sharedIterCommandCount = Value('i',0) IterCommandQuietCall = False def auxCall(command): global itemNb, sharedIterCommandCount, IterCommandQuietCall call(command,shell=True) if not IterCommandQuietCall: sharedIterCommandCount.value += 1 PrintProgressionBar(sharedIterCommandCount.value,itemNb,"remaining") # Take a generic command and a set of items # Apply the command to each item def iterCommand(items,command,opt={}): global itemNb,IterCommandQuietCall itemNb = len(items) IterCommandQuietCall= (opt.has_key("quiet") and opt["quiet"]) # Parallele execution if opt.has_key("parallele") and opt["parallele"] and not (opt.has_key("printCommand") and opt["printCommand"]) : tasks = [] sharedIterCommandCount.value = 0 for f,item in enumerate(items): newcommand = CommandReplaceBrackets(command,item) tasks.append(newcommand) pool = Pool(None) results = [] # not used for now if not opt.has_key("quiet") or not opt["quiet"]: r = pool.map_async(auxCall, tasks, callback=results.append) else: r = pool.map_async(auxCall, tasks, callback=results.append) try: r.get(999999) # Wait on the results except KeyboardInterrupt: pool.terminate() print "Program Cancelled" sys.exit(1) else: # sequential execution for f,item in enumerate(items): if not opt.has_key("quiet") or not opt["quiet"]: PrintProgressionBar(f+1,itemNb,"remaining"+str(item)[0:20],30) newcommand = CommandReplaceBrackets(command,str(item)) if opt.has_key("printCommand") and opt["printCommand"]: print newcommand else: if opt.has_key("stopAtFaillure") and opt["stopAtFaillure"]: ret = call(newcommand,shell=True) if ret != 0: exit(ret) else: call(newcommand,shell=True) if __name__ == "__main__": main()