From: Greg Darke <greg+pc-3w14-0@tsukasa.net.au>
Date: Tue, 25 Nov 2008 02:09:10 +0000 (+1100)
Subject: Allow batchrun to run more than one job at a time
X-Git-Url: https://code.delx.au/transcoding/commitdiff_plain/758a8e724dd7470d917156e3c5e80b03cbf678ae

Allow batchrun to run more than one job at a time
---

diff --git a/batchrun.py b/batchrun.py
index ab64380..ae6eef0 100755
--- a/batchrun.py
+++ b/batchrun.py
@@ -1,14 +1,22 @@
 #!/usr/bin/env python
 
-import optparse, shlex, subprocess, sys, itertools
+import optparse, shlex, subprocess, os, sys, itertools
 
 def parseArgs():
 	parser = optparse.OptionParser(usage="%prog batchfile1 [batchfile2] ...")
+	parser.add_option("-j", "--jobs",
+		action="store", dest="jobs", default=1, type="int",
+		help="The number of concurrent jobs to run")
 	opts, args = parser.parse_args(sys.argv[1:])
-	return args
+	opts.runningJobs = []
+	return opts, args
 
-def run(args):
-	subprocess.Popen(args).wait()
+def run(batchOpts, args):
+	batchOpts.runningJobs.append(subprocess.Popen(args, stdin=file(os.devnull, 'r')))
+	if len(batchOpts.runningJobs) >= batchOpts.jobs:
+		for job in batchOpts.runningJobs:
+			job.wait()
+		batchOpts.runningJobs = []
 
 def getblocks(fd):
 	def _countIndentationLevel(s):
@@ -27,13 +35,13 @@ def getblocks(fd):
 		line = line[level:] # Slice off the indentation
 		yield level, line
 			
-def batchProcess(fd):
+def batchProcess(batchOpts, fd):
 	opts = []
 
 	for level, line in getblocks(fd):
 		oldLevel = len(opts) - 1
 		if level <= oldLevel:
-			run(itertools.chain(*opts))
+			run(batchOpts, itertools.chain(*opts))
 
 		opts = opts[:level] # Delete all options that belong to groups that are indented more than this one
 		assert len(opts) == level, 'Seems we missed some options somewhere'
@@ -41,13 +49,13 @@ def batchProcess(fd):
 		opts.append(shlex.split(line))
 
 	if len(opts) > 0:
-		run(itertools.chain(*opts))
+		run(batchOpts, itertools.chain(*opts))
 
 
 def main():
-	args = parseArgs()
+	opts, args = parseArgs()
 	for name in args:
-		batchProcess(open(name))
+		batchProcess(opts, open(name))
 
 if __name__ == "__main__":
 	main()