[gs-cvs] rev 7576 - trunk/gs/toolbin/tests

[giles at ghostscript.com]

Author: giles
Date: 2007-01-06 22:47:46 -0800 (Sat, 06 Jan 2007)
New Revision: 7576

Added:
   trunk/gs/toolbin/tests/run_parallel
Log:
Check in rough script for launching a parallel regression run on an 
OpenPBS-based cluster queuing system. This script runs 'upnodes' and 
chooses a likely target and job size, generates comparefiles.* and 
testing.cfg.* for each node, then submits the job.

Output is returned as a file in the launch directory 
'run_regression..e<jobid>' when the run is finished.


Added: trunk/gs/toolbin/tests/run_parallel
===================================================================
--- trunk/gs/toolbin/tests/run_parallel	2007-01-07 06:42:22 UTC (rev 7575)
+++ trunk/gs/toolbin/tests/run_parallel	2007-01-07 06:47:46 UTC (rev 7576)
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+import os
+import string
+import re
+
+# globals 
+run=True
+try:
+  home=os.environ["HOME"]
+except KeyError:
+  home=''
+testdir=home+"/tests/ps/ps3fts"
+configfile="testing.cfg"
+files=os.listdir(testdir)
+
+
+def choosecluster():
+  # figure out how many nodes are free
+  upnodes = os.popen("upnodes")
+  r = re.compile('^\s+(?P<cluster>\w+).*\s+(?P<procs>\d+)\s+(?P<free>\d+)\s*$')
+  clusters=[]
+  nodes = 0
+  for line in upnodes.readlines():
+    m = r.match(line)
+    if m: 
+      name = m.group("cluster")
+      procs = int(m.group("procs"))
+      free = int(m.group("free"))
+      # remember the cluster with the most free nodes
+      if free > nodes and name != 'total': 
+        nodes = free
+        cluster = name
+      clusters.append((name,procs,free))
+  return (cluster, nodes)
+
+# create a config file from the template for node
+configfiles=[]
+comparefiledirs=[]
+(cluster, nodes) = choosecluster()
+print "choosing %s with %d nodes free" % (cluster, nodes)
+print "configuring job..."
+for node in range(nodes):
+  infile=open(configfile)
+  outfilename=configfile +"."+ str(node)
+  outfile=open(outfilename,"w")
+  # remember the filename for later cleanup
+  configfiles.append(outfilename)
+  for line in infile.readlines():
+    try: 
+      key,value=string.split(line)
+      if key == "comparefiledir": 
+        value=value[:-1]+"."+str(node)+"/"
+        # remember this for cleanup
+        comparefiledirs.append(value)
+      if key == "log_stderr": value=value[:-4]+"."+str(node)+".log"
+      if key == "log_stdout": value=value[:-4]+"."+str(node)+".log"
+      #print key,value
+      outfile.write(key+"\t"+value+"\n")
+    except ValueError:
+      outfile.write(line)
+    nodedir=home+"/comparefiles."+str(node)
+  # create the per-node directories
+  os.system("rm -rf " + comparefiledirs[node])
+  os.mkdir(comparefiledirs[node])
+
+# split the test files into directories for each node
+node=0
+for file in files:
+  nodedir=comparefiledirs[node]
+  os.system("ln -s %s/%s %s/" % (testdir,file,nodedir))
+  node=node+1
+  if node >= len(comparefiledirs): node=0
+   
+def makepbs(filename):
+   'Make a pbs file for a command'
+   outfile=open(filename+".pbs","w")
+   if cluster == 'red' and nodes > 1:
+     # upnodes reports dual-core nodes twice
+     outfile.write("#PBS -l nodes=%d:run:%s:ppn=2\n" % (nodes/2,cluster))
+   else:
+     outfile.write("#PBS -l nodes=%d:run:%s\n" % (nodes,cluster))
+   outfile.write("cd $HOME/regression\n")
+   outfile.write("mpiexec -comm none ./"+filename+"\n")
+   outfile.close()
+
+# create our job description files
+makepbs("run_regression")
+makepbs("make_testdb")
+
+if run:
+   # qsub the pbs file
+
+   job= os.popen("qsub run_regression.pbs")
+   jobid=string.strip(job.readline())
+   print "run submitted as", jobid 
+
+   # append a follow-up job to do the cleanup
+   outfile=open("run_regression_cleanup.pbs","w")
+   outfile.write("#PBS -l nodes=1:run:nina -W depend=afterany:%s\n" % jobid)
+   outfile.write("cd " + home + "/regression" + "\n")
+   for node in range(nodes):
+     outfile.write("rm -rf " + comparefiledirs[node] + "\n")
+     outfile.write("rm " + configfiles[node] + "\n")
+   outfile.close()
+   job = os.popen("qsub run_regression_cleanup.pbs")
+   jobid = string.strip(job.readline())
+   print "cleanup job is", jobid
+


Property changes on: trunk/gs/toolbin/tests/run_parallel
___________________________________________________________________
Name: svn:executable
   + *
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native