From 6cd42b6d1de1feaed37a14062f56952a91632317 Mon Sep 17 00:00:00 2001
From: "huanr@chromium.org"
<huanr@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>
Date: Wed, 29 Jul 2009 21:15:55 +0000
Subject: [PATCH] Update smoketests.py so we can run ui_tests in parallel, with
the number of shards equal to NUMBER_OF_PROCESSORS.
Review URL: http://codereview.chromium.org/159568
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@21996 0039d316-1c4b-4281-b951-d872f2087c98
---
chrome/tools/test/smoketests.py | 38 +++++++++++-
tools/python/google/process_utils.py | 86 ++++++++++++++++++++++++++++
2 files changed, 123 insertions(+), 1 deletion(-)
diff --git a/chrome/tools/test/smoketests.py b/chrome/tools/test/smoketests.py
index 50ddd83a989d3..f0513df67a3d5 100755
--- a/chrome/tools/test/smoketests.py
+++ b/chrome/tools/test/smoketests.py
@@ -109,6 +109,39 @@ def _MakeSubstitutions(list, options):
return [word % substitutions for word in list]
+def RunTestsInShards(test_command, verbose=True):
+ """Runs a test in shards. The number of shards is equal to
+ NUMBER_OF_PROCESSORS.
+
+ Args:
+ test_command: the test command to run, which is a list of one or more
+ strings.
+ verbose: if True, combines stdout and stderr into stdout.
+ Otherwise, prints only the command's stderr to stdout.
+
+ Returns:
+ The first shard process's exit status.
+
+ Raises:
+ CommandNotFound if the command executable could not be found.
+ """
+ processor_count = 2
+ try:
+ processor_count = int(os.environ['NUMBER_OF_PROCESSORS'])
+ except KeyError:
+ print 'No NUMBER_OF_PROCESSORS defined. Use 2 instances.'
+
+ commands = []
+ for i in xrange(processor_count):
+ command = [test_command[j] for j in xrange(len(test_command))]
+ # To support sharding, the test executable needs to provide --batch-count
+ # --batch-index command line switches.
+ command.append('--batch-count=%s' % processor_count)
+ command.append('--batch-index=%d' % i)
+ commands.append(command)
+ return google.process_utils.RunCommandsInParallel(commands, verbose)[0][0]
+
+
def main(options, args):
"""Runs all the selected tests for the given build type and target."""
options.build_type = options.build_type.lower()
@@ -184,7 +217,10 @@ def main(options, args):
print
print 'Running %s:' % test,
try:
- result = google.process_utils.RunCommand(command, options.verbose)
+ if test == 'ui':
+ result = RunTestsInShards(command, options.verbose)
+ else:
+ result = google.process_utils.RunCommand(command, options.verbose)
except google.process_utils.CommandNotFound, e:
print '%s' % e
raise
diff --git a/tools/python/google/process_utils.py b/tools/python/google/process_utils.py
index 5024ec7b524c9..64c92ea9f1976 100644
--- a/tools/python/google/process_utils.py
+++ b/tools/python/google/process_utils.py
@@ -133,3 +133,89 @@ def RunCommand(command, verbose=True):
"""
return RunCommandFull(command, verbose)[0]
+def RunCommandsInParallel(commands, verbose=True, collect_output=False,
+ print_output=True):
+ """Runs a list of commands in parallel, waits for all commands to terminate
+ and returns their status. If specified, the ouput of commands can be
+ returned and/or printed.
+
+ Args:
+ commands: the list of commands to run, each as a list of one or more
+ strings.
+ verbose: if True, combines stdout and stderr into stdout.
+ Otherwise, prints only the command's stderr to stdout.
+ collect_output: if True, collects the output of the each command as a list
+ of lines and returns it.
+ print_output: if True, prints the output of each command.
+
+ Returns:
+ A list of tuples consisting of each command's exit status and output. If
+ collect_output is False, the output will be [].
+
+ Raises:
+ CommandNotFound if any of the command executables could not be found.
+ """
+
+ command_num = len(commands)
+ outputs = [[] for i in xrange(command_num)]
+ procs = [None for i in xrange(command_num)]
+ eofs = [False for i in xrange(command_num)]
+
+ for command in commands:
+ print '\n' + subprocess.list2cmdline(command).replace('\\', '/') + '\n',
+
+ if verbose:
+ out = subprocess.PIPE
+ err = subprocess.STDOUT
+ else:
+ out = file(os.devnull, 'w')
+ err = subprocess.PIPE
+
+ for i in xrange(command_num):
+ try:
+ command = commands[i]
+ procs[i] = subprocess.Popen(command, stdout=out, stderr=err, bufsize=1)
+ except OSError, e:
+ if e.errno == errno.ENOENT:
+ raise CommandNotFound('Unable to find "%s"' % command[0])
+ raise
+ # We could consider terminating the processes already started.
+ # But Popen.kill() is only available in version 2.6.
+ # For now the clean up is done by KillAll.
+
+ while True:
+ eof_all = True
+ for i in xrange(command_num):
+ if eofs[i]:
+ continue
+ if verbose:
+ read_from = procs[i].stdout
+ else:
+ read_from = procs[i].stderr
+ line = read_from.readline()
+ if line:
+ eof_all = False
+ line = line.rstrip()
+ outputs[i].append(line)
+ if print_output:
+ # Windows Python converts \n to \r\n automatically whenever it
+ # encounters it written to a text file (including stdout). The only
+ # way around it is to write to a binary file, which isn't feasible
+ # for stdout. So we end up with \r\n here even though we explicitly
+ # write \n. (We could write \r instead, which doesn't get converted
+ # to \r\n, but that's probably more troublesome for people trying to
+ # read the files.)
+ print line + '\n',
+ else:
+ eofs[i] = True
+ if eof_all:
+ break
+
+ # Make sure the process terminates.
+ for i in xrange(command_num):
+ procs[i].wait()
+
+ if not verbose:
+ out.close()
+
+ return [(procs[i].returncode, outputs[i]) for i in xrange(command_num)]
--
GitLab