Commit 40274059 authored by Kenneth Russell's avatar Kenneth Russell Committed by Commit Bot

Add --isolated-script-test-filter-file argument.

This takes a file of test names separated by newlines and passes them
to the target test harness appropriately. Isolated script tests
abstract over multiple kinds of harnesses, and there are multiple ways
they receive their test lists.

Tested with the scenarios described in the bug report.

BUG=533481

Change-Id: I38edd00cabd47c31435d3a1cbc51d1e089f5a5bb
Reviewed-on: https://chromium-review.googlesource.com/754133
Commit-Queue: Kenneth Russell <kbr@chromium.org>
Reviewed-by: default avatarDirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#516125}
parent 9e7b5573
......@@ -179,6 +179,18 @@ def parse_common_test_results(json_results, test_separator='/'):
return results
def load_filter_list(filter_file):
"""Helper for isolated script test wrappers."""
lines = []
with open(filter_file, 'r') as f:
for line in f:
# Eliminate completely empty lines for robustness.
stripped = line.strip()
if stripped:
lines.append(stripped)
return lines
def run_integration_test(script_to_run, extra_args, log_file, output):
integration_test_res = subprocess.call(
[sys.executable, script_to_run] + extra_args)
......
......@@ -34,6 +34,9 @@ def main(argv):
parser.add_argument(
'--isolated-script-test-perf-output', type=str,
required=False)
parser.add_argument(
'--isolated-script-test-filter-file', type=str,
required=False)
args = parser.parse_args(argv)
......
......@@ -40,7 +40,8 @@ def main():
index = 0
for arg in rest_args:
if ('--isolated-script-test-chartjson-output' in arg or
'--isolated-script-test-perf-output' in arg):
'--isolated-script-test-perf-output' in arg or
'--isolated-script-test-filter-file' in arg):
rest_args.pop(index)
break
index += 1
......
......@@ -14,9 +14,18 @@ argument:
json is written to that file in the format produced by
common.parse_common_test_results.
Optional argument:
--isolated-script-test-filter-file=[FILENAME]
points to a file containing newline-separated test names, to run just
that subset of tests. This list is parsed by this harness and sent
down via the --test-filter argument.
This script is intended to be the base command invoked by the isolate,
followed by a subsequent Python script. It could be generalized to
invoke an arbitrary executable.
"""
import argparse
......@@ -44,6 +53,9 @@ def main():
parser.add_argument(
'--isolated-script-test-output', type=str,
required=True)
parser.add_argument(
'--isolated-script-test-filter-file', type=str,
required=False)
parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
args, rest_args = parser.parse_known_args()
# Remove the chartjson extra arg until this script cares about chartjson
......@@ -55,6 +67,13 @@ def main():
rest_args.pop(index)
break
index += 1
if args.isolated_script_test_filter_file:
# This test harness doesn't yet support reading the test list from
# a file.
filter_list = common.load_filter_list(args.isolated_script_test_filter_file)
# Need to convert this to a valid regex.
filter_regex = '(' + '|'.join(filter_list) + ')'
rest_args.append('--test-filter=' + filter_regex)
xvfb_proc = None
openbox_proc = None
......
......@@ -14,6 +14,14 @@ common.parse_common_test_results.
--isolated-script-test-chartjson-output=[FILE]
stdout is written to this file containing chart results for the perf dashboard
Optional argument:
--isolated-script-test-filter-file=[FILENAME]
points to a file containing newline-separated test names, to run just
that subset of tests. This gets remapped to the command line argument
--test-launcher-filter-file.
This script is intended to be the base command invoked by the isolate,
followed by a subsequent non-python executable. It is modeled after
run_gpu_integration_test_as_gtest.py
......@@ -67,6 +75,9 @@ def main():
parser.add_argument(
'--isolated-script-test-perf-output', type=str,
required=False)
parser.add_argument(
'--isolated-script-test-filter-file', type=str,
required=False)
parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
args, rest_args = parser.parse_known_args()
......@@ -97,6 +108,9 @@ def main():
extra_flags.append('--verbose')
if not '--test-launcher-print-test-stdio=always' in extra_flags:
extra_flags.append('--test-launcher-print-test-stdio=always')
if args.isolated_script_test_filter_file:
extra_flags.append('--test-launcher-filter-file=' +
args.isolated_script_test_filter_file)
if IsWindows():
executable = '.\%s.exe' % executable
......
......@@ -15,6 +15,14 @@ remapped to the command line argument --write-full-results-to.
json is written to that file in the format produced by
common.parse_common_test_results.
Optional argument:
--isolated-script-test-filter-file=[FILENAME]
points to a file containing newline-separated test names, to run just
that subset of tests. This gets remapped to the command line argument
--file-list.
This script is intended to be the base command invoked by the isolate,
followed by a subsequent Python script. It could be generalized to
invoke an arbitrary executable.
......@@ -47,12 +55,16 @@ def main():
parser.add_argument('--isolated-script-test-chartjson-output', type=str)
# This argument is ignored for now.
parser.add_argument('--isolated-script-test-perf-output', type=str)
# This argument is translated below.
parser.add_argument('--isolated-script-test-filter-file', type=str)
args, rest_args = parser.parse_known_args()
env = os.environ
cmd = [sys.executable] + rest_args
cmd += ['--write-full-results-to', args.isolated_script_test_output]
if args.isolated_script_test_filter_file:
cmd += ['--file-list=' + args.isolated_script_test_filter_file]
if args.xvfb:
return xvfb.run_executable(cmd, env)
else:
......
......@@ -14,6 +14,14 @@ argument:
json is written to that file in the format detailed here:
https://www.chromium.org/developers/the-json-test-results-format
Optional argument:
--isolated-script-test-filter-file=[FILENAME]
points to a file containing newline-separated test names, to run just
that subset of tests. This gets remapped to the command line argument
--file-list.
This script is intended to be the base command invoked by the isolate,
followed by a subsequent Python script. It could be generalized to
invoke an arbitrary executable.
......@@ -58,6 +66,8 @@ def main():
'--isolated-script-test-chartjson-output', required=False)
parser.add_argument(
'--isolated-script-test-perf-output', required=False)
parser.add_argument(
'--isolated-script-test-filter-file', type=str, required=False)
parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
parser.add_argument('--output-format', action='append')
parser.add_argument('--builder', required=True)
......
......@@ -14,6 +14,14 @@ argument:
json is written to that file in the format produced by
common.parse_common_test_results.
Optional argument:
--isolated-script-test-filter-file=[FILENAME]
points to a file containing newline-separated test names, to run just
that subset of tests. This is remapped to multiple arguments passed to
the target script.
This script is intended to be the base command invoked by the isolate,
followed by a subsequent Python script. It could be generalized to
invoke an arbitrary executable.
......@@ -24,7 +32,6 @@ import json
import os
import sys
import common
# Add src/testing/ into sys.path for importing xvfb.
......@@ -37,6 +44,9 @@ def main():
parser.add_argument(
'--isolated-script-test-output', type=str,
required=True)
parser.add_argument(
'--isolated-script-test-filter-file', type=str,
required=False)
parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
args, rest_args = parser.parse_known_args()
# Remove the chartjson extra arg until this script cares about chartjson
......@@ -48,6 +58,14 @@ def main():
rest_args.pop(index)
break
index += 1
if args.isolated_script_test_filter_file:
# This test harness doesn't yet support reading the test list from
# a file.
filter_list = common.load_filter_list(args.isolated_script_test_filter_file)
# This harness takes the test names to run as the first arguments.
# The first argument of rest_args is the script to run, so insert
# the test names after that.
rest_args = [rest_args[0]] + filter_list + rest_args[1:]
# Compatibility with gtest-based sharding.
total_shards = None
......
......@@ -14,6 +14,14 @@ argument:
json is written to that file in the format detailed here:
https://www.chromium.org/developers/the-json-test-results-format
Optional argument:
--isolated-script-test-filter-file=[FILENAME]
points to a file containing newline-separated test names, to run just
that subset of tests. This list is parsed by this harness and sent
down via the --story-filter argument.
This script is intended to be the base command invoked by the isolate,
followed by a subsequent Python script. It could be generalized to
invoke an arbitrary executable.
......@@ -48,6 +56,8 @@ def main():
'--isolated-script-test-chartjson-output', required=False)
parser.add_argument(
'--isolated-script-test-perf-output', required=False)
parser.add_argument(
'--isolated-script-test-filter-file', type=str, required=False)
parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
parser.add_argument('--output-format', action='append')
args, rest_args = parser.parse_known_args()
......@@ -87,8 +97,18 @@ def run_benchmark(args, rest_args):
json_test_results = None
results = None
cmd_args = rest_args
if args.isolated_script_test_filter_file:
# This test harness doesn't yet support reading the test list from
# a file.
filter_list = common.load_filter_list(args.isolated_script_test_filter_file)
# Need to convert this to a valid regex.
filter_regex = '(' + '|'.join(filter_list) + ')'
cmd_args = cmd_args + [
'--story-filter=' + filter_regex
]
try:
cmd = [sys.executable] + rest_args + [
cmd = [sys.executable] + cmd_args + [
'--output-dir', tempfile_dir,
'--output-format=json-test-results',
]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment