run_telemetry_benchmark_as_googletest.py 5.55 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
#!/usr/bin/env python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runs an isolate bundled Telemetry benchmark.

This script attempts to emulate the contract of gtest-style tests
invoked via recipes. The main contract is that the caller passes the
argument:

  --isolated-script-test-output=[FILENAME]

14 15
json is written to that file in the format detailed here:
https://www.chromium.org/developers/the-json-test-results-format
16

17 18 19 20 21 22 23 24
Optional argument:

  --isolated-script-test-filter-file=[FILENAME]

points to a file containing newline-separated test names, to run just
that subset of tests. This list is parsed by this harness and sent
down via the --story-filter argument.

25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
This script is intended to be the base command invoked by the isolate,
followed by a subsequent Python script. It could be generalized to
invoke an arbitrary executable.
"""

import argparse
import json
import os
import shutil
import sys
import tempfile
import traceback

import common

# Add src/testing/ into sys.path for importing xvfb.
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import xvfb

44 45 46 47 48
# Unfortunately we need to copy these variables from ../test_env.py.
# Importing it and using its get_sandbox_env breaks test runs on Linux
# (it seems to unset DISPLAY).
CHROME_SANDBOX_ENV = 'CHROME_DEVEL_SANDBOX'
CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
49 50 51 52 53 54

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--isolated-script-test-output', type=argparse.FileType('w'),
      required=True)
55
  parser.add_argument(
56
      '--isolated-script-test-chartjson-output', required=False)
57 58
  parser.add_argument(
      '--isolated-script-test-perf-output', required=False)
59 60
  parser.add_argument(
      '--isolated-script-test-filter-file', type=str, required=False)
61
  parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
62
  parser.add_argument('--output-format', action='append')
63
  args, rest_args = parser.parse_known_args()
64 65
  for output_format in args.output_format:
    rest_args.append('--output-format=' + output_format)
66

67
  rc, perf_results, json_test_results = run_benchmark(args, rest_args)
68

69
  if perf_results:
70 71 72 73 74 75 76 77
    if args.isolated_script_test_perf_output:
      filename = args.isolated_script_test_perf_output
    elif args.isolated_script_test_chartjson_output:
      filename = args.isolated_script_test_chartjson_output
    else:
      filename = None

    if filename is not None:
78 79
      with open(filename, 'w') as perf_results_output_file:
        json.dump(perf_results, perf_results_output_file)
80 81 82 83 84 85

  json.dump(json_test_results, args.isolated_script_test_output)

  return rc

def run_benchmark(args, rest_args):
86
  env = os.environ.copy()
87 88 89 90
  # Assume we want to set up the sandbox environment variables all the
  # time; doing so is harmless on non-Linux platforms and is needed
  # all the time on Linux.
  env[CHROME_SANDBOX_ENV] = CHROME_SANDBOX_PATH
91 92
  tempfile_dir = tempfile.mkdtemp('telemetry')
  valid = True
93
  num_failures = 0
94
  histogram_results_present = 'histograms' in args.output_format
95
  chartjson_results_present = 'chartjson' in args.output_format
96
  perf_results = None
97 98 99
  json_test_results = None

  results = None
100 101 102 103 104 105 106 107 108 109
  cmd_args = rest_args
  if args.isolated_script_test_filter_file:
    # This test harness doesn't yet support reading the test list from
    # a file.
    filter_list = common.load_filter_list(args.isolated_script_test_filter_file)
    # Need to convert this to a valid regex.
    filter_regex = '(' + '|'.join(filter_list) + ')'
    cmd_args = cmd_args + [
      '--story-filter=' + filter_regex
    ]
110
  try:
111
    cmd = [sys.executable] + cmd_args + [
112 113 114 115 116 117 118 119 120 121 122
      '--output-dir', tempfile_dir,
      '--output-format=json-test-results',
    ]
    if args.xvfb:
      rc = xvfb.run_executable(cmd, env)
    else:
      rc = common.run_command(cmd, env=env)

    # If we have also output chartjson read it in and return it.
    # results-chart.json is the file name output by telemetry when the
    # chartjson output format is included
123 124 125 126 127 128 129 130 131 132
    if histogram_results_present:
      tempfile_name = os.path.join(tempfile_dir, 'histograms.json')
    elif chartjson_results_present:
      tempfile_name = os.path.join(tempfile_dir, 'results-chart.json')
    else:
      tempfile_name = None

    if tempfile_name is not None:
      with open(tempfile_name) as f:
        perf_results = json.load(f)
133

134 135 136 137 138 139 140
    # test-results.json is the file name output by telemetry when the
    # json-test-results format is included
    tempfile_name = os.path.join(tempfile_dir, 'test-results.json')
    with open(tempfile_name) as f:
      json_test_results = json.load(f)
    num_failures = json_test_results['num_failures_by_type'].get('FAIL', 0)
    valid = bool(rc == 0 or num_failures != 0)
141

142 143 144 145 146 147
  except Exception:
    traceback.print_exc()
    if results:
      print 'results, which possibly caused exception: %s' % json.dumps(
          results, indent=2)
    valid = False
148
  finally:
149 150
    shutil.rmtree(tempfile_dir)

151
  if not valid and num_failures == 0:
152 153 154
    if rc == 0:
      rc = 1  # Signal an abnormal exit.

155
  return rc, perf_results, json_test_results
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172


# This is not really a "script test" so does not need to manually add
# any additional compile targets.
def main_compile_targets(args):
  json.dump([], args.output)


if __name__ == '__main__':
  # Conform minimally to the protocol defined by ScriptTest.
  if 'compile_targets' in sys.argv:
    funcs = {
      'run': None,
      'compile_targets': main_compile_targets,
    }
    sys.exit(common.run_script(sys.argv[1:], funcs))
  sys.exit(main())