diff --git a/deqp/results.py b/deqp/results.py index f6ccb9a28a3e43e7946b43dd3b53e0241bf740e4..0d1fabdae6bfff8d79b3120decc99a55bc1000ae 100644 --- a/deqp/results.py +++ b/deqp/results.py @@ -16,6 +16,20 @@ import json import re +import sys + + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + STRIKETHROUGH = '\033[9m' + UNKNOWN = '\033[4m' class Results: @@ -26,6 +40,16 @@ class Results: COMMENTS = "Comments" VERSION = "Version" + StateColors = { + "Fail": bcolors.FAIL, + "InternalError": bcolors.WARNING, + "Incomplete": bcolors.WARNING, + "NotSupported": bcolors.STRIKETHROUGH, + "Pass": bcolors.OKGREEN, + "QualityWarning": bcolors.OKBLUE, + "Unknown": bcolors.UNKNOWN + } + def __init__(self, results=None, header=None): if results is None: self._results = {} @@ -80,6 +104,87 @@ class Results: if len(results._header[self.ENV]) > 0: self._header[self.ENV].update(results._header[self.ENV]) + def get_key_sets(self, results): + old_keys = set(self._results.keys()) + new_keys = set(results._results.keys()) + shared_keys = old_keys.intersection(new_keys) + return (shared_keys, + new_keys.difference(shared_keys), + old_keys.difference(shared_keys)) + + def update_with_compare(self, results): + change_counts = {} + (shared_keys, new_keys, dropped_keys) = self.get_key_sets(results) + + for k in shared_keys: + oldval = self._results[k] + newval = results._results[k] + if oldval != newval: + change = "{} -> {}".format(oldval, newval) + + c = change_counts.setdefault(change, 0) + change_counts[change] = c + 1 + + if len(new_keys) > 0: + change_counts["Tests new"] = len(new_keys) + if len(dropped_keys) > 0: + change_counts["Tests dropped"] = len(dropped_keys) + + self.update(results) + self._header[self.COMMENTS]["Difference"] = change_counts + return change_counts + + def diff(self, results): + out_string = "" + change_counts = {} + (shared_keys, new_keys, dropped_keys) = self.get_key_sets(results) + + for k in sorted(shared_keys): + oldval = self._results[k] + newval = results._results[k] + col = bcolors.ENDC + if oldval != newval: + if newval == "Fail": + col = bcolors.FAIL + elif newval == "Pass": + col = bcolors.OKGREEN + elif newval == "QualityWarning": + if "oldval" == "Pass": + col = bcolors.WARNING + else: + col = bcolors.OKBLUE + elif newval == "NotSupported": + col = bcolors.STRIKETHROUGH + elif newval == "Incomplete" or newval == "InternalError": + if oldval == "Pass" or oldval == "QualityWarning": + col = bcolors.FAIL + else: + col = bcolors.UNDERLINE + else: + col = bcolors.UNKNOWN + + change = "{} -> {}".format(oldval, newval) + c = change_counts.setdefault(change, 0) + change_counts[change] = c + 1 + + # Only print colors when on a TTY + if sys.stdout.isatty(): + oldval = self.StateColors.get(oldval, bcolors.UNKNOWN) + oldval + bcolors.ENDC + newval = self.StateColors.get(newval, bcolors.UNKNOWN) + newval + bcolors.ENDC + out_string = out_string + "{}{} {} -> {}\n".format(col, k, oldval, newval) + else: + out_string = out_string + "{} {} -> {}\n".format(k, oldval, newval) + + if len(new_keys) > 0: + change_counts["Tests new"] = len(new_keys) + if len(dropped_keys) > 0: + change_counts["Tests dropped"] = len(dropped_keys) + + out_string = out_string + "Change stats:\n" + for key, value in sorted(change_counts.items()): + out_string = out_string + " {}:{}\n".format(key, value) + return out_string + @staticmethod def atoi(t): return int(t) if t.isdigit() else t diff --git a/deqp_submit.py b/deqp_submit.py old mode 100755 new mode 100644 index 632c6730d5ca157918785c797fbce1ed7c02efeb..1ec746be2e1a402241fedb72de9dee287ce164e6 --- a/deqp_submit.py +++ b/deqp_submit.py @@ -196,51 +196,80 @@ def parse_input_file(f): return Results.read(f) elif '#beginTestCaseResult' in head: return parse_input_file_xml(f) - elif head.startswith('# Fail'): + elif head.startswith('# Fail') or head.startswith('# Com') : return parse_input_file_volt_deqp(f) else: return parse_input_file_deqp_runner(f) -def update_results_file(git, args): - results_dir_full = get_results_dir(git.repo) - results_path = os.path.join(results_dir_full, args.results_file) - +def get_old_results(results_path): results = None - if os.path.isfile(results_path): with log("Reading existing results from '%s'" % results_path): with open_file(results_path, "r") as fout: results = parse_input_file(fout) + return results + + +def save_results(results_path, results): + with log("Writing results to '%s'" % results_path): + with open_file(results_path, "w") as fout: + results.write(fout) + + +def read_results(git, args): + results_dir_full = get_results_dir(git.repo) + results_path = os.path.join(results_dir_full, args.results_file) + + results = get_old_results(results_path) + new_results = None for f in args.file: with log("Reading new results from '%s'" % f): with open(f, "r") as fin: new_results = parse_input_file(fin) - if results is not None: - results.update(new_results) - else: - results = new_results + return (results_path, results, new_results) - with log("Writing results to '%s'" % results_path): - with open_file(results_path, "w") as fout: - results.write(fout) - return results_path +def print_diff(git, args): + (results_path, results, new_results) = read_results(git, args) + if (new_results is not None) and (results is not None): + print(results.diff(new_results)) + else: + log("No results to compare") + + +def update_results_file(git, args): + (results_path, results, new_results) = read_results(git, args) + diff_info = None + + if new_results is not None: + if results is not None: + diff_info = results.update_with_compare(new_results) + else: + results = new_results + save_results(results_path, results) + + return (results_path, diff_info) def cmd_upload(args): git = Git(git_repo, git_url, args.email) - try: - results_path = update_results_file(git, args) + (results_path, diff_info) = update_results_file(git, args) + try: with log("Commiting results to git"): git.commit(results_path, args.message) if not args.commit_only: with log("Uploading results to gitlab"): git.upload() + + if diff_info is not None: + print("Change stats:") + for key, value in sorted(diff_info.items()): + print(" {}:{}".format(key, value)) except: git.reset_hard() raise @@ -248,17 +277,7 @@ def cmd_upload(args): def cmd_diff(args): git = Git(git_repo, git_url) - - try: - results_path = update_results_file(git, args) - - git.add(results_path) - - for l in git.diff_iter(): - print(l, end='') - finally: - with log("Restoring local git state"): - git.reset_hard() + print_diff(git, args) if __name__ == '__main__': diff --git a/tests/test_results.py b/tests/test_results.py index 50e3db6e76b0f1b113e036a5aad1c99455b2cbb9..30f47392b5e8330808720aafce020ec4e2758d39 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -41,6 +41,24 @@ Test4 Unsupported Test5 Unsupported """ +input_unknown = """{ + "Summary" : { + "CompatibilityWarning" : 1, + "Fail" : 1, + "Pass" : 2, + "Unsupported" : 1 + }, + "Environment" : { + "host-mesa" : "18.2.0-git111112" + } +} +Test1 Fail +Test2 Pass +Test3 Pass +Test4 Unsupported +Test5 CompatibilityWarning +""" + input2 = """{ "Summary" : { @@ -59,6 +77,11 @@ Test4 Unsupported Test6 Fail """ +diff_1_unknown = """\x1b[9mTest5 \x1b[9mUnsupported\x1b[0m -> \x1b[9mCompatibilityWarning\x1b[0m +Change stats: + Unsupported -> CompatibilityWarning:1 +""" + update_1_with_2 = """{ "Comments": {}, "Environment": { @@ -79,6 +102,26 @@ Test5 Unsupported Test6 Fail """ +update_1_with_unknown = """{ + "Comments": {}, + "Environment": { + "host-mesa": "18.2.0-git111112" + }, + "Summary": { + "CompatibilityWarning": 1, + "Fail": 1, + "Pass": 2, + "Unsupported": 1 + }, + "Version": "deqp 1" +} +Test1 Fail +Test2 Pass +Test3 Pass +Test4 Unsupported +Test5 CompatibilityWarning +""" + class ReadJson(unittest.TestCase): @@ -133,6 +176,29 @@ class ReadJson(unittest.TestCase): self.assertEqual(output_file.getvalue(), update_1_with_2) + def test_update_1_with_unknown_output(self): + input_file = io.StringIO(input) + data1 = Results.read(input_file) + + input_file = io.StringIO(input_unknown) + data2 = Results.read(input_file) + + data1.update(data2) + + output_file = io.StringIO() + data1.write(output_file) + + self.assertEqual(output_file.getvalue(), update_1_with_unknown) + + def test_print_diff_with_unknown(self): + input_file = io.StringIO(input) + data1 = Results.read(input_file) + + input_file = io.StringIO(input_unknown) + data2 = Results.read(input_file) + + self.assertEqual(data1.diff(data2), diff_1_unknown) + if __name__ == '__main__': unittest.main()