Skip to content
Snippets Groups Projects
Commit 93950d89 authored by Gert Wollny's avatar Gert Wollny
Browse files

deqp_submit: Add stats output to upload and beautify diff

* Print out the summary of changes directly from running "upload"

deqp_submit: Change diff output to not go through a "git commit"
"git diff"  "git reset" cycle and simplify the output to make it easier
readable. The output is now

testname oldState -> newState

When printed to a terminal the output is colored, and when piping, e.g. to
grep the color escape codes are not printed to make grep work properly.
To filter the result for the current changed outcome (e.g. Pass)

  deqp_submit.py diff .... | grep Pass$

v2: Correct diff for cases when a result value is not known
parent 932c4318
No related branches found
No related tags found
1 merge request!5Improve submit by adding some stats and beautify diff
......@@ -16,6 +16,20 @@
import json
import re
import sys
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
STRIKETHROUGH = '\033[9m'
UNKNOWN = '\033[4m'
class Results:
......@@ -26,6 +40,16 @@ class Results:
COMMENTS = "Comments"
VERSION = "Version"
StateColors = {
"Fail": bcolors.FAIL,
"InternalError": bcolors.WARNING,
"Incomplete": bcolors.WARNING,
"NotSupported": bcolors.STRIKETHROUGH,
"Pass": bcolors.OKGREEN,
"QualityWarning": bcolors.OKBLUE,
"Unknown": bcolors.UNKNOWN
}
def __init__(self, results=None, header=None):
if results is None:
self._results = {}
......@@ -80,6 +104,87 @@ class Results:
if len(results._header[self.ENV]) > 0:
self._header[self.ENV].update(results._header[self.ENV])
def get_key_sets(self, results):
old_keys = set(self._results.keys())
new_keys = set(results._results.keys())
shared_keys = old_keys.intersection(new_keys)
return (shared_keys,
new_keys.difference(shared_keys),
old_keys.difference(shared_keys))
def update_with_compare(self, results):
change_counts = {}
(shared_keys, new_keys, dropped_keys) = self.get_key_sets(results)
for k in shared_keys:
oldval = self._results[k]
newval = results._results[k]
if oldval != newval:
change = "{} -> {}".format(oldval, newval)
c = change_counts.setdefault(change, 0)
change_counts[change] = c + 1
if len(new_keys) > 0:
change_counts["Tests new"] = len(new_keys)
if len(dropped_keys) > 0:
change_counts["Tests dropped"] = len(dropped_keys)
self.update(results)
self._header[self.COMMENTS]["Difference"] = change_counts
return change_counts
def diff(self, results):
out_string = ""
change_counts = {}
(shared_keys, new_keys, dropped_keys) = self.get_key_sets(results)
for k in sorted(shared_keys):
oldval = self._results[k]
newval = results._results[k]
col = bcolors.ENDC
if oldval != newval:
if newval == "Fail":
col = bcolors.FAIL
elif newval == "Pass":
col = bcolors.OKGREEN
elif newval == "QualityWarning":
if "oldval" == "Pass":
col = bcolors.WARNING
else:
col = bcolors.OKBLUE
elif newval == "NotSupported":
col = bcolors.STRIKETHROUGH
elif newval == "Incomplete" or newval == "InternalError":
if oldval == "Pass" or oldval == "QualityWarning":
col = bcolors.FAIL
else:
col = bcolors.UNDERLINE
else:
col = bcolors.UNKNOWN
change = "{} -> {}".format(oldval, newval)
c = change_counts.setdefault(change, 0)
change_counts[change] = c + 1
# Only print colors when on a TTY
if sys.stdout.isatty():
oldval = self.StateColors.get(oldval, bcolors.UNKNOWN) + oldval + bcolors.ENDC
newval = self.StateColors.get(newval, bcolors.UNKNOWN) + newval + bcolors.ENDC
out_string = out_string + "{}{} {} -> {}\n".format(col, k, oldval, newval)
else:
out_string = out_string + "{} {} -> {}\n".format(k, oldval, newval)
if len(new_keys) > 0:
change_counts["Tests new"] = len(new_keys)
if len(dropped_keys) > 0:
change_counts["Tests dropped"] = len(dropped_keys)
out_string = out_string + "Change stats:\n"
for key, value in sorted(change_counts.items()):
out_string = out_string + " {}:{}\n".format(key, value)
return out_string
@staticmethod
def atoi(t):
return int(t) if t.isdigit() else t
......
deqp_submit.py 100755 → 100644
......@@ -196,51 +196,80 @@ def parse_input_file(f):
return Results.read(f)
elif '#beginTestCaseResult' in head:
return parse_input_file_xml(f)
elif head.startswith('# Fail'):
elif head.startswith('# Fail') or head.startswith('# Com') :
return parse_input_file_volt_deqp(f)
else:
return parse_input_file_deqp_runner(f)
def update_results_file(git, args):
results_dir_full = get_results_dir(git.repo)
results_path = os.path.join(results_dir_full, args.results_file)
def get_old_results(results_path):
results = None
if os.path.isfile(results_path):
with log("Reading existing results from '%s'" % results_path):
with open_file(results_path, "r") as fout:
results = parse_input_file(fout)
return results
def save_results(results_path, results):
with log("Writing results to '%s'" % results_path):
with open_file(results_path, "w") as fout:
results.write(fout)
def read_results(git, args):
results_dir_full = get_results_dir(git.repo)
results_path = os.path.join(results_dir_full, args.results_file)
results = get_old_results(results_path)
new_results = None
for f in args.file:
with log("Reading new results from '%s'" % f):
with open(f, "r") as fin:
new_results = parse_input_file(fin)
if results is not None:
results.update(new_results)
else:
results = new_results
return (results_path, results, new_results)
with log("Writing results to '%s'" % results_path):
with open_file(results_path, "w") as fout:
results.write(fout)
return results_path
def print_diff(git, args):
(results_path, results, new_results) = read_results(git, args)
if (new_results is not None) and (results is not None):
print(results.diff(new_results))
else:
log("No results to compare")
def update_results_file(git, args):
(results_path, results, new_results) = read_results(git, args)
diff_info = None
if new_results is not None:
if results is not None:
diff_info = results.update_with_compare(new_results)
else:
results = new_results
save_results(results_path, results)
return (results_path, diff_info)
def cmd_upload(args):
git = Git(git_repo, git_url, args.email)
try:
results_path = update_results_file(git, args)
(results_path, diff_info) = update_results_file(git, args)
try:
with log("Commiting results to git"):
git.commit(results_path, args.message)
if not args.commit_only:
with log("Uploading results to gitlab"):
git.upload()
if diff_info is not None:
print("Change stats:")
for key, value in sorted(diff_info.items()):
print(" {}:{}".format(key, value))
except:
git.reset_hard()
raise
......@@ -248,17 +277,7 @@ def cmd_upload(args):
def cmd_diff(args):
git = Git(git_repo, git_url)
try:
results_path = update_results_file(git, args)
git.add(results_path)
for l in git.diff_iter():
print(l, end='')
finally:
with log("Restoring local git state"):
git.reset_hard()
print_diff(git, args)
if __name__ == '__main__':
......
......@@ -41,6 +41,24 @@ Test4 Unsupported
Test5 Unsupported
"""
input_unknown = """{
"Summary" : {
"CompatibilityWarning" : 1,
"Fail" : 1,
"Pass" : 2,
"Unsupported" : 1
},
"Environment" : {
"host-mesa" : "18.2.0-git111112"
}
}
Test1 Fail
Test2 Pass
Test3 Pass
Test4 Unsupported
Test5 CompatibilityWarning
"""
input2 = """{
"Summary" : {
......@@ -59,6 +77,11 @@ Test4 Unsupported
Test6 Fail
"""
diff_1_unknown = """\x1b[9mTest5 \x1b[9mUnsupported\x1b[0m -> \x1b[9mCompatibilityWarning\x1b[0m
Change stats:
Unsupported -> CompatibilityWarning:1
"""
update_1_with_2 = """{
"Comments": {},
"Environment": {
......@@ -79,6 +102,26 @@ Test5 Unsupported
Test6 Fail
"""
update_1_with_unknown = """{
"Comments": {},
"Environment": {
"host-mesa": "18.2.0-git111112"
},
"Summary": {
"CompatibilityWarning": 1,
"Fail": 1,
"Pass": 2,
"Unsupported": 1
},
"Version": "deqp 1"
}
Test1 Fail
Test2 Pass
Test3 Pass
Test4 Unsupported
Test5 CompatibilityWarning
"""
class ReadJson(unittest.TestCase):
......@@ -133,6 +176,29 @@ class ReadJson(unittest.TestCase):
self.assertEqual(output_file.getvalue(), update_1_with_2)
def test_update_1_with_unknown_output(self):
input_file = io.StringIO(input)
data1 = Results.read(input_file)
input_file = io.StringIO(input_unknown)
data2 = Results.read(input_file)
data1.update(data2)
output_file = io.StringIO()
data1.write(output_file)
self.assertEqual(output_file.getvalue(), update_1_with_unknown)
def test_print_diff_with_unknown(self):
input_file = io.StringIO(input)
data1 = Results.read(input_file)
input_file = io.StringIO(input_unknown)
data2 = Results.read(input_file)
self.assertEqual(data1.diff(data2), diff_1_unknown)
if __name__ == '__main__':
unittest.main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment