Skip to content

Commit

Permalink
Dedup stdout / stderr test output
Browse files Browse the repository at this point in the history
Since we typically run many iterations of tests, we end up with a long
log of messages. Most of them are likely the same. We now dedup this
list, for example:

  stdout_capture:          FLAKY FAIL:  272 runs in 50.090ms
    testing.NotEqualError: Expected equal values but they are non-equal. A: 1 B: 2
    STDOUT:
      == 140 test runs with this output:
      Random number: 2

      == 132 test runs with this output:
      Random number: 1
  • Loading branch information
Kristian Larsson committed Jan 27, 2025
1 parent b134a6a commit 1588146
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 4 deletions.
53 changes: 49 additions & 4 deletions base/src/testing.act
Original file line number Diff line number Diff line change
Expand Up @@ -1052,19 +1052,64 @@ class ProjectTestResults(object):
return True
return False

def dedup(buf: str) -> dict[str, set[int]]:
# Initialize result dictionary
result = {}

# Split buffer into individual test iterations
iterations = buf.split("== Running test, iteration: ")

# Clean up iterations - remove empty entries
cleaned_iterations = []
for it in iterations:
if it.strip():
cleaned_iterations.append(it.strip())

# Process each iteration
for iteration in cleaned_iterations:
# Extract iteration number and content
lines = iteration.split('\n', 1)
iteration_num = int(lines[0])
content = lines[1].strip()

# Add to result dictionary
if content not in result:
result[content] = set()
result[content].add(iteration_num)

return result

std_out = tinfo.std_out
if std_out != None and test_output(std_out):
print(" STDOUT:")
self.printed_lines += 1
for line in std_out.strip().splitlines():
print(" " + line)
chunks = dedup(std_out.strip())
for chunk, test_runs in chunks.items():
if len(chunks) == 1:
pass
else:
print(" == %d test runs with this output:" % len(test_runs))
self.printed_lines += 1
for line in chunk.splitlines():
print(" " + line)
self.printed_lines += 1
print("")
self.printed_lines += 1
std_err = tinfo.std_err
if std_err != None and test_output(std_err):
print(" STDERR:")
self.printed_lines += 1
for line in std_err.strip().splitlines():
print(" " + line)
chunks = dedup(std_err.strip())
for chunk, test_runs in chunks.items():
if len(chunks) == 1:
pass
else:
print(" == %d test runs with this output:" % len(test_runs))
self.printed_lines += 1
for line in chunk.splitlines():
print(" " + line)
self.printed_lines += 1
print("")
self.printed_lines += 1

print("")
Expand Down
7 changes: 7 additions & 0 deletions test/test_testing/src/errs.act
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import testing

import random

def _test_equal_failure():
"""Demonstrate assertEqual failure with different strings"""
testing.assertEqual("foo", "bar", "Strings should be equal")
Expand Down Expand Up @@ -60,3 +62,8 @@ def _test_performance_failure():
def _test_generic_error():
"""Demonstrate generic error"""
testing.error("This is a generic test error")

def _test_stdout_capture():
r = random.randint(1, 3)
print("Random number: %d" % r)
testing.assertEqual(1, r)

0 comments on commit 1588146

Please sign in to comment.