diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d324..cb7382604f80bd2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -6,7 +6,7 @@ import sysconfig import time import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -284,7 +284,8 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + github_annotations=bool(self.github_summary)) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -395,6 +396,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + result.print_github_annotation(runtests) return result @@ -480,6 +482,25 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.write_github_summary() + + def write_github_summary(self) -> None: + # Tests which failed in the last run (the re-run, if any) + failed = self.results.rerun_results + # Only write a summary on failure: a CI run has many test jobs + exitcode = self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + if not self.github_summary or not failed or not exitcode: + return + ncases = sum(len(result.failed_test_cases) for result in failed) + with open(self.github_summary, "a", encoding="utf-8") as fp: + fp.write(f"## {decolor(self.get_state())}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed\n\n") + for result in failed: + # Expand the tracebacks of up to 5 test cases + fp.write(result.format_github_summary(expand=ncases <= 5)) + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( @@ -517,6 +538,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + # Only annotate the failure that results in job failure. + # Ex: When reruning failed tests, only annotate the the final run. + github_annotations=(bool(self.github_summary) and + not (self.want_rerun and not self.python_cmd)), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -764,6 +789,8 @@ def _init(self): strip_py_suffix(self.cmdline_args) + self.github_summary = os.environ.pop("GITHUB_STEP_SUMMARY", None) + self._tmp_dir = get_temp_dir(self._tmp_dir) @property diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c5..8f59fcd124900c0 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,11 +1,16 @@ import dataclasses +import html import json -from _colorize import get_colors # type: ignore[import-not-found] +import pathlib +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import Any +from .findtests import findtestdir +from .runtests import RunTests from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning) + format_duration, normalize_test_name, print_warning, + print_github_annotation) @dataclasses.dataclass(slots=True) @@ -103,6 +108,9 @@ class TestResult: # short descriptions of how the test altered the execution environment env_changed_reasons: list[str] | None = None + # test file, set once the test module is imported + filename: str | None = None + def is_failed(self, fail_env_changed: bool) -> bool: if self.state == State.ENV_CHANGED: return fail_env_changed @@ -178,6 +186,48 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + @property + def failed_test_cases(self) -> list[tuple[str, str]]: + return (self.errors or []) + (self.failures or []) + + def format_github_summary(self, expand: bool) -> str: + """Format the Markdown section of the GitHub Actions job summary.""" + blocks = [f"### {decolor(str(self))}"] + if self.env_changed_reasons: + blocks.append("\n".join(f"- {reason}" + for reason in self.env_changed_reasons)) + for name, traceback in self.failed_test_cases: + # Only default open short tracebacks. + is_open = expand and traceback.count("\n") < 30 + blocks += [f"" + f"{html.escape(name)}", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + ""] + + # Separate Markdown blocks with an empty line + return "".join(f"{block}\n\n" for block in blocks) + + def print_github_annotation(self, runtests: RunTests) -> None: + """Annotate a failed test file without failed test cases.""" + if (not runtests.github_annotations + or not self.is_failed(runtests.fail_env_changed) + or self.failed_test_cases): + return + filename = self.filename + if not filename: + # The worker crashed or timed out before reporting the test file: + # convert python import test name to relative file path. + # Ex: "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(runtests.test_dir), + *name.split(".")) + if path.with_suffix(".py").exists(): + filename = str(path.with_suffix(".py")) + else: + filename = str(path / "__init__.py") + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + print_github_annotation(self.test_name, message, filename) + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba31..3327269ae1c622c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,6 +624,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a604..541a8146001cabf 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e5..8216541e454f82d 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,6 +12,7 @@ from .filter import set_match_tests from .runtests import RunTests +lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,9 +113,9 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index d7a1c6bea202a65..3aa4ffd00f8f8fa 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -153,6 +153,7 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None: test_name = result.test_name module_name = abs_module_name(test_name, runtests.test_dir) test_mod = importlib.import_module(module_name) + result.filename = test_mod.__file__ if hasattr(test_mod, "test_main"): # https://github.com/python/cpython/issues/89392 diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb6..79add3d39b373ab 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,11 @@ import traceback import unittest from test import support -from test.libregrtest.utils import sanitize_xml +from test.libregrtest.utils import print_github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +131,14 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + for test, err in errors: + super().printErrorList(flavour, [(test, err)]) + # Annotate just after the failure report, so that the annotation + # links to the report in the job log + if self.GITHUB_ANNOTATIONS: + print_github_annotation(str(test), err, stream=self.stream) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -146,6 +155,11 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) + # Failures are not reported in quiet mode: annotate them at the end + if self.result.GITHUB_ANNOTATIONS: + for test, err in self.result.errors + self.result.failures: + print_github_annotation(str(test), err, + stream=self.result.stream) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 83e0575619c99a0..6034fa50b008177 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -3,6 +3,7 @@ import locale import math import os.path +import pathlib import platform import random import re @@ -13,6 +14,7 @@ import tempfile import textwrap import types +from _colorize import decolor # type: ignore[import-not-found] from collections.abc import Callable _winapi: types.ModuleType | None try: @@ -140,6 +142,44 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +# Escape the data and the properties of GitHub Actions workflow commands +GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) +GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", + ",": "%2C"}) + + +def print_github_annotation(title: str, message: str, + filename: str | None = None, + stream=None) -> None: + """Print a GitHub Actions error annotation for the failure.""" + message = decolor(message) + + # Try to get a file + line number from backtrace. + line = None + if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, + re.MULTILINE): + test_frames = [frame for frame in frames + if pathlib.Path(frame[0]).name.startswith('test_')] + filename, line = test_frames[-1] if test_frames else frames[0] + if not filename: + raise ValueError(f"missing filename to annotate {title!r}") + + # Normalize stdlib path to be relative inside the repository checkout. + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + props = {"file": filename, "line": line, "title": title} + # Only keep the exception: strip the traceback frames, each one a + # ' File' line followed by its indented source lines + message = re.split(r'^ File .*\n(?: .*\n)*', message, + flags=re.MULTILINE)[-1].strip() + props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" + for key, value in props.items() if value) + print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", + file=stream, flush=True) + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index dcdf7bdce03b800..2214e7408aa5603 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -4,6 +4,7 @@ from test.support import os_helper import os +import sys class BoolTest(unittest.TestCase): @@ -25,6 +26,11 @@ def test_repr(self): self.assertIs(eval(repr(True)), True) def test_str(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.assertEqual( + str(True), + 'Yes') self.assertEqual(str(False), 'False') self.assertEqual(str(True), 'True') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 7bff16ddf2155e9..d01368254f6701e 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import sys import unittest.mock from io import StringIO from collections import OrderedDict @@ -9,6 +10,9 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.loads('{"bad": }') self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a1250c668223e7f..40dc0d9acc75fff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -513,6 +513,11 @@ def testFabs(self): def testFloor(self): self.assertRaises(TypeError, math.floor) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + for x in (0.5, 1.5): + with self.subTest(x=x): + self.assertEqual(math.floor(x), x) self.assertEqual(int, type(math.floor(0.5))) self.assertEqual(math.floor(0.5), 0) self.assertEqual(math.floor(1.0), 1) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 81b3043eb7e75bc..04321264e6f121d 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -4080,7 +4080,8 @@ def test_memfd_create(self): self.assertFalse(os.get_inheritable(fd)) with open(fd, "wb", closefd=False) as f: f.write(b'memfd_create') - self.assertEqual(f.tell(), 12) + # XXX: Intentional failure to test GitHub Actions job summaries + self.assertEqual(f.tell(), 13) fd2 = os.memfd_create("Hi") self.addCleanup(os.close, fd2) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 53f0a2863dc4e45..f445eb2b6a0760c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1639,6 +1639,82 @@ def test_env_changed(self): success=True), stats=2) + def test_github_actions(self): + # Failures are annotated in the GitHub Actions job log at the failing + # line of the test file, and listed in the job summary + cases = self.create_test('github_cases', textwrap.dedent(""" + import doctest, sys, unittest + + def load_tests(loader, tests, pattern): + tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) + return tests + + def doctest_fail(): + ''' + >>> 1 + 1 + 3 + ''' + + class Tests(unittest.TestCase): + def test_subtest(self): + with self.subTest(x=1): + self.fail() + """)) + crash = self.create_test('github_crash', textwrap.dedent(""" + import os, unittest + + class Tests(unittest.TestCase): + def test_crash(self): + os._exit(1) + """)) + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) + env.pop('SOURCE_DATE_EPOCH', None) + # A single worker without --randomize runs the tests in order + output = self.run_tests('-j1', '--rerun', cases, crash, + env=env, exitcode=EXITCODE_BAD_TEST) + + def unescape(text): + for char in '\r\n:,%': + text = text.replace(f'%{ord(char):02X}', char) + return text + + annotations = [] + for props, message in re.findall(r'^::error (.*?)::(.*)$', output, + re.MULTILINE): + props = dict(prop.split('=', 1) for prop in props.split(',')) + annotations.append(({key: unescape(value) + for key, value in props.items()}, + unescape(message))) + # One annotation per failure of the re-run, not of the first run + cases_file = os.path.join(self.tmptestdir, f'{cases}.py') + crash_file = os.path.join(self.tmptestdir, f'{crash}.py') + self.assertEqual(annotations, [ + ({'file': cases_file, 'line': '17', + 'title': f'test_subtest ({cases}.Tests.test_subtest) (x=1)'}, + 'AssertionError: None'), + ({'file': cases_file, 'line': '10', + 'title': f'doctest_fail ({cases}) [0]'}, + 'AssertionError: Failed example:\n' + ' 1 + 1\n' + 'Expected:\n' + ' 3\n' + 'Got:\n' + ' 2'), + ({'file': crash_file, 'title': crash}, + f'{crash} worker non-zero exit code'), + ], output) + with open(filename, encoding='utf-8') as fp: + summary = fp.read() + self.assertIn('## FAILURE: 2 test files and 2 test cases failed', + summary) + + # No annotation and no job summary when all tests pass + os_helper.unlink(filename) + output = self.run_tests(self.create_test(), env=env) + self.assertNotIn('::error', output) + self.assertFalse(os.path.exists(filename)) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index aca1f427656bb50..c1d670c978c9702 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -8,6 +8,8 @@ # $Id$ # +import os +import sys import unittest from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten @@ -66,6 +68,9 @@ def test_simple(self): self.check_wrap(text, 80, [text]) def test_empty_string(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + os.environ["GHA_ANNOTATION_TEST"] = "1" # Check that wrapping the empty string returns an empty list. self.check_wrap("", 6, []) self.check_wrap("", 6, [], drop_whitespace=False) diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 000000000000000..a1c9e648e5c744e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,3 @@ +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary.