From 01abb1a24705b7ab95a800118e973a311cf4212c Mon Sep 17 00:00:00 2001 From: Allard Hendriksen Date: Fri, 8 Sep 2023 08:28:09 +0200 Subject: [PATCH] Add support for tests that fail at runtime (#418) That is compilation must be successful, but running the test should return a non-zero exit code. This is useful for testing that code correctly asserts. --- libcudacxx/.upstream-tests/test/lit.cfg | 2 +- .../utils/libcudacxx/test/format.py | 19 ++++++++++++---- libcudacxx/libcxx/utils/libcxx/test/format.py | 22 ++++++++++++++----- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/libcudacxx/.upstream-tests/test/lit.cfg b/libcudacxx/.upstream-tests/test/lit.cfg index eaa03d41e8..04d78bd100 100644 --- a/libcudacxx/.upstream-tests/test/lit.cfg +++ b/libcudacxx/.upstream-tests/test/lit.cfg @@ -15,7 +15,7 @@ if 'PYLINT_IMPORT' in os.environ: config.name = 'libcu++' # suffixes: A list of file extensions to treat as test files. -config.suffixes = ['.pass.cpp', '.fail.cpp', '.sh.cpp'] +config.suffixes = ['.pass.cpp', '.fail.cpp', '.runfail.cpp', '.sh.cpp'] # test_source_root: The root path where tests are located. config.test_source_root = os.path.dirname(__file__) diff --git a/libcudacxx/.upstream-tests/utils/libcudacxx/test/format.py b/libcudacxx/.upstream-tests/utils/libcudacxx/test/format.py index 7697b421ac..f2b6f478fb 100644 --- a/libcudacxx/.upstream-tests/utils/libcudacxx/test/format.py +++ b/libcudacxx/.upstream-tests/utils/libcudacxx/test/format.py @@ -29,6 +29,8 @@ class LibcxxTestFormat(object): FOO.pass.cpp - Executable test which should compile, run, and exit with code 0. FOO.fail.cpp - Negative test case which is expected to fail compilation. + FOO.runfail.cpp - Negative test case which is expected to compile, run, + and exit with non-zero exit code. FOO.sh.cpp - A test that uses LIT's ShTest format. """ @@ -88,6 +90,7 @@ def _execute(self, test, lit_config): is_sh_test = name_root.endswith('.sh') is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm') is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm') + is_runfail_test = name.endswith('.runfail.cpp') or name.endswith('.runfail.mm') is_objcxx_test = name.endswith('.mm') is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \ name.endswith('.arc.fail.mm') @@ -166,6 +169,10 @@ def _execute(self, test, lit_config): elif is_pass_test: return self._evaluate_pass_test(test, tmpBase, lit_config, test_cxx, parsers) + elif is_runfail_test: + return self._evaluate_pass_test(test, tmpBase, lit_config, + test_cxx, parsers, + run_should_pass=False) else: # No other test type is supported assert False @@ -174,7 +181,7 @@ def _clean(self, exec_path): # pylint: disable=no-self-use libcudacxx.util.cleanFile(exec_path) def _evaluate_pass_test(self, test, tmpBase, lit_config, - test_cxx, parsers): + test_cxx, parsers, run_should_pass=True): execDir = os.path.dirname(test.getExecPath()) source_path = test.getSourcePath() exec_path = tmpBase + '.exe' @@ -210,14 +217,18 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config, env) report = "Compiled With: '%s'\n" % ' '.join(compile_cmd) report += libcudacxx.util.makeReport(cmd, out, err, rc) - if rc == 0: + result_expected = (rc == 0) == run_should_pass + if result_expected: res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS return lit.Test.Result(res, report) # Rarely devices are unavailable, so just restart the test to avoid false negatives. elif rc != 0 and "cudaErrorDevicesUnavailable" in out and max_retry <= 5: max_retry += 1 - elif rc != 0 and retry_count + 1 >= max_retry: - report += "Compiled test failed unexpectedly!" + elif retry_count + 1 == max_retry: + if run_should_pass: + report += "Compiled test failed unexpectedly!" + else: + report += "Compiled test succeeded unexpectedly!" return lit.Test.Result(lit.Test.FAIL, report) assert False # Unreachable diff --git a/libcudacxx/libcxx/utils/libcxx/test/format.py b/libcudacxx/libcxx/utils/libcxx/test/format.py index 31c54a4f65..bc57a4dda7 100644 --- a/libcudacxx/libcxx/utils/libcxx/test/format.py +++ b/libcudacxx/libcxx/utils/libcxx/test/format.py @@ -29,6 +29,8 @@ class LibcxxTestFormat(object): FOO.pass.cpp - Executable test which should compile, run, and exit with code 0. FOO.fail.cpp - Negative test case which is expected to fail compilation. + FOO.runfail.cpp - Negative test case which is expected to compile, run, + and exit with non-zero exit code. FOO.sh.cpp - A test that uses LIT's ShTest format. """ @@ -88,6 +90,7 @@ def _execute(self, test, lit_config): is_sh_test = name_root.endswith('.sh') is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm') is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm') + is_runfail_test = name.endswith('.runfail.cpp') or name.endswith('.runfail.mm') is_objcxx_test = name.endswith('.mm') is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \ name.endswith('.arc.fail.mm') @@ -163,15 +166,18 @@ def _execute(self, test, lit_config): elif is_pass_test: return self._evaluate_pass_test(test, tmpBase, lit_config, test_cxx, parsers) + elif is_runfail_test: + return self._evaluate_pass_test(test, tmpBase, lit_config, + test_cxx, parsers, + run_should_pass=False) else: - # No other test type is supported - assert False + assert False and "no other test" # No other test type is supported def _clean(self, exec_path): # pylint: disable=no-self-use libcxx.util.cleanFile(exec_path) def _evaluate_pass_test(self, test, tmpBase, lit_config, - test_cxx, parsers): + test_cxx, parsers, run_should_pass=True): execDir = os.path.dirname(test.getExecPath()) source_path = test.getSourcePath() exec_path = tmpBase + '.exe' @@ -207,11 +213,15 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config, env) report = "Compiled With: '%s'\n" % ' '.join(compile_cmd) report += libcxx.util.makeReport(cmd, out, err, rc) - if rc == 0: + result_expected = (rc == 0) == run_should_pass + if result_expected: res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS return lit.Test.Result(res, report) - elif rc != 0 and retry_count + 1 == max_retry: - report += "Compiled test failed unexpectedly!" + elif retry_count + 1 == max_retry: + if run_should_pass: + report += "Compiled test failed unexpectedly!" + else: + report += "Compiled test succeeded unexpectedly!" return lit.Test.Result(lit.Test.FAIL, report) assert False # Unreachable