diff --git a/scripts/check.py b/scripts/check.py index 171981f3e..cebdc9c3c 100755 --- a/scripts/check.py +++ b/scripts/check.py @@ -21,6 +21,7 @@ parser.add_argument('-d', '--dry-run', '-N', action='store_true', help='dry run') parser.add_argument('--regex-include', type=str, nargs='?', default="", help='Tests to be run must also match this regex (known failures will still be excluded)') parser.add_argument('--regex-exclude', type=str, nargs='?', default="", help='Specifically exclude tests that match this regex (known failures will still be excluded)') +parser.add_argument('--test-mode-failures', type=str, choices=['exclude', 'include'], default='exclude', help='Control how to handle test failures: exclude (default) or include') # --total-runtime cannot be used with --num-tries group = parser.add_mutually_exclusive_group() @@ -112,15 +113,36 @@ def run_cmd(cmd): failed_test_list = f"./test_lists/{args.backend.upper()}_{device_type_stripped.upper()}.txt" def run_tests(num_tries): - if len(args.regex_exclude) > 0: - args.regex_exclude = f"{args.regex_exclude}|" - if len(args.regex_include) > 0: - args.regex_include = f"-R {args.regex_include}" - # if failed_test_list is not empty, separator is |, otherwise it is empty - separator = "|" if os.path.exists(failed_test_list) and os.path.getsize(failed_test_list) > 0 else "" - cmd = f"{modules} {env_vars} ctest --output-on-failure --timeout {args.timeout} --repeat until-fail:{num_tries} -j {args.num_threads} {args.regex_include} -E \"{args.regex_exclude}`cat {failed_test_list}`{separator}`cat {all_test_list}`{texture_cmd}\" -O checkpy_{args.backend}_{args.device_type}.txt" - res, err = run_cmd(cmd) - return res, err + if len(args.regex_exclude) > 0: + args.regex_exclude = f"{args.regex_exclude}|" + if len(args.regex_include) > 0: + args.regex_include = f"{args.regex_include}|" + + # Determine the test mode based on the new argument + test_mode = "-E" if args.test_mode_failures == "exclude" else "-R" + + # if failed_test_list is not empty, separator is |, otherwise it is empty + separator = "|" if os.path.exists(failed_test_list) and os.path.getsize(failed_test_list) > 0 else "" + + cmd = f"{modules} {env_vars} ctest --output-on-failure --timeout {args.timeout} --repeat until-fail:{num_tries} -j {args.num_threads} {test_mode} \"{args.regex_exclude}{args.regex_include}`cat {failed_test_list}`{separator}`cat {all_test_list}`{texture_cmd}{double_cmd}\" -O checkpy_{args.backend}_{args.device_type}.txt" + res, err = run_cmd(cmd) + + # If using -R, print the tests that passed + if test_mode == "-R": + passed_tests = [] + for line in res.split('\n'): + if "Test" in line and "Passed" in line: + test_name = line.split(':')[1].strip() + passed_tests.append(test_name) + + if passed_tests: + print("The following tests FAILED:") + for test in passed_tests: + print(f" {test}") + else: + print("No tests passed.") + + return res, err # if --total-runtime is set, calculate the number of tries by running run_tests and checking the time @@ -148,4 +170,4 @@ def run_tests(num_tries): if "0 tests failed" in res: exit(0) else: - exit(1) + exit(1) \ No newline at end of file