Deprecate should_error, renamed to expected_error and add expected_exitcode for exitcode tests

Update mesonbuild/interpreter/interpreter.py

Co-authored-by: Charles Brunet <charles.brunet@optelgroup.com>

docs/markdown/Unit-tests.md

@@ -132,7 +132,12 @@ In addition, sometimes a test fails set up so that it should fail even
 if it is marked as an expected failure. The GNU standard approach in
 this case is to exit the program with error code 99. Again, Meson will
 detect this and report these tests as `ERROR`, ignoring the setting of
-`should_fail`. This behavior was added in version 0.50.0.
+`should_fail`. This behavior was added in version 0.50.0. In version
+1.11.0 `should_fail` has been deprecated and renamed to `expected_fail`.
+
+In version 1.11.0, `expected_exitcode` has been introduced. This makes
+it possible to positively test for non-zero return codes. An example
+of this would be to test if failure injection is detected in a test.
 
 ## Testing tool
 

docs/markdown/snippets/expected_error.md

@@ -0,0 +1,5 @@
+## Deprecate `should_fail` and rename it to `expected_fail`, also introduce `expected_exitcode`
+
+In 1.11.0 `should_fail` has been renamed to `expected_error`.
+
+Before 1.11.0, there was no way to positively test a command/binary returning error/non-zero exit code when the used protocol was set to exitcode, so `expected_exitcode` has been introduced to achieve this. Do note that if the exitcode does not match the expected value, GNU skip and exit codes are still valid and the test result may be skip or error.

docs/yaml/functions/benchmark.yaml

@@ -40,11 +40,29 @@ kwargs:
 
   should_fail:
     type: bool
+    deprecated: 1.11.0
     default: false
     description: |
       when true the test is considered passed if the
       executable returns a non-zero return value (i.e. reports an error)
 
+  expected_fail:
+    type: bool
+    since: 1.11.0
+    default: false
+    description: |
+      when true the test is considered passed if the
+      executable returns a non-zero return value (i.e. reports an error)
+
+  expected_exitcode:
+    type: int
+    since: 1.11.0
+    default: 0
+    description: |
+      the test is considered passed if the
+      executable returns the specified returncode,
+      only has effect when protocol is set to exitcode
+
   suite:
     type: str | array[str]
     description: |

mesonbuild/backend/backends.py

@@ -201,7 +201,8 @@ class TestSerialisation:
     is_parallel: bool
     cmd_args: T.List[str]
     env: mesonlib.EnvironmentVariables
-    should_fail: bool
+    expected_fail: bool
+    expected_exitcode: T.Optional[int]
     timeout: T.Optional[int]
     workdir: T.Optional[str]
     extra_paths: T.List[str]
@@ -1262,7 +1263,7 @@ class Backend:
             ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross,
                                    exe_wrapper, self.environment.need_exe_wrapper(),
                                    t.is_parallel, cmd_args, t_env,
-                                   t.should_fail, t.timeout, t.workdir,
+                                   t.expected_fail, t.expected_exitcode, t.timeout, t.workdir,
                                    extra_paths, t.protocol, t.priority,
                                    isinstance(exe, (build.Target, build.CustomTargetIndex)),
                                    isinstance(exe, build.Executable),

mesonbuild/interpreter/interpreter.py

@@ -2290,6 +2290,14 @@ class Interpreter(InterpreterBase, HoldableObject):
         if kwargs['timeout'] <= 0:
             FeatureNew.single_use('test() timeout <= 0', '0.57.0', self.subproject, location=node)
 
+        expected_fail = False
+        if kwargs['should_fail'] is not None and kwargs['expected_fail'] is not None:
+            raise InvalidArguments("Tried to use both 'should_fail' and 'expected_fail'")
+        elif kwargs['should_fail'] is not None:
+            expected_fail = kwargs['should_fail']
+        elif kwargs['expected_fail'] is not None:
+            expected_fail = kwargs['expected_fail']
+
         prj = self.subproject if self.is_subproject() else self.build.project_name
 
         suite: T.List[str] = []
@@ -2306,7 +2314,8 @@ class Interpreter(InterpreterBase, HoldableObject):
                      kwargs.get('is_parallel', False),
                      kwargs['args'],
                      env,
-                     kwargs['should_fail'],
+                     expected_fail,
+                     kwargs['expected_exitcode'],
                      kwargs['timeout'],
                      kwargs['workdir'],
                      kwargs['protocol'],

mesonbuild/interpreter/interpreterobjects.py

@@ -820,7 +820,7 @@ class Test(MesonInterpreterObject):
                  is_parallel: bool,
                  cmd_args: T.List[T.Union[str, mesonlib.File, build.Target, ExternalProgram]],
                  env: mesonlib.EnvironmentVariables,
-                 should_fail: bool, timeout: int, workdir: T.Optional[str], protocol: str,
+                 expected_fail: bool, expected_exitcode: int, timeout: int, workdir: T.Optional[str], protocol: str,
                  priority: int, verbose: bool):
         super().__init__()
         self.name = name
@@ -831,7 +831,8 @@ class Test(MesonInterpreterObject):
         self.is_parallel = is_parallel
         self.cmd_args = cmd_args
         self.env = env
-        self.should_fail = should_fail
+        self.expected_fail = expected_fail
+        self.expected_exitcode = expected_exitcode
         self.timeout = timeout
         self.workdir = workdir
         self.protocol = TestProtocol.from_str(protocol)

mesonbuild/interpreter/kwargs.py

@@ -42,7 +42,9 @@ class BaseTest(TypedDict):
 
     """Shared base for the Rust module."""
 
-    should_fail: bool
+    should_fail: T.Optional[bool]
+    expected_fail: T.Optional[bool]
+    expected_exitcode: T.Optional[int]
     timeout: int
     workdir: T.Optional[str]
     depends: T.List[T.Union[build.CustomTarget, build.BuildTarget]]

mesonbuild/interpreter/type_checking.py

@@ -511,7 +511,9 @@ def suite_convertor(suite: T.List[str]) -> T.List[str]:
     return suite
 
 TEST_KWS_NO_ARGS: T.List[KwargInfo] = [
-    KwargInfo('should_fail', bool, default=False),
+    KwargInfo('should_fail', (bool, NoneType), deprecated='1.11.0', deprecated_message='Use expected_fail instead of should_fail'),
+    KwargInfo('expected_fail', (bool, NoneType), since='1.11.0'),
+    KwargInfo('expected_exitcode', (int, NoneType), since='1.11.0'),
     KwargInfo('timeout', int, default=30),
     KwargInfo('workdir', (str, NoneType), default=None,
               validator=lambda x: 'must be an absolute path' if not os.path.isabs(x) else None),

mesonbuild/mtest.py

@@ -981,7 +981,8 @@ class TestRun:
         self.additional_error = ''
         self.cmd: T.Optional[T.List[str]] = None
         self.env = test_env
-        self.should_fail = test.should_fail
+        self.expected_fail = test.expected_fail
+        self.expected_exitcode = test.expected_exitcode
         self.project = test.project_name
         self.junit: T.Optional[et.ElementTree] = None
         self.is_parallel = is_parallel
@@ -1041,7 +1042,7 @@ class TestRun:
         if self.needs_parsing and self.console_mode is ConsoleUser.INTERACTIVE:
             self.res = TestResult.IGNORED
         assert isinstance(self.res, TestResult)
-        if self.should_fail and self.res in (TestResult.OK, TestResult.FAIL):
+        if self.expected_fail and self.res in (TestResult.OK, TestResult.FAIL):
             self.res = TestResult.UNEXPECTEDPASS if self.res is TestResult.OK else TestResult.EXPECTEDFAIL
         if self.stdo and not self.stdo.endswith('\n'):
             self.stdo += '\n'
@@ -1097,6 +1098,8 @@ class TestRunExitCode(TestRun):
     def complete(self) -> None:
         if self.res != TestResult.RUNNING:
             pass
+        elif self.returncode == (self.expected_exitcode or 0):
+            self.res = TestResult.OK
         elif self.returncode == GNU_SKIP_RETURNCODE:
             self.res = TestResult.SKIP
         elif self.returncode == GNU_ERROR_RETURNCODE:

test cases/common/278 expected exitcode/failing.c

@@ -0,0 +1,3 @@
+int main(void) {
+    return EXIT_CODE;
+}

test cases/common/278 expected exitcode/meson.build

@@ -0,0 +1,10 @@
+project('expected exitcode', 'c')
+
+foreach num : [0,1,5,77,99,255]
+define = '-DEXIT_CODE=' + num.to_string()
+exe = executable('prog' + num.to_string(), 'failing.c', c_args: [define])
+test('errorcode-'+ num.to_string(), exe, expected_exitcode: num)
+endforeach
+
+exe = executable('prog-skip', 'failing.c', c_args: ['-DEXIT_CODE=77'])
+test('gnu-skip-on-expected-exitcode', exe, expected_exitcode: 5)

test cases/common/68 should fail/meson.build

@@ -1,4 +1,5 @@
 project('should fail', 'c')
 
 exe = executable('prog', 'failing.c')
-test('failing', exe, should_fail : true)
+test('should-failing', exe, should_fail : true)
+test('expected-failing', exe, expected_fail : true)

test cases/failing test/7 expected exitcode/main.c

@@ -0,0 +1,3 @@
+int main(void) {
+    return EXIT_CODE;
+}

test cases/failing test/7 expected exitcode/meson.build

@@ -0,0 +1,7 @@
+project('expected exitcode', 'c')
+
+passing_exe = executable('passing', 'main.c', c_args: ['-DEXIT_CODE=0'])
+gnu_error_exe = executable('failing', 'main.c', c_args: ['-DEXIT_CODE=99'])
+test('errorcode', passing_exe, expected_exitcode: 1)
+test('errorcode-gnu-error-expected-not-specified', gnu_error_exe)
+test('errorcode-gnu-error-expected-specified', gnu_error_exe, expected_exitcode: 33)

test cases/rust/9 unit tests/meson.build

@@ -1,4 +1,4 @@
-project('rust unit tests', 'rust', meson_version: '>=1.8.0')
+project('rust unit tests', 'rust', meson_version: '>=1.11.0')
 
 t = executable(
   'rust_test',
@@ -11,7 +11,7 @@ test(
   t,
   protocol : 'rust',
   suite : ['foo'],
-  should_fail : true,
+  expected_fail : true,
 )
 
 test(
@@ -50,7 +50,7 @@ endif
 
 exe = executable('rust_exe', ['test2.rs', 'test.rs'], build_by_default : false)
 
-rust.test('rust_test_from_exe', exe, should_fail : true)
+rust.test('rust_test_from_exe', exe, expected_fail : true)
 
 lib = static_library('rust_static', ['test.rs'], build_by_default : false, rust_abi: 'c')
 rust.test('rust_test_from_static', lib, args: ['--skip', 'test_add_intentional_fail'])