From 35a913589f0375105b230e537e1177033b625503 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 16 Mar 2026 13:47:57 +0100 Subject: [PATCH 1/2] Introduce unit_test macro for writing unit tests runnable with bazel test //... --- test/README.md | 30 ++- .../{virtual_include/__init__.py => BUILD} | 2 + test/unit/grep_check.py | 202 ++++++++++++++++++ test/unit/template/BUILD | 68 ++++++ test/unit/template/template.cpp | 21 ++ test/unit/template/test_template.py | 14 +- test/unit/unit_test.bzl | 88 ++++++++ 7 files changed, 413 insertions(+), 12 deletions(-) rename test/unit/{virtual_include/__init__.py => BUILD} (94%) create mode 100644 test/unit/grep_check.py create mode 100644 test/unit/template/BUILD create mode 100644 test/unit/template/template.cpp create mode 100644 test/unit/unit_test.bzl diff --git a/test/README.md b/test/README.md index 14bde169..1b7e8cd1 100644 --- a/test/README.md +++ b/test/README.md @@ -2,11 +2,11 @@ ## Running Tests -Our projects use both **`pytest`** and **`unittest`** frameworks. -You can run tests using either method. -The **`-vvv`** flag is used for **verbosity**, which provides more detailed output and is very helpful for debugging. +Our projects use both **`bazel tests`** and **`pytest`**. +You can run bazel tests with: `bazel test //...`. +For more verbosity in python tests use **`-vvv`** or **`--log-cli-level=DEBUG`** for pytest. -### To run all tests, use one of the following command: +### To run all unit tests, use one of the following command: * **Using Pytest:** ```bash pytest unit -vvv @@ -32,11 +32,12 @@ python3 -m unittest discover unit/my_test_dir -vvv Inside the `unit` directory, create a folder for your new test. This folder should contain: - All source/header files needed for the test - `BUILD` - - A python test script - - `__init__.py` 2. **Creating the BUILD File** - - Make sure that all failing test targets get the `"manual"` tag. For example: + - Create the `cc_binary/library` targets. + - Create the `codechecker_test` targets. + - Create `unit_test` targets to assert on the outputs of the codechecker targets. (See `unit/unit_test.bzl` for documentation) + - Make sure that all failing `codechecker_test` targets get the `"manual"` tag. For example: ``` # This is a test I expect to fail codechecker_test( @@ -49,8 +50,10 @@ python3 -m unittest discover unit/my_test_dir -vvv ], ) ``` + - Tip: To test these failing tests, create a unit_test target and assert the bug being found. -2. **Creating the Test File** +3. **Create a python test if you must** + - If you are writing a python test, have an `__init__.py` file in the test directory! - Your test script must follow the naming convention: ```text test_*.py @@ -70,6 +73,17 @@ python3 -m unittest discover unit/my_test_dir -vvv ## Testing on open source projects +### To run all FOSS tests, use one of the following command: +* **Using Pytest:** + ```bash + pytest foss -vvv + ``` + +* **Using Unittest:** + ```bash + python3 -m unittest discover foss -vvv + ``` + ## Add a new open source project: 1. Create a folder in the foss folder with the name of the project. diff --git a/test/unit/virtual_include/__init__.py b/test/unit/BUILD similarity index 94% rename from test/unit/virtual_include/__init__.py rename to test/unit/BUILD index 78bab5f1..35047442 100644 --- a/test/unit/virtual_include/__init__.py +++ b/test/unit/BUILD @@ -11,3 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +exports_files(["grep_check.py"]) diff --git a/test/unit/grep_check.py b/test/unit/grep_check.py new file mode 100644 index 00000000..2565db17 --- /dev/null +++ b/test/unit/grep_check.py @@ -0,0 +1,202 @@ +# Copyright 2023 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Validates wether a list of patterns is found in a file + +This test reads a file and asserts that all provided patterns +are present within its contents. + +Intended to be used as the main of a py_test Bazel target. +""" + +import argparse +import glob +from itertools import chain +import re +import sys +from typing import Callable + + +def parse_args() -> argparse.Namespace: + """ + Parse command-line arguments. + Returns: + Parsed arguments containing the file path and list of patterns. + """ + parser = argparse.ArgumentParser( + description=( + "Assert that all given patterns exist in the provided file." + ) + ) + parser.add_argument( + "--files", + nargs="+", + required=True, + help="Path or glob pattern to the file(s) to search within.", + ) + parser.add_argument( + "--contains", + nargs="+", + required=False, + help="One or more string to assert are present in the file(s).", + ) + parser.add_argument( + "--excludes", + nargs="+", + required=False, + help="One or more string to assert are not present in the file(s).", + ) + parser.add_argument( + "--regex_patterns", + nargs="+", + required=False, + help="One or more patterns to assert are present in the file(s).", + ) + parser.add_argument( + "--any", + required=False, + action="store_true", + help="If provided, the program will succeed if at least one file " + "contains the patterns", + ) + return parser.parse_args() + + +def check_args(args): + """Checks wether the arguments are correct, aborts if not""" + if ( + not args.contains + and not args.excludes + and not args.regex_patterns + ): + print(" [ERROR] Must define at least one pattern or negative pattern.") + sys.exit(1) + + +def exact_match(pattern: str, content: str) -> bool: + """Default search: checks if pattern is exactly in content.""" + return pattern in content + + +def check_patterns( + content: str, + patterns: list[str], + search: Callable[[str, str], bool] = exact_match, + negative: bool = False, +) -> tuple[bool, set[str], set[str]]: + """ + Checks wether a string contains every pattern in a list. + + Args: + content: Text to search in. + patterns: List of search patterns. + search: Function with signature func(pattern, content) -> bool. + Defaults to `pattern in content`. + negative: Boolean, wether to check patterns as positive or negative. + Returns: + bool - Wether all patterns are correctly (not) found. + set[str] - Set of patterns that are correctly (not) found. + set[str] - Set of patterns that are incorrectly (not) found. + """ + all_passed = True + found_patterns = set() + missing_pattern = set() + for pattern in patterns: + if bool(search(pattern, content)) == negative: + missing_pattern.add(pattern) + all_passed = False + else: + found_patterns.add(pattern) + return all_passed, found_patterns, missing_pattern + + +def check_file(content: str, args) -> tuple[bool, set[str], set[str]]: + """ + Checks if file contains all regexes. + Returns boolean value, and set of patterns correctly identified. + """ + all_passed = True + found_patterns = set() + missing_patterns = set() + + groups = [ + (args.contains, exact_match, False), + (args.excludes, exact_match, True), + (args.regex_patterns, re.search, False), + ] + + for patterns, search, negative in groups: + if patterns: + group_pass, found, missing = check_patterns( + content, patterns, search, negative + ) + all_passed = all_passed and group_pass + found_patterns.update(found) + missing_patterns.update(missing) + + return all_passed, found_patterns, missing_patterns + + +def main() -> None: + """Entry point for the pattern-matching test.""" + args = parse_args() + check_args(args) + + all_passed = True + found_patterns = set() + missing_patterns = set() + + file_paths = [] + for file_pattern in args.files: + matched_files = glob.glob(file_pattern, recursive=True) + if not matched_files: + print(f" [WARN] No files matched pattern/path: '{file_pattern}'") + file_paths.extend(matched_files) + + if not file_paths: + print(" [ERR] No files specified") + sys.exit(1) + + for file in file_paths: + with open(file, "r", encoding="utf-8") as f: + content = f.read() + all_found_in_file, patterns_in_file, missing_patterns_in_file = ( + check_file(content, args) + ) + all_passed = all_passed and all_found_in_file + found_patterns.update(patterns_in_file) + for pattern in missing_patterns_in_file: + missing_patterns.add((file, pattern)) + + if args.any: + all_passed = True + for pattern in chain( + args.contains or [], + args.excludes or [], + args.regex_patterns or [], + ): + if pattern not in found_patterns: + all_passed = False + break + + if not all_passed: + for file, pattern in missing_patterns: + print(f"Missing pattern {pattern} in file {file}") + print("\nOne or more patterns missing. Test FAILED.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/unit/template/BUILD b/test/unit/template/BUILD new file mode 100644 index 00000000..2b451e67 --- /dev/null +++ b/test/unit/template/BUILD @@ -0,0 +1,68 @@ +# Copyright 2023 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "@rules_cc//cc:defs.bzl", + "cc_library", +) +load( + "//src:codechecker.bzl", + "codechecker_test", +) +load( + "//test/unit:unit_test.bzl", + "unit_test", +) + +cc_library( + name = "template_target", + srcs = ["template.cpp"], + tags = ["manual"], +) + +codechecker_test( + name = "template_codechecker", + tags = ["manual"], + targets = [ + "template_target", + ], +) + +codechecker_test( + name = "template_per_file", + per_file = True, + tags = ["manual"], + targets = [ + "template_target", + ], +) + +unit_test( + name = "template_codechecker_test", + contains = ["core.DivideZero"], + data = [":template_codechecker"], + excludes = ["Text not in file"], + files = "test/unit/template/template_codechecker/codechecker.log", + regex_patterns = ["[a-z]"], +) + +unit_test( + name = "template_per_file_test", + contains = ["Division by zero"], + data = [":template_per_file"], + excludes = ["Text not in file"], + files = "test/unit/template/template_per_file/**/*.plist", + regex_patterns = ["[a-z]"], + require_patterns_in_each_file = False, +) diff --git a/test/unit/template/template.cpp b/test/unit/template/template.cpp new file mode 100644 index 00000000..ea73f4ec --- /dev/null +++ b/test/unit/template/template.cpp @@ -0,0 +1,21 @@ +/* + * Copyright 2023 Ericsson AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +int main(){ + int a = 0; + int b = 0/a; + return b; +} diff --git a/test/unit/template/test_template.py b/test/unit/template/test_template.py index 8113cdd2..3de86808 100644 --- a/test/unit/template/test_template.py +++ b/test/unit/template/test_template.py @@ -14,7 +14,10 @@ """ TODO: Describe what this file does + +Do not write python tests, unless absolutely necessary! """ + import os import unittest from typing import final @@ -23,13 +26,16 @@ class TestTemplate(TestBase): """TODO: Add a description""" + # Set working directory __test_path__ = os.path.dirname(os.path.abspath(__file__)) # TODO: fix folder name - BAZEL_BIN_DIR = os.path.join("../../..", "bazel-bin", "test", - "unit", "my_test_folder") - BAZEL_TESTLOGS_DIR = os.path.join("../../..", "bazel-testlogs", "test", - "unit", "my_test_folder") + BAZEL_BIN_DIR = os.path.join( + "../../..", "bazel-bin", "test", "unit", "my_test_folder" + ) + BAZEL_TESTLOGS_DIR = os.path.join( + "../../..", "bazel-testlogs", "test", "unit", "my_test_folder" + ) @final @classmethod diff --git a/test/unit/unit_test.bzl b/test/unit/unit_test.bzl new file mode 100644 index 00000000..0079ea08 --- /dev/null +++ b/test/unit/unit_test.bzl @@ -0,0 +1,88 @@ +# Copyright 2026 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Macro for generating unit tests for rules_codechecker. + +Each unit_test() generates a local py_test that: + Depends on an other action, and checks for patterns on its output. + +Example: + unit_test( + name = "my_unit_test", + files = ["my_target_file.ext"], + contains = ["my_pattern"], + excludes = ["my_negative_pattern"], + regex_patterns = ["my_.*regex.*_pattern"], + ) +""" + +def unit_test( + name, + files, + contains = None, + excludes = None, + regex_patterns = None, + require_patterns_in_each_file = True, + tags = [], + size = "medium", + **kwargs): + """Generate a py_test that checks if provided patterns are in the files. + + Args: + name: Test name. + files: Path or glob to the files to be checked. + contains: Text that should be inside the files. + excludes: Text that shouldn't be inside the files. + regex_patterns: Regex patterns that should be found inside the files. + require_patterns_in_each_file: If False its enough if every pattern is found in at least one file. + tags: Additional test tags. + size: Test size (default: medium). + **kwargs: Forwarded to py_test. + """ + if type(files) == "string": + files = [files] + if type(contains) == "string": + contains = [contains] + if type(excludes) == "string": + excludes = [excludes] + if type(regex_patterns) == "string": + regex_patterns = [regex_patterns] + + python_args = ["--files"] + files + if contains: + python_args.append("--contains") + python_args.extend(["\"{}\"".format(pat) for pat in contains]) + if excludes: + python_args.append("--excludes") + python_args.extend(["\"{}\"".format(pat) for pat in excludes]) + if regex_patterns: + python_args.append("--regex_patterns") + python_args.extend(["\"{}\"".format(pat) for pat in regex_patterns]) + if not require_patterns_in_each_file: + python_args.append("--any") + + # Since we use a custom python toolchain instead of rules_python in WORKSPACE + # we cannot include py_test + # buildifier: disable=native-pys + native.py_test( + name = name, + srcs = ["//test/unit:grep_check.py"], + main = "//test/unit:grep_check.py", + args = python_args, + local = True, + tags = ["unit"] + tags, + size = size, + **kwargs + ) From 2766f718f36a4bd23ad1018b23e8f47c2d54b367 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Thu, 18 Jun 2026 13:54:00 +0200 Subject: [PATCH 2/2] Rewrite virtual_include test to use unit_test macro --- test/unit/virtual_include/BUILD | 36 +++++ .../virtual_include/test_virtual_include.py | 139 ------------------ 2 files changed, 36 insertions(+), 139 deletions(-) delete mode 100644 test/unit/virtual_include/test_virtual_include.py diff --git a/test/unit/virtual_include/BUILD b/test/unit/virtual_include/BUILD index f8f2ccca..8725ec14 100644 --- a/test/unit/virtual_include/BUILD +++ b/test/unit/virtual_include/BUILD @@ -23,6 +23,10 @@ load( "//src:codechecker.bzl", "codechecker_test", ) +load( + "//test/unit:unit_test.bzl", + "unit_test", +) # Test for strip_include_prefix cc_library( @@ -87,3 +91,35 @@ codechecker_test( "virtual_implementation_deps_include", ], ) + +unit_test( + name = "per_file_plist_path_resolved_test", + contains = "/_virtual_includes/", + data = [":per_file_virtual_include"], + files = "test/unit/virtual_include/per_file_virtual_include/**/*.plist", +) + +unit_test( + name = "codechecker_plist_path_resolved_test", + # FIXME: In the postprocessed plists, all _virtual_include paths + # should've been removed. Update to negative_patterns + contains = "/_virtual_includes/", + data = [":codechecker_virtual_include"], + files = "test/unit/virtual_include/codechecker_virtual_include/**/*.plist", + require_patterns_in_each_file = False, +) + +unit_test( + name = "per_file_impl_deps_include_test", + contains = "Division by zero", + data = [":per_file_impl_deps_include"], + files = "test/unit/virtual_include/per_file_impl_deps_include/**/*.plist", + require_patterns_in_each_file = False, +) + +unit_test( + name = "codechecker_impl_deps_include_test", + contains = "core.DivideZero", + data = [":codechecker_impl_deps_include"], + files = "test/unit/virtual_include/codechecker_impl_deps_include/codechecker.log", +) diff --git a/test/unit/virtual_include/test_virtual_include.py b/test/unit/virtual_include/test_virtual_include.py deleted file mode 100644 index 04eb7fb3..00000000 --- a/test/unit/virtual_include/test_virtual_include.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2023 Ericsson AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -We want CodeChecker to point to the original files in its results, this needs -post processing. -Bazel creates _virtual_includes folder for headers, declared in a cc_library -rule with the include_prefix or strip_include_prefix. When warnings are found -in these headers, their paths in the plist files should get resolved to the -original file path. -This unittest test whether these paths containing `_virtual_include` have been -resolved -""" -import logging -import os -import unittest -import glob -from typing import final -from common.base import TestBase - - -class TestVirtualInclude(TestBase): - """Tests checking virtual include path resolution""" - - # Set working directory - __test_path__ = os.path.dirname(os.path.abspath(__file__)) - BAZEL_BIN_DIR = os.path.join( - "../../..", "bazel-bin", "test", "unit", "virtual_include" - ) - BAZEL_TESTLOGS_DIR = os.path.join( - "../../..", "bazel-testlogs", "test", "unit", "virtual_include" - ) - - @final - @classmethod - def setUpClass(cls): - """Set up before the test suite""" - super().setUpClass() - cls.run_command("bazel clean") - - def contains_in_files(self, regex, file_list): - """ - Select files containing a regex from a list of files. - - Args: - regex - Pattern to be searched. - file_list - List of files to be checked. - - Returns: - List of files containing the regex pattern - """ - result = [] - for file in file_list: - logging.debug("Checking file: %s", file) - if self.contains_regex_in_file(file, regex): - result.append(file) - return result - - def test_bazel_per_file_plist_path_resolved(self): - """Test: bazel build :per_file_virtual_include""" - ret, _, stderr = self.run_command( - "bazel build //test/unit/virtual_include:per_file_virtual_include", - ) - self.assertEqual(ret, 0, stderr) - plist_files = glob.glob( - os.path.join( - self.BAZEL_BIN_DIR, # pyright: ignore - "per_file_virtual_include", - "**", - "*.plist", - ), - recursive=True, - ) - # Test whether the _virtual_include directory was actually created. - self.assertTrue( - os.path.isdir(f"{self.BAZEL_BIN_DIR}/_virtual_includes") - ) - # FIXME: In the postprocessed plists, all _virtual_include paths - # should've been removed. Possible fix is in the github PR #14. - self.assertNotEqual( - self.contains_in_files(r"/_virtual_includes/", plist_files), [] - ) - - def test_bazel_codechecker_plist_path_resolved(self): - """Test: bazel build :codechecker_virtual_include""" - ret, _, stderr = self.run_command( - "bazel build " - "//test/unit/virtual_include:codechecker_virtual_include" - ) - self.assertEqual(ret, 0, stderr) - plist_files = glob.glob( - os.path.join( - self.BAZEL_BIN_DIR, # pyright: ignore - "codechecker_virtual_include", - "**", - "*.plist", - ), - recursive=True, - ) - # Test whether the _virtual_include directory was actually created. - self.assertTrue( - os.path.isdir(f"{self.BAZEL_BIN_DIR}/_virtual_includes") - ) - # FIXME: In the postprocessed plists, all _virtual_include paths - # should've been removed. Possible fix is in the github PR #14. - self.assertNotEqual( - self.contains_in_files(r"/_virtual_includes/", plist_files), [] - ) - - def test_bazel_codechecker_implementation_deps_virtual_include(self): - """Test: bazel build :codechecker_impl_deps_include""" - ret, _, stderr = self.run_command( - "bazel build --experimental_cc_implementation_deps " - "//test/unit/virtual_include:codechecker_impl_deps_include" - ) - self.assertEqual(ret, 0, stderr) - - def test_bazel_per_file_implementation_deps_virtual_include(self): - """Test: bazel build :per_file_impl_deps_include""" - ret, _, stderr = self.run_command( - "bazel build --experimental_cc_implementation_deps " - "//test/unit/virtual_include:per_file_impl_deps_include" - ) - self.assertEqual(ret, 0, stderr) - - -if __name__ == "__main__": - unittest.main(buffer=True)