Create a unit_test macro

test/README.md

@@ -2,11 +2,11 @@
 
 ## Running Tests
 
-Our projects use both **`pytest`** and **`unittest`** frameworks.
-You can run tests using either method.
-The **`-vvv`** flag is used for **verbosity**, which provides more detailed output and is very helpful for debugging.
+Our projects use both **`bazel tests`** and **`pytest`**.
+You can run bazel tests with: `bazel test //...`.
+For more verbosity in python tests use **`-vvv`** or **`--log-cli-level=DEBUG`** for pytest.
 
-### To run all tests, use one of the following command:
+### To run all unit tests, use one of the following command:
 * **Using Pytest:**
     ```bash
     pytest unit -vvv
@@ -32,11 +32,12 @@ python3 -m unittest discover unit/my_test_dir -vvv
    Inside the `unit` directory, create a folder for your new test. This folder should contain:
    - All source/header files needed for the test
    - `BUILD`
-   - A python test script
-   - `__init__.py`
 
 2. **Creating the BUILD File**
-    - Make sure that all failing test targets get the `"manual"` tag. For example:
+    - Create the `cc_binary/library` targets.
+    - Create the `codechecker_test` targets.
+    - Create `unit_test` targets to assert on the outputs of the codechecker targets. (See `unit/unit_test.bzl` for documentation)
+    - Make sure that all failing `codechecker_test` targets get the `"manual"` tag. For example:
     ```
     # This is a test I expect to fail
     codechecker_test(
@@ -49,8 +50,10 @@ python3 -m unittest discover unit/my_test_dir -vvv
         ],
     )
     ```
+    - Tip: To test these failing tests, create a unit_test target and assert the bug being found.
 
-2. **Creating the Test File**  
+3. **Create a python test if you must**
+    - If you are writing a python test, have an `__init__.py` file in the test directory!
     - Your test script must follow the naming convention:
         ```text
         test_*.py
@@ -70,6 +73,17 @@ python3 -m unittest discover unit/my_test_dir -vvv
 
 ## Testing on open source projects
 
+### To run all FOSS tests, use one of the following command:
+* **Using Pytest:**
+    ```bash
+    pytest foss -vvv
+    ```
+
+* **Using Unittest:**
+    ```bash
+    python3 -m unittest discover foss -vvv
+    ```
+
 ## Add a new open source project:
 
 1. Create a folder in the foss folder with the name of the project.

test/unit/virtual_include/__init__.py → test/unit/BUILD

@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+exports_files(["grep_check.py"])

test/unit/grep_check.py

@@ -0,0 +1,198 @@
+# Copyright 2023 Ericsson AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Validates wether a list of patterns is found in a file
+
+This test reads a file and asserts that all provided patterns
+are present within its contents.
+
+Intended to be used as the main of a py_test Bazel target.
+"""
+
+import argparse
+import glob
+from itertools import chain
+import re
+import sys
+from typing import Callable
+
+
+def parse_args() -> argparse.Namespace:
+    """
+    Parse command-line arguments.
+    Returns:
+        Parsed arguments containing the file path and list of patterns.
+    """
+    parser = argparse.ArgumentParser(
+        description=(
+            "Assert that all given patterns exist in the provided file."
+        )
+    )
+    parser.add_argument(
+        "--files",
+        nargs="+",
+        required=True,
+        help="Path or glob pattern to the file(s) to search within.",
+    )
+    parser.add_argument(
+        "--contains",
+        nargs="+",
+        required=False,
+        help="One or more string to assert are present in the file(s).",
+    )
+    parser.add_argument(
+        "--excludes",
+        nargs="+",
+        required=False,
+        help="One or more string to assert are not present in the file(s).",
+    )
+    parser.add_argument(
+        "--regex_patterns",
+        nargs="+",
+        required=False,
+        help="One or more patterns to assert are present in the file(s).",
+    )
+    parser.add_argument(
+        "--any",
+        required=False,
+        action="store_true",
+        help="If provided, the program will succeed if at least one file "
+        "contains the patterns",
+    )
+    return parser.parse_args()
+
+
+def check_args(args):
+    """Checks wether the arguments are correct, aborts if not"""
+    if not args.contains and not args.excludes and not args.regex_patterns:
+        print("  [ERROR] Must define at least one pattern or negative pattern.")
+        sys.exit(1)
+
+
+def exact_match(pattern: str, content: str) -> bool:
+    """Default search: checks if pattern is exactly in content."""
+    return pattern in content
+
+
+def check_patterns(
+    content: str,
+    patterns: list[str],
+    search: Callable[[str, str], bool] = exact_match,
+    negative: bool = False,
+) -> tuple[bool, set[str], set[str]]:
+    """
+    Checks wether a string contains every pattern in a list.
+
+    Args:
+        content: Text to search in.
+        patterns: List of search patterns.
+        search: Function with signature func(pattern, content) -> bool.
+                Defaults to `pattern in content`.
+        negative: Boolean, wether to check patterns as positive or negative.
+    Returns:
+        bool - Wether all patterns are correctly (not) found.
+        set[str] - Set of patterns that are correctly (not) found.
+        set[str] - Set of patterns that are incorrectly (not) found.
+    """
+    all_passed = True
+    found_patterns = set()
+    missing_pattern = set()
+    for pattern in patterns:
+        if bool(search(pattern, content)) == negative:
+            missing_pattern.add(pattern)
+            all_passed = False
+        else:
+            found_patterns.add(pattern)
+    return all_passed, found_patterns, missing_pattern
+
+
+def check_file(content: str, args) -> tuple[bool, set[str], set[str]]:
+    """
+    Checks if file contains all regexes.
+    Returns boolean value, and set of patterns correctly identified.
+    """
+    all_passed = True
+    found_patterns = set()
+    missing_patterns = set()
+
+    groups = [
+        (args.contains, exact_match, False),
+        (args.excludes, exact_match, True),
+        (args.regex_patterns, re.search, False),
+    ]
+
+    for patterns, search, negative in groups:
+        if patterns:
+            group_pass, found, missing = check_patterns(
+                content, patterns, search, negative
+            )
+            all_passed = all_passed and group_pass
+            found_patterns.update(found)
+            missing_patterns.update(missing)
+
+    return all_passed, found_patterns, missing_patterns
+
+
+def main() -> None:
+    """Entry point for the pattern-matching test."""
+    args = parse_args()
+    check_args(args)
+
+    all_passed = True
+    found_patterns = set()
+    missing_patterns = set()
+
+    file_paths = []
+    for file_pattern in args.files:
+        matched_files = glob.glob(file_pattern, recursive=True)
+        if not matched_files:
+            print(f"  [WARN] No files matched pattern/path: '{file_pattern}'")
+        file_paths.extend(matched_files)
+
+    if not file_paths:
+        print("  [ERR] No file collected to be checked.")
+        sys.exit(1)
+
+    for file in file_paths:
+        with open(file, "r", encoding="utf-8") as f:
+            content = f.read()
+            all_found_in_file, patterns_in_file, missing_patterns_in_file = (
+                check_file(content, args)
+            )
+            all_passed = all_passed and all_found_in_file
+            found_patterns.update(patterns_in_file)
+            for pattern in missing_patterns_in_file:
+                missing_patterns.add((file, pattern))
+
+    if args.any:
+        all_passed = True
+        for pattern in chain(
+            args.contains or [],
+            args.excludes or [],
+            args.regex_patterns or [],
+        ):
+            if pattern not in found_patterns:
+                all_passed = False
+                break
+
+    if not all_passed:
+        for file, pattern in missing_patterns:
+            print(f"Missing pattern {pattern} in file {file}")
+        print("\nOne or more patterns missing. Test FAILED.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

test/unit/template/BUILD

@@ -0,0 +1,68 @@
+# Copyright 2023 Ericsson AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "@rules_cc//cc:defs.bzl",
+    "cc_library",
+)
+load(
+    "//src:codechecker.bzl",
+    "codechecker_test",
+)
+load(
+    "//test/unit:unit_test.bzl",
+    "unit_test",
+)
+
+cc_library(
+    name = "template_target",
+    srcs = ["template.cpp"],
+    tags = ["manual"],
+)
+
+codechecker_test(
+    name = "template_codechecker",
+    tags = ["manual"],
+    targets = [
+        "template_target",
+    ],
+)
+
+codechecker_test(
+    name = "template_per_file",
+    per_file = True,
+    tags = ["manual"],
+    targets = [
+        "template_target",
+    ],
+)
+
+unit_test(
+    name = "template_codechecker_test",
+    contains = ["core.DivideZero"],
+    data = [":template_codechecker"],
+    excludes = ["Text not in file"],
+    files = "test/unit/template/template_codechecker/codechecker.log",
+    regex_patterns = ["[a-z]"],
+)
+
+unit_test(
+    name = "template_per_file_test",
+    contains = ["Division by zero"],
+    data = [":template_per_file"],
+    excludes = ["Text not in file"],
+    files = "test/unit/template/template_per_file/**/*.plist",
+    regex_patterns = ["[a-z]"],
+    require_patterns_in_each_file = False,
+)

test/unit/template/template.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2023 Ericsson AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+int main(){
+    int a = 0;
+    int b = 0/a;
+    return b;
+}

test/unit/template/test_template.py

@@ -14,7 +14,10 @@
 
 """
 TODO: Describe what this file does
+
+Do not write python tests, unless absolutely necessary!
 """
+
 import os
 import unittest
 from typing import final
@@ -23,13 +26,16 @@
 
 class TestTemplate(TestBase):
     """TODO: Add a description"""
+
     # Set working directory
     __test_path__ = os.path.dirname(os.path.abspath(__file__))
     # TODO: fix folder name
-    BAZEL_BIN_DIR = os.path.join("../../..", "bazel-bin", "test", 
-                                        "unit", "my_test_folder")
-    BAZEL_TESTLOGS_DIR = os.path.join("../../..", "bazel-testlogs", "test", 
-                                        "unit", "my_test_folder")
+    BAZEL_BIN_DIR = os.path.join(
+        "../../..", "bazel-bin", "test", "unit", "my_test_folder"
+    )
+    BAZEL_TESTLOGS_DIR = os.path.join(
+        "../../..", "bazel-testlogs", "test", "unit", "my_test_folder"
+    )
 
     @final
     @classmethod