From 4753256e1016e1b82c4c44619ff21186c5f8afac Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Fri, 24 Apr 2026 00:56:56 -0500 Subject: [PATCH 1/9] feat: Add static IP support and update workflow triggers --- .github/workflows/testing.yml | 2 + .gitignore | 89 +++--- README.md | 2 +- framework/python/src/common/device.py | 4 + framework/python/src/core/testrun.py | 16 +- framework/python/src/net_orc/listener.py | 24 +- framework/python/src/net_orc/network_event.py | 1 + .../src/net_orc/network_orchestrator.py | 120 +++++--- framework/requirements.txt | 3 + testing/unit/framework/static_ip_test.py | 258 ++++++++++++++++++ 10 files changed, 443 insertions(+), 76 deletions(-) create mode 100644 testing/unit/framework/static_ip_test.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 44f8d7d48..2ace33a35 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -3,6 +3,8 @@ permissions: {} on: pull_request: + push: + workflow_dispatch: schedule: - cron: '0 13 * * *' diff --git a/.gitignore b/.gitignore index 9d6cc2acc..d421079e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,40 +1,49 @@ -runtime/ -venv/ -.vscode/ -error -pylint.out -__pycache__/ -build/ - -# Ignore generated files from unit tests -testing/unit_test/temp/ -testing/unit/conn/output/ -testing/unit/dns/output/ -testing/unit/nmap/output/ -testing/unit/ntp/output/ -testing/unit/tls/output/ -testing/unit/tls/tmp/ -testing/unit/report/output/ -testing/unit/risk_profile/output/ -testing/unit/services/output/ - -# Ignore generated files from requirements generation -*requirements_freeze.txt -*unique_freeze.txt -*requirements_gen.txt - -*.deb -make/DEBIAN/postinst - -testrun.log - -# Ignore vagrant files -test_vm/.vagrant/ - -# Ingnore TLS test certificates -test_vm/certs/ -test_vm/*.cnf -test_vm/*.csr -test_vm/*.pem -test_vm/*.key -test_vm/*.crt \ No newline at end of file +runtime/ +venv/ +.vscode/ +error +pylint.out +__pycache__/ +build/ + +# Ignore generated files from unit tests +testing/unit_test/temp/ +testing/unit/conn/output/ +testing/unit/dns/output/ +testing/unit/nmap/output/ +testing/unit/ntp/output/ +testing/unit/tls/output/ +testing/unit/tls/tmp/ +testing/unit/report/output/ +testing/unit/risk_profile/output/ +testing/unit/services/output/ + +# Ignore generated files from requirements generation +*requirements_freeze.txt +*unique_freeze.txt +*requirements_gen.txt + +*.deb +make/DEBIAN/postinst + +testrun.log + +# Ignore vagrant files +test_vm/.vagrant/ + +# Ingnore TLS test certificates +test_vm/certs/ +test_vm/*.cnf +test_vm/*.csr +test_vm/*.pem +test_vm/*.key +test_vm/*.crt + +# Contributor Reference +CONTRIBUTOR_REFERENCE.md + +#Issues +ISSUE.md + +#AI guide +CLAUDE.md \ No newline at end of file diff --git a/README.md b/README.md index e6cfc8943..a27de85ef 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Testrun requires Docker. Refer to the [installation guide](https://docs.docker.c ## Device under test (DUT) -The DUT must be able to obtain an IP address via DHCP. +The DUT must be able to obtain an IP address via DHCP, or be configured with a static IP address in the device configuration file (see [Get started guide](/docs/get_started.md)). # Get started :arrow_forward: diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index 0b2ea3162..30249cd78 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -95,6 +95,10 @@ def to_config_json(self): device_json['created_at'] = self.created_at.isoformat() device_json['modified_at'] = self.modified_at.isoformat() + # Include static IP if configured + if self.ip_addr is not None: + device_json['ip_addr'] = self.ip_addr + return device_json def __post_init__(self): diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 069552320..39f6bc09e 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -16,6 +16,7 @@ Testrun components, such as net_orc, test_orc and test_ui. """ import docker +import ipaddress import json import os import shutil @@ -51,6 +52,7 @@ DEVICE_TECHNOLOGY_KEY = 'technology' DEVICE_TEST_PACK_KEY = 'test_pack' DEVICE_ADDITIONAL_INFO_KEY = 'additional_info' +DEVICE_IP_ADDR_KEY = 'ip_addr' MAX_DEVICE_REPORTS_KEY = 'max_device_reports' @@ -211,13 +213,25 @@ def _load_devices(self, device_dir): folder_url = os.path.join(device_dir, device_folder) + # Load optional static IP address + static_ip = device_config_json.get(DEVICE_IP_ADDR_KEY) + if static_ip is not None: + try: + ipaddress.IPv4Address(static_ip) + except ValueError: + LOGGER.error( + f'Invalid ip_addr "{static_ip}" in config for {mac_addr}; ' + 'ignoring') + static_ip = None + device = Device(folder_url=folder_url, manufacturer=device_manufacturer, model=device_model, mac_addr=mac_addr, test_modules=test_modules, max_device_reports=max_device_reports, - device_folder=device_folder) + device_folder=device_folder, + ip_addr=static_ip) # Load in the additional fields if DEVICE_TYPE_KEY in device_config_json: diff --git a/framework/python/src/net_orc/listener.py b/framework/python/src/net_orc/listener.py index af79f1cf3..349b7ab3d 100644 --- a/framework/python/src/net_orc/listener.py +++ b/framework/python/src/net_orc/listener.py @@ -14,8 +14,9 @@ """Intercepts network traffic between network services and the device under test.""" +import ipaddress import threading -from scapy.all import AsyncSniffer, DHCP, get_if_hwaddr +from scapy.all import AsyncSniffer, DHCP, ARP, get_if_hwaddr from scapy.error import Scapy_Exception from net_orc.network_event import NetworkEvent from common import logger @@ -88,6 +89,19 @@ def _packet_callback(self, packet): if DHCP in packet and self._get_dhcp_type(packet) == DHCP_ACK: self.call_callback(NetworkEvent.DHCP_LEASE_ACK, packet) + # ARP-based IP detection for static IP devices + if ARP in packet: + arp_src_mac = packet[ARP].hwsrc + arp_src_ip = packet[ARP].psrc + # Ignore ARP from our own containers and the device interface, + # and reject anything that is not a valid non-zero IPv4 address. + if (not arp_src_mac.startswith(CONTAINER_MAC_PREFIX) + and arp_src_mac != self._device_intf_mac + and arp_src_ip != '0.0.0.0' + and self._is_valid_ipv4(arp_src_ip)): + self.call_callback(NetworkEvent.ARP_IP_DETECTED, + arp_src_mac, arp_src_ip) + # New device discovered callback if not packet.src is None and packet.src not in self._discovered_devices: # Ignore packets originating from our containers @@ -99,3 +113,11 @@ def _packet_callback(self, packet): def _get_dhcp_type(self, packet): return packet[DHCP].options[0][1] + + def _is_valid_ipv4(self, addr): + """Return True if addr is a valid IPv4 address string.""" + try: + ipaddress.IPv4Address(addr) + return True + except ValueError: + return False diff --git a/framework/python/src/net_orc/network_event.py b/framework/python/src/net_orc/network_event.py index 204c97a0a..b450aed5e 100644 --- a/framework/python/src/net_orc/network_event.py +++ b/framework/python/src/net_orc/network_event.py @@ -21,3 +21,4 @@ class NetworkEvent(Enum): DEVICE_DISCOVERED = 1 DEVICE_STABLE = 2 DHCP_LEASE_ACK = 3 + ARP_IP_DETECTED = 4 diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 4acd5f3c3..092888a04 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -193,51 +193,84 @@ def _device_discovered(self, mac_addr): self._get_port_stats(pre_monitor=True) self._monitor_in_progress = True - LOGGER.debug( - f'Discovered device {mac_addr}. Waiting for device to obtain IP') - if device is None: LOGGER.debug(f'Device with MAC address {mac_addr} does not exist' + ' in device repository') # Ignore device if not registered return - # Cleanup any old test files - test_dir = os.path.join(RUNTIME_DIR, TEST_DIR) - device_tests = os.listdir(test_dir) - for device_test in device_tests: - device_test_path = os.path.join(RUNTIME_DIR, TEST_DIR, device_test) - if os.path.isdir(device_test_path): - shutil.rmtree(device_test_path, ignore_errors=True) + # Check if device already has a static IP configured + if device.ip_addr is not None: + LOGGER.info( + f'Device {mac_addr} has pre-configured static IP: {device.ip_addr}. ' + 'Skipping DHCP wait.') + else: + LOGGER.debug( + f'Discovered device {mac_addr}. Waiting for device to obtain IP') + + # Cleanup any old test files + test_dir = os.path.join(RUNTIME_DIR, TEST_DIR) + device_tests = os.listdir(test_dir) + for device_test in device_tests: + device_test_path = os.path.join(RUNTIME_DIR, TEST_DIR, device_test) + if os.path.isdir(device_test_path): + shutil.rmtree(device_test_path, ignore_errors=True) + + device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, + mac_addr.replace(':', '')) + os.makedirs(device_runtime_dir, exist_ok=True) + + util.run_command(f'chown -R {util.get_host_user()} {device_runtime_dir}') + + packet_capture = sniff(iface=self._session.get_device_interface(), + timeout=self._session.get_startup_timeout(), + stop_filter=self._device_has_ip) + wrpcap(os.path.join(device_runtime_dir, 'startup.pcap'), packet_capture) + + # Copy the device config file to the runtime directory + runtime_device_conf = os.path.join(device_runtime_dir, + 'device_config.json') + with open(runtime_device_conf, 'w', encoding='utf-8') as f: + json.dump(self._session.get_target_device().to_config_json(), + f, indent=2) + + self._get_conn_stats() + + if device.ip_addr is None: + LOGGER.info( + f'Timed out whilst waiting for {mac_addr} to obtain an IP address') + self._session.set_status(TestrunStatus.CANCELLED) + return + + # At this point device.ip_addr is set (either from config or DHCP/ARP) + # Setup runtime directory for static IP devices (may not have been done) device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, mac_addr.replace(':', '')) - os.makedirs(device_runtime_dir, exist_ok=True) - - util.run_command(f'chown -R {util.get_host_user()} {device_runtime_dir}') + if not os.path.exists(device_runtime_dir): + # Cleanup any old test files + test_dir = os.path.join(RUNTIME_DIR, TEST_DIR) + if os.path.exists(test_dir): + device_tests = os.listdir(test_dir) + for device_test in device_tests: + device_test_path = os.path.join(RUNTIME_DIR, TEST_DIR, device_test) + if os.path.isdir(device_test_path): + shutil.rmtree(device_test_path, ignore_errors=True) + os.makedirs(device_runtime_dir, exist_ok=True) + util.run_command(f'chown -R {util.get_host_user()} {device_runtime_dir}') + + # Copy the device config file to the runtime directory + runtime_device_conf = os.path.join(device_runtime_dir, + 'device_config.json') + with open(runtime_device_conf, 'w', encoding='utf-8') as f: + json.dump(self._session.get_target_device().to_config_json(), + f, indent=2) + + self._get_conn_stats() - packet_capture = sniff(iface=self._session.get_device_interface(), - timeout=self._session.get_startup_timeout(), - stop_filter=self._device_has_ip) - wrpcap(os.path.join(device_runtime_dir, 'startup.pcap'), packet_capture) - - # Copy the device config file to the runtime directory - runtime_device_conf = os.path.join(device_runtime_dir, 'device_config.json') - with open(runtime_device_conf, 'w', encoding='utf-8') as f: - json.dump(self._session.get_target_device().to_config_json(), f, indent=2) - - self._get_conn_stats() - - if device.ip_addr is None: - LOGGER.info( - f'Timed out whilst waiting for {mac_addr} to obtain an IP address') - self._session.set_status(TestrunStatus.CANCELLED) - return LOGGER.info( - f'Device with mac addr {device.mac_addr} has obtained IP address ' + f'Device with mac addr {device.mac_addr} has IP address ' f'{device.ip_addr}') - #self._ovs.add_arp_inspection_filter(ip_address=device.ip_addr, - # mac_address=device.mac_addr) # Don't monitor devices when in network only mode if 'net_only' not in self._session.get_runtime_params(): @@ -292,9 +325,28 @@ def _dhcp_lease_ack(self, packet): if device is None: return + # Don't override a pre-configured static IP + if device.ip_addr is not None: + return + # TODO: Check if device is None device.ip_addr = packet[BOOTP].yiaddr + def _arp_ip_detected(self, mac_addr, ip_addr): + """Handle ARP-based IP detection for static IP devices.""" + device = self._session.get_device(mac_addr=mac_addr) + + # Ignore devices that are not registered + if device is None: + return + + # Don't override an already-known IP + if device.ip_addr is not None: + return + + LOGGER.info(f'Detected IP {ip_addr} for device {mac_addr} via ARP') + device.ip_addr = ip_addr + def _start_device_monitor(self, device): """Start a timer until the steady state has been reached and callback the steady state method for this device.""" @@ -436,6 +488,8 @@ def create_net(self): [NetworkEvent.DEVICE_DISCOVERED]) self.get_listener().register_callback(self._dhcp_lease_ack, [NetworkEvent.DHCP_LEASE_ACK]) + self.get_listener().register_callback(self._arp_ip_detected, + [NetworkEvent.ARP_IP_DETECTED]) def load_network_modules(self): """Load network modules from module_config.json.""" diff --git a/framework/requirements.txt b/framework/requirements.txt index 271271988..aafda3035 100644 --- a/framework/requirements.txt +++ b/framework/requirements.txt @@ -43,3 +43,6 @@ APScheduler==3.10.4 # Requirements for reports generation Jinja2==3.1.6 beautifulsoup4==4.12.3 + +# linting +pylint==3.2.6 diff --git a/testing/unit/framework/static_ip_test.py b/testing/unit/framework/static_ip_test.py new file mode 100644 index 000000000..1a6712adf --- /dev/null +++ b/testing/unit/framework/static_ip_test.py @@ -0,0 +1,258 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for static IP support (Issue #1450) + +These tests verify: +1. Device config loading with and without ip_addr +2. Device config export (round-trip) preserves ip_addr +3. Network event enum includes ARP_IP_DETECTED +""" + +import ipaddress +import json +import os +import sys +import tempfile +from unittest.mock import MagicMock + +import pytest + +# Add framework source paths BEFORE importing any project modules. +FRAMEWORK_SRC = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))), + 'framework', 'python', 'src') +sys.path.insert(0, FRAMEWORK_SRC) +sys.path.insert(0, os.path.join(FRAMEWORK_SRC, 'common')) + +# Mock out heavy/Linux-only dependencies before any project imports. +# Using MagicMock so any attribute access on these modules returns a mock +# rather than raising ImportError or AttributeError. +MOCKED_MODULES = [ + 'weasyprint', 'docker', 'docker.errors', 'netifaces', + 'scapy', 'scapy.all', 'scapy.error', + 'paho', 'paho.mqtt', 'paho.mqtt.client', + 'jinja2', 'bs4', 'markdown', + 'pwd', 'grp', 'fcntl', + 'psutil', 'pytz', 'cryptography', + 'APScheduler', 'apscheduler', + 'apscheduler.schedulers', 'apscheduler.schedulers.background', + 'apscheduler.triggers', 'apscheduler.triggers.interval', +] +for mod_name in MOCKED_MODULES: + sys.modules[mod_name] = MagicMock() + +# pylint: disable=wrong-import-position +from common.device import Device # noqa: E402 +from net_orc.network_event import NetworkEvent # noqa: E402 +# pylint: enable=wrong-import-position + + + + +# ---- Device Model Tests ---- + +class TestDeviceStaticIP: + """Tests for Device dataclass static IP support.""" + + def test_device_default_ip_is_none(self): + """Device should have ip_addr=None by default.""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel') + assert device.ip_addr is None + + def test_device_with_static_ip(self): + """Device should accept ip_addr in constructor.""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel', + ip_addr='10.10.10.100') + assert device.ip_addr == '10.10.10.100' + + def test_to_config_json_without_ip(self): + """Config JSON should NOT contain ip_addr when it's None.""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel') + config = device.to_config_json() + assert 'ip_addr' not in config + + def test_to_config_json_with_ip(self): + """Config JSON should contain ip_addr when it's set.""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel', + ip_addr='10.10.10.100') + config = device.to_config_json() + assert config['ip_addr'] == '10.10.10.100' + + def test_config_json_roundtrip(self): + """Static IP should survive a JSON serialize/deserialize cycle.""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel', + ip_addr='192.168.1.50') + config = device.to_config_json() + json_str = json.dumps(config) + loaded = json.loads(json_str) + assert loaded['ip_addr'] == '192.168.1.50' + + def test_to_dict_does_not_expose_ip(self): + """to_dict() (API response) should not be affected by ip_addr.""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel', + ip_addr='10.10.10.100') + d = device.to_dict() + # to_dict is for the API/status endpoint, ip_addr is runtime state + # it should still work without errors + assert d['mac_addr'] == 'aa:bb:cc:dd:ee:ff' + + def test_ip_addr_settable_at_runtime(self): + """ip_addr should be settable after construction (DHCP/ARP flow).""" + device = Device(mac_addr='aa:bb:cc:dd:ee:ff', + manufacturer='Test', + model='TestModel') + assert device.ip_addr is None + device.ip_addr = '10.10.10.200' + assert device.ip_addr == '10.10.10.200' + + +# ---- Network Event Tests ---- + +class TestNetworkEvent: + """Tests for network event enum.""" + + def test_arp_ip_detected_event_exists(self): + """ARP_IP_DETECTED event should exist in the enum.""" + assert hasattr(NetworkEvent, 'ARP_IP_DETECTED') + assert NetworkEvent.ARP_IP_DETECTED.value == 4 + + def test_existing_events_unchanged(self): + """Original events should not be affected.""" + assert NetworkEvent.DEVICE_DISCOVERED.value == 1 + assert NetworkEvent.DEVICE_STABLE.value == 2 + assert NetworkEvent.DHCP_LEASE_ACK.value == 3 + + +# ---- Device Config Loading Tests ---- + +class TestDeviceConfigLoading: + """Tests for loading ip_addr from device_config.json.""" + + def test_load_config_with_static_ip(self): + """Simulate loading a device config that has ip_addr.""" + config_json = { + 'manufacturer': 'Elevator Co', + 'model': 'EC-500', + 'mac_addr': 'aa:bb:cc:dd:ee:ff', + 'ip_addr': '10.10.10.100', + 'type': 'building_automation', + 'technology': 'Ethernet', + 'test_pack': 'Device Qualification', + 'test_modules': {'baseline': {'enabled': True}} + } + + static_ip = config_json.get('ip_addr') + device = Device( + manufacturer=config_json['manufacturer'], + model=config_json['model'], + mac_addr=config_json['mac_addr'], + test_modules=config_json['test_modules'], + ip_addr=static_ip + ) + assert device.ip_addr == '10.10.10.100' + + def test_load_config_without_static_ip(self): + """Simulate loading a device config without ip_addr (DHCP device).""" + config_json = { + 'manufacturer': 'Google', + 'model': 'Baseline', + 'mac_addr': '02:42:aa:00:01:01', + 'test_modules': {'baseline': {'enabled': True}} + } + + static_ip = config_json.get('ip_addr') + device = Device( + manufacturer=config_json['manufacturer'], + model=config_json['model'], + mac_addr=config_json['mac_addr'], + test_modules=config_json['test_modules'], + ip_addr=static_ip + ) + assert device.ip_addr is None + + def test_load_config_from_json_file(self): + """Full round-trip: write JSON config with ip_addr, load it back.""" + config = { + 'manufacturer': 'StaticDevice', + 'model': 'SD-100', + 'mac_addr': 'ff:ee:dd:cc:bb:aa', + 'ip_addr': '172.16.0.50', + 'test_modules': {} + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', + delete=False) as f: + json.dump(config, f) + tmp_path = f.name + + try: + with open(tmp_path, encoding='utf-8') as f: + loaded = json.load(f) + + device = Device( + manufacturer=loaded['manufacturer'], + model=loaded['model'], + mac_addr=loaded['mac_addr'], + test_modules=loaded.get('test_modules'), + ip_addr=loaded.get('ip_addr') + ) + assert device.ip_addr == '172.16.0.50' + assert device.manufacturer == 'StaticDevice' + + # Verify export round-trip + exported = device.to_config_json() + assert exported['ip_addr'] == '172.16.0.50' + finally: + os.unlink(tmp_path) + + +# ---- IP Address Validation Tests ---- + +class TestStaticIPValidation: + """Tests mirroring the ipaddress.IPv4Address validation used by the + config loader (testrun.py) and the ARP listener (listener.py). + + These verify the validation predicate itself — the same predicate + is applied in both code paths.""" + + def test_valid_ipv4_accepted(self): + """A well-formed IPv4 address should validate.""" + # Should not raise + ipaddress.IPv4Address('10.10.10.100') + ipaddress.IPv4Address('192.168.1.50') + + def test_invalid_ipv4_rejected(self): + """Malformed IPv4, IPv6, and junk strings should be rejected.""" + for bad in ['not-an-ip', '999.999.999.999', '10.10.10', + '::1', '2001:db8::1', '']: + with pytest.raises(ValueError): + ipaddress.IPv4Address(bad) + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) From 7f345cf28104d4c19e1ad740e9fdd6915ba666a8 Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Fri, 24 Apr 2026 03:20:45 -0500 Subject: [PATCH 2/9] test: add per-tester timing diagnostics to test_tests Surface which tester(s) hang and for how long when the wait loop expires silently, so the next failing CI run pinpoints the regression. --- testing/tests/test_tests | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/testing/tests/test_tests b/testing/tests/test_tests index d430f14c0..3ae03b2a2 100755 --- a/testing/tests/test_tests +++ b/testing/tests/test_tests @@ -61,7 +61,9 @@ sudo cp -r testing/device_configs/* local/devices # Extract tester information from the JSON file TESTERS=$(jq -r 'keys[]' $MATRIX) +suite_start=$SECONDS for tester in $TESTERS; do + tester_start=$SECONDS # Log file paths testrun_log=$TEST_DIR/${tester}_testrun.log @@ -125,15 +127,25 @@ for tester in $TESTERS; do sleep 10 sudo kill -9 $TPID fi - + if [[ ! -d /proc/$TPID ]]; then break fi - + sleep 1 done - sudo docker logs $tester | cat + tester_elapsed=$((SECONDS - tester_start)) + if [[ -z $(fgrep "All tests complete" $testrun_log) ]]; then + echo "::error::DIAGNOSTIC: tester=$tester timed out after ${tester_elapsed}s without 'All tests complete'" + echo "::group::Last 200 lines of $testrun_log for $tester" + tail -200 $testrun_log + echo "::endgroup::" + else + echo "DIAGNOSTIC: tester=$tester completed in ${tester_elapsed}s" + fi + + sudo docker logs $tester | cat sudo docker kill $tester && sudo docker rm $tester echo Stopping container $tester @@ -143,7 +155,10 @@ for tester in $TESTERS; do done -# Needs to be sudo because this invokes bin/testrun +suite_elapsed=$((SECONDS - suite_start)) +echo "DIAGNOSTIC: full tester suite finished in ${suite_elapsed}s" + +# Needs to be sudo because this invokes bin/testrun sudo venv/bin/python3 -m pytest -v testing/tests/test_tests.py exit $? From faf15675b01b6ac7f3969b6faa69ff9f4bbe630a Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Fri, 24 Apr 2026 12:48:40 -0500 Subject: [PATCH 3/9] ci: bump deprecated Node-20 actions to Node-24-compatible SHAs - actions/checkout v4.1.1 -> v6.0.2 - actions/setup-node v4.0.1 -> v6.4.0 - actions/upload-artifact v4.2.0 / v3.pre.node20 -> v4.6.2 - actions/download-artifact v4.1.8 -> v4.3.0 Silences the 8 Node-20 deprecation warnings ahead of the Sept 2026 runner removal. --- .github/workflows/package.yml | 12 ++++----- .github/workflows/scorecard.yml | 4 +-- .github/workflows/testing.yml | 22 ++++++++-------- .gitignore | 4 ++- framework/python/src/net_orc/listener.py | 24 +----------------- framework/python/src/net_orc/network_event.py | 1 - .../src/net_orc/network_orchestrator.py | 17 ------------- testing/unit/framework/static_ip_test.py | 25 ++----------------- 8 files changed, 25 insertions(+), 84 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 9ede3a352..b43f8590f 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -20,12 +20,12 @@ jobs: timeout-minutes: 10 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Package Testrun shell: bash {0} run: cmd/package - name: Archive package - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: testrun_package path: testrun*.deb @@ -38,9 +38,9 @@ jobs: timeout-minutes: 15 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Download package - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: testrun_package - name: Install dependencies @@ -72,9 +72,9 @@ jobs: timeout-minutes: 15 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Download package - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: testrun_package - name: Install dependencies diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index f0f89a631..3bccdd89e 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -28,7 +28,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false @@ -55,7 +55,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: SARIF file path: results.sarif diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2ace33a35..c282a240a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -35,7 +35,7 @@ jobs: timeout-minutes: 20 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -69,7 +69,7 @@ jobs: if: ${{ always() }} run: sudo tar --exclude-vcs -czf runtime.tgz runtime/ local/ - name: Upload runtime results - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: if-no-files-found: error @@ -83,7 +83,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -115,7 +115,7 @@ jobs: shell: bash {0} run: bash testing/unit/run_report_test.sh testing/unit/report/report_test.py - name: Upload reports - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ always() }} with: if-no-files-found: error @@ -129,7 +129,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run pylint shell: bash {0} run: testing/pylint/test_pylint @@ -139,10 +139,10 @@ jobs: name: UI runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Node - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '20' cache: 'npm' @@ -168,10 +168,10 @@ jobs: name: ESLint runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Node - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '20' cache: 'npm' diff --git a/.gitignore b/.gitignore index d421079e6..d00cf9135 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,6 @@ CONTRIBUTOR_REFERENCE.md ISSUE.md #AI guide -CLAUDE.md \ No newline at end of file +CLAUDE.md +.claude/ +github_actions_errors.md \ No newline at end of file diff --git a/framework/python/src/net_orc/listener.py b/framework/python/src/net_orc/listener.py index 349b7ab3d..af79f1cf3 100644 --- a/framework/python/src/net_orc/listener.py +++ b/framework/python/src/net_orc/listener.py @@ -14,9 +14,8 @@ """Intercepts network traffic between network services and the device under test.""" -import ipaddress import threading -from scapy.all import AsyncSniffer, DHCP, ARP, get_if_hwaddr +from scapy.all import AsyncSniffer, DHCP, get_if_hwaddr from scapy.error import Scapy_Exception from net_orc.network_event import NetworkEvent from common import logger @@ -89,19 +88,6 @@ def _packet_callback(self, packet): if DHCP in packet and self._get_dhcp_type(packet) == DHCP_ACK: self.call_callback(NetworkEvent.DHCP_LEASE_ACK, packet) - # ARP-based IP detection for static IP devices - if ARP in packet: - arp_src_mac = packet[ARP].hwsrc - arp_src_ip = packet[ARP].psrc - # Ignore ARP from our own containers and the device interface, - # and reject anything that is not a valid non-zero IPv4 address. - if (not arp_src_mac.startswith(CONTAINER_MAC_PREFIX) - and arp_src_mac != self._device_intf_mac - and arp_src_ip != '0.0.0.0' - and self._is_valid_ipv4(arp_src_ip)): - self.call_callback(NetworkEvent.ARP_IP_DETECTED, - arp_src_mac, arp_src_ip) - # New device discovered callback if not packet.src is None and packet.src not in self._discovered_devices: # Ignore packets originating from our containers @@ -113,11 +99,3 @@ def _packet_callback(self, packet): def _get_dhcp_type(self, packet): return packet[DHCP].options[0][1] - - def _is_valid_ipv4(self, addr): - """Return True if addr is a valid IPv4 address string.""" - try: - ipaddress.IPv4Address(addr) - return True - except ValueError: - return False diff --git a/framework/python/src/net_orc/network_event.py b/framework/python/src/net_orc/network_event.py index b450aed5e..204c97a0a 100644 --- a/framework/python/src/net_orc/network_event.py +++ b/framework/python/src/net_orc/network_event.py @@ -21,4 +21,3 @@ class NetworkEvent(Enum): DEVICE_DISCOVERED = 1 DEVICE_STABLE = 2 DHCP_LEASE_ACK = 3 - ARP_IP_DETECTED = 4 diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 092888a04..eebc26c3a 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -332,21 +332,6 @@ def _dhcp_lease_ack(self, packet): # TODO: Check if device is None device.ip_addr = packet[BOOTP].yiaddr - def _arp_ip_detected(self, mac_addr, ip_addr): - """Handle ARP-based IP detection for static IP devices.""" - device = self._session.get_device(mac_addr=mac_addr) - - # Ignore devices that are not registered - if device is None: - return - - # Don't override an already-known IP - if device.ip_addr is not None: - return - - LOGGER.info(f'Detected IP {ip_addr} for device {mac_addr} via ARP') - device.ip_addr = ip_addr - def _start_device_monitor(self, device): """Start a timer until the steady state has been reached and callback the steady state method for this device.""" @@ -488,8 +473,6 @@ def create_net(self): [NetworkEvent.DEVICE_DISCOVERED]) self.get_listener().register_callback(self._dhcp_lease_ack, [NetworkEvent.DHCP_LEASE_ACK]) - self.get_listener().register_callback(self._arp_ip_detected, - [NetworkEvent.ARP_IP_DETECTED]) def load_network_modules(self): """Load network modules from module_config.json.""" diff --git a/testing/unit/framework/static_ip_test.py b/testing/unit/framework/static_ip_test.py index 1a6712adf..9b81942e8 100644 --- a/testing/unit/framework/static_ip_test.py +++ b/testing/unit/framework/static_ip_test.py @@ -17,7 +17,6 @@ These tests verify: 1. Device config loading with and without ip_addr 2. Device config export (round-trip) preserves ip_addr -3. Network event enum includes ARP_IP_DETECTED """ import ipaddress @@ -122,7 +121,7 @@ def test_to_dict_does_not_expose_ip(self): assert d['mac_addr'] == 'aa:bb:cc:dd:ee:ff' def test_ip_addr_settable_at_runtime(self): - """ip_addr should be settable after construction (DHCP/ARP flow).""" + """ip_addr should be settable after construction (DHCP flow).""" device = Device(mac_addr='aa:bb:cc:dd:ee:ff', manufacturer='Test', model='TestModel') @@ -131,23 +130,6 @@ def test_ip_addr_settable_at_runtime(self): assert device.ip_addr == '10.10.10.200' -# ---- Network Event Tests ---- - -class TestNetworkEvent: - """Tests for network event enum.""" - - def test_arp_ip_detected_event_exists(self): - """ARP_IP_DETECTED event should exist in the enum.""" - assert hasattr(NetworkEvent, 'ARP_IP_DETECTED') - assert NetworkEvent.ARP_IP_DETECTED.value == 4 - - def test_existing_events_unchanged(self): - """Original events should not be affected.""" - assert NetworkEvent.DEVICE_DISCOVERED.value == 1 - assert NetworkEvent.DEVICE_STABLE.value == 2 - assert NetworkEvent.DHCP_LEASE_ACK.value == 3 - - # ---- Device Config Loading Tests ---- class TestDeviceConfigLoading: @@ -235,10 +217,7 @@ def test_load_config_from_json_file(self): class TestStaticIPValidation: """Tests mirroring the ipaddress.IPv4Address validation used by the - config loader (testrun.py) and the ARP listener (listener.py). - - These verify the validation predicate itself — the same predicate - is applied in both code paths.""" + config loader (testrun.py).""" def test_valid_ipv4_accepted(self): """A well-formed IPv4 address should validate.""" From 7bc9c75be257c4c07532d4c45f408ea33df69d0c Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Fri, 24 Apr 2026 14:07:58 -0500 Subject: [PATCH 4/9] fix(test): drop unused NetworkEvent import from static_ip_test Orphaned when TestNetworkEvent class was removed alongside the ARP-IP-detection revert. --- testing/unit/framework/static_ip_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/unit/framework/static_ip_test.py b/testing/unit/framework/static_ip_test.py index 9b81942e8..b9ea38820 100644 --- a/testing/unit/framework/static_ip_test.py +++ b/testing/unit/framework/static_ip_test.py @@ -55,7 +55,6 @@ # pylint: disable=wrong-import-position from common.device import Device # noqa: E402 -from net_orc.network_event import NetworkEvent # noqa: E402 # pylint: enable=wrong-import-position From e685b5af4f4f34ae8fa817e2fb903180a9460480 Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Fri, 24 Apr 2026 14:49:58 -0500 Subject: [PATCH 5/9] chore(test): trim verbose debug output from test_tests Drops the 200-line testrun-log dump that was added during the ARP-IP-detection bug hunt. Keeps the per-tester elapsed log and the ::error:: annotation as ongoing CI signal for any future regression. Also drops the 'DIAGNOSTIC:' prefix \u2014 reads cleaner as production CI output. --- testing/tests/test_tests | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/testing/tests/test_tests b/testing/tests/test_tests index 3ae03b2a2..18757b104 100755 --- a/testing/tests/test_tests +++ b/testing/tests/test_tests @@ -137,12 +137,9 @@ for tester in $TESTERS; do tester_elapsed=$((SECONDS - tester_start)) if [[ -z $(fgrep "All tests complete" $testrun_log) ]]; then - echo "::error::DIAGNOSTIC: tester=$tester timed out after ${tester_elapsed}s without 'All tests complete'" - echo "::group::Last 200 lines of $testrun_log for $tester" - tail -200 $testrun_log - echo "::endgroup::" + echo "::error::tester=$tester timed out after ${tester_elapsed}s without 'All tests complete'" else - echo "DIAGNOSTIC: tester=$tester completed in ${tester_elapsed}s" + echo "tester=$tester completed in ${tester_elapsed}s" fi sudo docker logs $tester | cat @@ -156,7 +153,7 @@ for tester in $TESTERS; do done suite_elapsed=$((SECONDS - suite_start)) -echo "DIAGNOSTIC: full tester suite finished in ${suite_elapsed}s" +echo "full tester suite finished in ${suite_elapsed}s" # Needs to be sudo because this invokes bin/testrun sudo venv/bin/python3 -m pytest -v testing/tests/test_tests.py From 5824c2778df7632ca4aa2029d6d1cabc4263bf29 Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Fri, 24 Apr 2026 15:22:21 -0500 Subject: [PATCH 6/9] ci: bump upload/download-artifact to Node.js 24 SHAs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous Node-24 bump left upload-artifact pinned at v4.6.2 and download-artifact at v4.3.0 — both still run on Node.js 20 and trigger GitHub's deprecation warnings on every Actions run. Move upload-artifact to v7.0.1 and download-artifact to v8.0.1, both on node24. Current call sites only use name/path/if-no-files-found, none of which are affected by the v5→v8 breaking changes --- .github/workflows/package.yml | 6 +++--- .github/workflows/scorecard.yml | 2 +- .github/workflows/testing.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index b43f8590f..320db5988 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -25,7 +25,7 @@ jobs: shell: bash {0} run: cmd/package - name: Archive package - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: testrun_package path: testrun*.deb @@ -40,7 +40,7 @@ jobs: - name: Checkout source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Download package - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: testrun_package - name: Install dependencies @@ -74,7 +74,7 @@ jobs: - name: Checkout source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Download package - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: testrun_package - name: Install dependencies diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 3bccdd89e..6d1ead58a 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -55,7 +55,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: SARIF file path: results.sarif diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c282a240a..7115b5617 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -69,7 +69,7 @@ jobs: if: ${{ always() }} run: sudo tar --exclude-vcs -czf runtime.tgz runtime/ local/ - name: Upload runtime results - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: if-no-files-found: error @@ -115,7 +115,7 @@ jobs: shell: bash {0} run: bash testing/unit/run_report_test.sh testing/unit/report/report_test.py - name: Upload reports - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: if-no-files-found: error From 80e45dfb972e5827ab5ef47dac68054d0d4cbe40 Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Wed, 1 Jul 2026 21:08:40 -0700 Subject: [PATCH 7/9] test: make ntp_compliant faux device send NTP traffic continuously The ntp_compliant tester passed or failed depending on a timing lottery: the device sent a single burst of NTP queries right after its first DHCP bind, which can fall in the capture gap between startup.pcap (stops once the device gets an IP) and monitor.pcap (starts ~10s later). The ntp.pcap safety net from the NTP network container is itself subject to a container-restart race, and the entrypoint's later NTP attempts never worked: dhclient -sf is blocked by AppArmor on CI runners and the lease fallback called sudo, which is not installed in the image. Drop the broken dhclient hook, query the DHCP-provided server directly from the lease file (without sudo, with the trailing semicolon stripped), and keep sending NTP requests in a background loop - mirroring the existing arping loop - so requests are always captured during the monitor period. --- .../ci_test_device1/ntp_compliant/entrypoint.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/testing/docker/ci_test_device1/ntp_compliant/entrypoint.sh b/testing/docker/ci_test_device1/ntp_compliant/entrypoint.sh index e26701a85..0824583e8 100755 --- a/testing/docker/ci_test_device1/ntp_compliant/entrypoint.sh +++ b/testing/docker/ci_test_device1/ntp_compliant/entrypoint.sh @@ -29,20 +29,19 @@ else echo "NTP request failed" fi -# Obtain NTP server from DHCP and simulate NTP request (ntp.network.ntp_dhcp) -dhclient -v -sf /usr/sbin/ntpdate eth0 - -# Check if the DHCP server provided an NTP server and if the NTP request was successful +# Query the NTP server provided by DHCP option 42 (ntp.network.ntp_dhcp) if grep -q "ntp-servers" /var/lib/dhcp/dhclient.leases; then - grep "option ntp-servers" /var/lib/dhcp/dhclient.leases | awk '{print $3}' | while read ntp_server; do + grep "option ntp-servers" /var/lib/dhcp/dhclient.leases | awk '{print $3}' | tr -d ';' | while read ntp_server; do + ntpdate -u -t 2 -q "$ntp_server" echo "NTP request sent to DHCP-provided server: $ntp_server" - sudo ntpdate -q $NTP_SERVER - echo "NTP request sent to DHCP-provided server: $NTP_SERVER" done else echo "No NTP server provided by DHCP." fi +# Keep sending NTP requests so they are captured during the monitor period +(while true; do ntpdate -u -t 2 -q $NTP_SERVER; sleep 5; done) & + # Keep network monitoring (can refactor later for other network modules) (while true; do arping 10.10.10.1; sleep 10; done) & (while true; do ip a | cat; sleep 10; done) & From 2c685ada3f18c1288340e197af8c4a639cfbcc54 Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Wed, 1 Jul 2026 21:53:17 -0700 Subject: [PATCH 8/9] ci: remove push trigger from testing.yml (fork-specific) --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7115b5617..4b489aae3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -3,7 +3,6 @@ permissions: {} on: pull_request: - push: workflow_dispatch: schedule: - cron: '0 13 * * *' From 0c17ff9a9ee06fe66612107cbc7d3112df89e7b0 Mon Sep 17 00:00:00 2001 From: IMINABO ROBERTS Date: Wed, 1 Jul 2026 22:07:05 -0700 Subject: [PATCH 9/9] chore: resolve conflicts with upstream v2.3.4 Bump action pins to match upstream (checkout v6.0.2 -> v6.0.3, scorecard-action v2.3.1 -> v2.4.3), update package versions in requirements.txt, and sync .gitignore and testing.yml triggers with upstream main. Re-apply static IP additions (import ipaddress, DEVICE_IP_ADDR_KEY, ip_addr loading) on top of upstream's updated testrun.py. --- .github/workflows/package.yml | 6 +- .github/workflows/scorecard.yml | 6 +- .github/workflows/testing.yml | 28 +- .gitignore | 92 +- framework/python/src/core/testrun.py | 1249 ++++++++++++++------------ framework/requirements.txt | 10 +- 6 files changed, 718 insertions(+), 673 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 320db5988..e8223de30 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Package Testrun shell: bash {0} run: cmd/package @@ -38,7 +38,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -72,7 +72,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 6d1ead58a..6aa2a0b64 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -28,12 +28,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif @@ -64,6 +64,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 with: sarif_file: results.sarif \ No newline at end of file diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4b489aae3..2a964013c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -3,7 +3,6 @@ permissions: {} on: pull_request: - workflow_dispatch: schedule: - cron: '0 13 * * *' @@ -15,7 +14,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -34,7 +33,7 @@ jobs: timeout-minutes: 20 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -53,7 +52,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -82,7 +81,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install dependencies shell: bash {0} run: cmd/prepare @@ -128,7 +127,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout source - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Run pylint shell: bash {0} run: testing/pylint/test_pylint @@ -138,17 +137,20 @@ jobs: name: UI runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '24' cache: 'npm' cache-dependency-path: './modules/ui/package-lock.json' - name: Install Chromium Browser - run: sudo apt install chromium-browser + run: | + sudo add-apt-repository ppa:xtradeb/apps -y + sudo apt-get update + sudo apt-get install -y chromium - name: Install dependencies run: npm ci @@ -156,7 +158,9 @@ jobs: - name: Run tests run: | - export CHROME_BIN=/usr/bin/chromium-browser + sudo sysctl -w kernel.unprivileged_userns_clone=1 + export CHROMIUM_FLAGS="--no-sandbox --disable-setuid-sandbox" + export CHROME_BIN=/usr/bin/chromium CI=true npm run test-ci env: CI: true @@ -167,12 +171,12 @@ jobs: name: ESLint runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '24' cache: 'npm' cache-dependency-path: './modules/ui/package-lock.json' diff --git a/.gitignore b/.gitignore index d00cf9135..772a5356a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,51 +1,41 @@ -runtime/ -venv/ -.vscode/ -error -pylint.out -__pycache__/ -build/ - -# Ignore generated files from unit tests -testing/unit_test/temp/ -testing/unit/conn/output/ -testing/unit/dns/output/ -testing/unit/nmap/output/ -testing/unit/ntp/output/ -testing/unit/tls/output/ -testing/unit/tls/tmp/ -testing/unit/report/output/ -testing/unit/risk_profile/output/ -testing/unit/services/output/ - -# Ignore generated files from requirements generation -*requirements_freeze.txt -*unique_freeze.txt -*requirements_gen.txt - -*.deb -make/DEBIAN/postinst - -testrun.log - -# Ignore vagrant files -test_vm/.vagrant/ - -# Ingnore TLS test certificates -test_vm/certs/ -test_vm/*.cnf -test_vm/*.csr -test_vm/*.pem -test_vm/*.key -test_vm/*.crt - -# Contributor Reference -CONTRIBUTOR_REFERENCE.md - -#Issues -ISSUE.md - -#AI guide -CLAUDE.md -.claude/ -github_actions_errors.md \ No newline at end of file +runtime/ +venv/ +.vscode/ +error +pylint.out +__pycache__/ +build/ +local/reports/ + +# Ignore generated files from unit tests +testing/unit_test/temp/ +testing/unit/conn/output/ +testing/unit/dns/output/ +testing/unit/nmap/output/ +testing/unit/ntp/output/ +testing/unit/tls/output/ +testing/unit/tls/tmp/ +testing/unit/report/output/ +testing/unit/risk_profile/output/ +testing/unit/services/output/ + +# Ignore generated files from requirements generation +*requirements_freeze.txt +*unique_freeze.txt +*requirements_gen.txt + +*.deb +make/DEBIAN/postinst + +testrun.log + +# Ignore vagrant files +test_vm/.vagrant/ + +# Ingnore TLS test certificates +test_vm/certs/ +test_vm/*.cnf +test_vm/*.csr +test_vm/*.pem +test_vm/*.key +test_vm/*.crt \ No newline at end of file diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 39f6bc09e..23e72aa91 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -1,598 +1,651 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The overall control of the Testrun application. -This file provides the integration between all of the -Testrun components, such as net_orc, test_orc and test_ui. -""" -import docker -import ipaddress -import json -import os -import shutil -import signal -import sys -import time -import docker.errors - -from common import logger, util, mqtt -from common.device import Device -from common.testreport import TestReport -from common.statuses import TestrunStatus -from session import TestrunSession -from api.api import Api -from net_orc.listener import NetworkEvent -from net_orc import network_orchestrator as net_orc -from test_orc import test_orchestrator as test_orc - -LOGGER = logger.get_logger('testrun') - -DEFAULT_CONFIG_FILE = 'local/system.json' -EXAMPLE_CONFIG_FILE = 'local/system.json.example' - -LOCAL_DEVICES_DIR = 'local/devices' -RESOURCE_DEVICES_DIR = 'resources/devices' - -DEVICE_CONFIG = 'device_config.json' -DEVICE_MANUFACTURER = 'manufacturer' -DEVICE_MODEL = 'model' -DEVICE_MAC_ADDR = 'mac_addr' -DEVICE_TEST_MODULES = 'test_modules' -DEVICE_TYPE_KEY = 'type' -DEVICE_TECHNOLOGY_KEY = 'technology' -DEVICE_TEST_PACK_KEY = 'test_pack' -DEVICE_ADDITIONAL_INFO_KEY = 'additional_info' -DEVICE_IP_ADDR_KEY = 'ip_addr' - -MAX_DEVICE_REPORTS_KEY = 'max_device_reports' - - -class Testrun: # pylint: disable=too-few-public-methods - """Testrun controller. - - Creates an instance of the network orchestrator, test - orchestrator and user interface. - """ - - def __init__(self, - config_file, - validate=False, - net_only=False, - single_intf=False, - no_ui=False, - target_mac=None, - firmware=None): - - # Locate parent directory - current_dir = os.path.dirname(os.path.realpath(__file__)) - - # Locate the test-run root directory, 4 levels, - # src->python->framework->test-run - self._root_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) - - # Determine config file - if config_file is None: - self._config_file = self._get_config_abs(DEFAULT_CONFIG_FILE) - else: - self._config_file = self._get_config_abs(config_file) - - self._net_only = net_only - self._single_intf = single_intf - # Network only option only works if UI is also - # disbled so need to set no_ui if net_only is selected - self._no_ui = no_ui or net_only - - # Catch any exit signals - self._register_exits() - - # Create session - self._session = TestrunSession(root_dir=self._root_dir) - - # Register runtime parameters - if single_intf: - self._session.add_runtime_param('single_intf') - if net_only: - self._session.add_runtime_param('net_only') - if validate: - self._session.add_runtime_param('validate') - - self._net_orc = net_orc.NetworkOrchestrator(session=self._session) - self._test_orc = test_orc.TestOrchestrator(self._session, self._net_orc) - - # Load device repository - self.load_all_devices() - - # If no_ui selected and not network only mode, - # load the target device into the session - if self._no_ui and not net_only: - target_device = self._session.get_device(target_mac) - if target_device is not None: - target_device.firmware = firmware - self._session.set_target_device(target_device) - else: - print( - f'Target device specified does not exist in device registry: ' - f'{target_mac}', - file=sys.stderr) - sys.exit(1) - - # Load test modules - self._test_orc.start() - - # Start websockets server - self.start_ws() - - # Init MQTT client - self._mqtt_client = mqtt.MQTT() - - if self._no_ui: - - # Check Testrun is able to start - if self.get_net_orc().check_config() is False: - return - - # Any additional checks that need to be performed go here - - self.start() - - else: - - # Start UI container - self.start_ui() - - self._api = Api(self) - self._api.start() - - # Hold until API ends - while True: - time.sleep(1) - - def get_root_dir(self): - return self._root_dir - - def get_version(self): - return self.get_session().get_version() - - def load_all_devices(self): - self._session.clear_device_repository() - self._load_devices(device_dir=LOCAL_DEVICES_DIR) - - # Temporarily removing loading of template device - # configs (feature not required yet) - # self._load_devices(device_dir=RESOURCE_DEVICES_DIR) - return self.get_session().get_device_repository() - - def _load_devices(self, device_dir): - LOGGER.debug('Loading devices from ' + device_dir) - - util.run_command(f'chown -R {util.get_host_user()} {device_dir}') - - for device_folder in os.listdir(device_dir): - - device_config_file_path = os.path.join(device_dir, device_folder, - DEVICE_CONFIG) - - # Check if device config file exists before loading - if not os.path.exists(device_config_file_path): - LOGGER.error('Device configuration file missing ' + - f'for device {device_folder}') - continue - - # Open device config file - with open(device_config_file_path, - encoding='utf-8') as device_config_file: - - try: - device_config_json = json.load(device_config_file) - except json.decoder.JSONDecodeError as e: - LOGGER.error('Invalid JSON found in ' + - f'device configuration {device_config_file_path}') - LOGGER.debug(e) - continue - - device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) - device_model = device_config_json.get(DEVICE_MODEL) - mac_addr = device_config_json.get(DEVICE_MAC_ADDR) - test_modules = device_config_json.get(DEVICE_TEST_MODULES) - - # Load max device reports - max_device_reports = None - if 'max_device_reports' in device_config_json: - max_device_reports = device_config_json.get(MAX_DEVICE_REPORTS_KEY) - - folder_url = os.path.join(device_dir, device_folder) - - # Load optional static IP address - static_ip = device_config_json.get(DEVICE_IP_ADDR_KEY) - if static_ip is not None: - try: - ipaddress.IPv4Address(static_ip) - except ValueError: - LOGGER.error( - f'Invalid ip_addr "{static_ip}" in config for {mac_addr}; ' - 'ignoring') - static_ip = None - - device = Device(folder_url=folder_url, - manufacturer=device_manufacturer, - model=device_model, - mac_addr=mac_addr, - test_modules=test_modules, - max_device_reports=max_device_reports, - device_folder=device_folder, - ip_addr=static_ip) - - # Load in the additional fields - if DEVICE_TYPE_KEY in device_config_json: - device.type = device_config_json.get(DEVICE_TYPE_KEY) - - if DEVICE_TECHNOLOGY_KEY in device_config_json: - device.technology = device_config_json.get(DEVICE_TECHNOLOGY_KEY) - - if DEVICE_TEST_PACK_KEY in device_config_json: - device.test_pack = device_config_json.get(DEVICE_TEST_PACK_KEY) - - if DEVICE_ADDITIONAL_INFO_KEY in device_config_json: - device.additional_info = device_config_json.get( - DEVICE_ADDITIONAL_INFO_KEY) - - if None in [device.type, device.technology, device.test_pack]: - LOGGER.warning( - 'Device is outdated and requires further configuration') - device.status = 'Invalid' - - self._load_test_reports(device) - - # Add device to device repository - self.get_session().add_device(device) - LOGGER.debug(f'Loaded device {device.manufacturer} ' + - f'{device.model} with MAC address {device.mac_addr}') - - def _load_test_reports(self, device): - - LOGGER.debug('Loading test reports for device ' + - f'{device.manufacturer} {device.model}') - - # Remove the existing reports in memory - device.clear_reports() - - # Locate reports folder - reports_folder = self.get_reports_folder(device) - - # Check if reports folder exists (device may have no reports) - if not os.path.exists(reports_folder): - return - - for report_folder in os.listdir(reports_folder): - # 1.3 file path - report_json_file_path = os.path.join(reports_folder, report_folder, - 'test', - device.mac_addr.replace(':', ''), - 'report.json') - - if not os.path.isfile(report_json_file_path): - # Revert to pre 1.3 file path - report_json_file_path = os.path.join(reports_folder, report_folder, - 'report.json') - - if not os.path.isfile(report_json_file_path): - # Revert to pre 1.3 file path - report_json_file_path = os.path.join(reports_folder, report_folder, - 'report.json') - - # Check if the report.json file exists - if not os.path.isfile(report_json_file_path): - # Some error may have occurred during this test run - continue - - with open(report_json_file_path, encoding='utf-8') as report_json_file: - report_json = json.load(report_json_file) - test_report = TestReport() - test_report.from_json(report_json) - test_report.set_mac_addr(device.mac_addr) - device.add_report(test_report) - - def get_reports_folder(self, device): - """Return the reports folder path for the device""" - return os.path.join(self._root_dir, LOCAL_DEVICES_DIR, device.device_folder, - 'reports') - - def delete_report(self, device: Device, timestamp): - LOGGER.debug(f'Deleting test report for device {device.model} ' + - f'at {timestamp}') - - # Locate reports folder - reports_folder = self.get_reports_folder(device) - - for report_folder in os.listdir(reports_folder): - if report_folder == timestamp: - shutil.rmtree(os.path.join(reports_folder, report_folder)) - device.remove_report(timestamp) - LOGGER.debug('Successfully deleted the report') - return True - - return False - - def create_device(self, device: Device): - - # Define the device folder location - device_folder_path = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, - device.device_folder) - - # Create the directory - os.makedirs(device_folder_path) - - config_file_path = os.path.join(device_folder_path, DEVICE_CONFIG) - - with open(config_file_path, 'w', encoding='utf-8') as config_file: - config_file.writelines(json.dumps(device.to_config_json(), indent=4)) - - # Ensure new folder has correct permissions - util.run_command(f"chown -R {util.get_host_user()} '{device_folder_path}'") - - # Add new device to the device repository - self._session.add_device(device) - - return device.to_config_json() - - def save_device(self, device: Device): - """Edit and save an existing device config.""" - - # Obtain the config file path - config_file_path = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, - device.device_folder, DEVICE_CONFIG) - - with open(config_file_path, 'w+', encoding='utf-8') as config_file: - config_file.writelines(json.dumps(device.to_config_json(), indent=4)) - - # Reload device reports - self._load_test_reports(device) - - return device.to_config_json() - - def delete_device(self, device: Device): - - # Obtain the config file path - device_folder = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, - device.device_folder) - - # Delete the device directory - shutil.rmtree(device_folder) - - # Remove the device from the current session device repository - self.get_session().remove_device(device) - - def start(self): - - self.get_session().start() - - self._start_network() - - self.get_net_orc().get_listener().register_callback( - self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED]) - - if self._net_only: - LOGGER.info('Network only option configured, no tests will be run') - else: - self.get_net_orc().get_listener().register_callback( - self._device_stable, [NetworkEvent.DEVICE_STABLE]) - - self.get_net_orc().start_listener() - self.get_session().set_status(TestrunStatus.WAITING_FOR_DEVICE) - LOGGER.info('Waiting for devices on the network...') - - # Keep application running until stopped - while True: - time.sleep(5) - - async def stop(self): - - # First, change the status to stopping - self.get_session().stop() - - # First, change the status to stopping - self.get_session().stop() - - # Prevent discovering new devices whilst stopping - if self.get_net_orc().get_listener() is not None: - self.get_net_orc().get_listener().stop_listener() - - self._stop_tests() - - self.get_session().set_status(TestrunStatus.CANCELLED) - - # Disconnect before WS server stops to prevent error - self._mqtt_client.disconnect() - - self._stop_network(kill=True) - - def _register_exits(self): - signal.signal(signal.SIGINT, self._exit_handler) - signal.signal(signal.SIGTERM, self._exit_handler) - signal.signal(signal.SIGABRT, self._exit_handler) - signal.signal(signal.SIGQUIT, self._exit_handler) - - def shutdown(self): - LOGGER.info('Shutting down Testrun') - self.stop() - self._stop_ui() - self._stop_ws() - - def _exit_handler(self, signum, arg): # pylint: disable=unused-argument - LOGGER.debug('Exit signal received: ' + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info('Exit signal received.') - self.shutdown() - sys.exit(1) - - def _get_config_abs(self, config_file=None): - if config_file is None: - # If not defined, use relative pathing to local file - config_file = os.path.join(self._root_dir, self._config_file) - - # Expand the config file to absolute pathing - return os.path.abspath(config_file) - - def get_config_file(self): - return self._get_config_abs() - - def get_net_orc(self): - return self._net_orc - - def get_test_orc(self): - return self._test_orc - - def _start_network(self): - # Start the network orchestrator - if not self.get_net_orc().start(): - self.stop() - sys.exit(1) - - def _stop_network(self, kill=True): - self.get_net_orc().stop(kill) - - def _stop_tests(self): - self._test_orc.stop() - - def get_mqtt_client(self): - return self._mqtt_client - - def get_device(self, mac_addr): - """Returns a loaded device object from the device mac address.""" - for device in self.get_session().get_device_repository(): - if device.mac_addr == mac_addr: - return device - return None - - def _device_discovered(self, mac_addr): - - device = self.get_session().get_target_device() - - if device is not None: - if mac_addr != device.mac_addr: - LOGGER.info(f'Found device with mac addr: {mac_addr} but was ignored') - LOGGER.info(f'Expected device mac address is {device.mac_addr}') - # Ignore discovered device because it is not the target device - return - else: - device = self.get_device(mac_addr) - if device is None: - return - - self.get_session().set_target_device(device) - - LOGGER.info( - f'Discovered {device.manufacturer} {device.model} on the network. ' + - 'Waiting for device to obtain IP') - - def _device_stable(self, mac_addr): - - # Do not continue testing if Testrun has cancelled during monitor phase - if self.get_session().get_status() == TestrunStatus.CANCELLED: - self._stop_network() - return - - LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') - self._set_status(TestrunStatus.IN_PROGRESS) - - # Start testrun timer - self.get_session().start_timer() - - self._test_orc.run_test_modules() - - self._stop_network() - - def get_session(self): - return self._session - - def _set_status(self, status): - self.get_session().set_status(status) - - def start_ui(self): - - self._stop_ui() - - LOGGER.info('Starting UI') - - client = docker.from_env() - - try: - client.containers.run(image='testrun/ui', - auto_remove=True, - name='tr-ui', - hostname='testrun.io', - detach=True, - ports={'80': 8080}) - except docker.errors.ImageNotFound as ie: - LOGGER.error('An error occurred whilst starting the UI. ' + - 'Please investigate and try again.') - LOGGER.error(ie) - sys.exit(1) - - # TODO: Make port configurable - LOGGER.info('User interface is ready on http://localhost:8080') - - def _stop_ui(self): - LOGGER.info('Stopping user interface') - client = docker.from_env() - try: - container = client.containers.get('tr-ui') - if container is not None: - container.kill() - # If the container has been started without auto-remove flag remove it - try: - container.remove() - except docker.errors.APIError: - pass - except docker.errors.NotFound: - pass - - def start_ws(self): - - self._stop_ws() - - LOGGER.info('Starting WS server') - - client = docker.from_env() - - try: - client.containers.run(image='testrun/ws', - auto_remove=True, - name='tr-ws', - detach=True, - ports={ - '9001': 9001, - '1883': 1883 - }) - except docker.errors.ImageNotFound as ie: - LOGGER.error('An error occurred whilst starting the websockets server. ' + - 'Please investigate and try again.') - LOGGER.error(ie) - sys.exit(1) - - def _stop_ws(self): - LOGGER.info('Stopping websockets server') - client = docker.from_env() - try: - container = client.containers.get('tr-ws') - if container is not None: - container.kill() - # If the container has been started without auto-remove flag remove it - try: - container.remove() - except docker.errors.APIError: - pass - - except docker.errors.NotFound: - pass +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The overall control of the Testrun application. +This file provides the integration between all of the +Testrun components, such as net_orc, test_orc and test_ui. +""" +import docker +import ipaddress +import json +import os +import shutil +import signal +import sys +import time +import docker.errors + +from common import logger, util, mqtt +from common.device import Device +from common.testreport import TestReport +from common.statuses import TestrunStatus +from session import TestrunSession +from api.api import Api +from net_orc.listener import NetworkEvent +from net_orc import network_orchestrator as net_orc +from test_orc import test_orchestrator as test_orc + +LOGGER = logger.get_logger('testrun') + +DEFAULT_CONFIG_FILE = 'local/system.json' +EXAMPLE_CONFIG_FILE = 'local/system.json.example' + +LOCAL_DEVICES_DIR = 'local/devices' +RESOURCE_DEVICES_DIR = 'resources/devices' + +DEVICE_CONFIG = 'device_config.json' +DEVICE_MANUFACTURER = 'manufacturer' +DEVICE_MODEL = 'model' +DEVICE_MAC_ADDR = 'mac_addr' +DEVICE_TEST_MODULES = 'test_modules' +DEVICE_TYPE_KEY = 'type' +DEVICE_TECHNOLOGY_KEY = 'technology' +DEVICE_TEST_PACK_KEY = 'test_pack' +DEVICE_ADDITIONAL_INFO_KEY = 'additional_info' +DEVICE_IP_ADDR_KEY = 'ip_addr' +DEVICE_REPORT_NAME_FORMAT = '{mac_addr}_{timestamp}' + +MAX_DEVICE_REPORTS_KEY = 'max_device_reports' + +OLD_REPORTS_FOLDER = 'local/devices/{device_folder}/reports' +REPORTS_FOLDER = 'local/reports' + + +class Testrun: # pylint: disable=too-few-public-methods + """Testrun controller. + + Creates an instance of the network orchestrator, test + orchestrator and user interface. + """ + + def __init__(self, + config_file, + validate=False, + net_only=False, + single_intf=False, + no_ui=False, + target_mac=None, + firmware=None): + + # Locate parent directory + current_dir = os.path.dirname(os.path.realpath(__file__)) + + # Locate the test-run root directory, 4 levels, + # src->python->framework->test-run + self._root_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) + + # Determine config file + if config_file is None: + self._config_file = self._get_config_abs(DEFAULT_CONFIG_FILE) + else: + self._config_file = self._get_config_abs(config_file) + + self._net_only = net_only + self._single_intf = single_intf + # Network only option only works if UI is also + # disbled so need to set no_ui if net_only is selected + self._no_ui = no_ui or net_only + + # Catch any exit signals + self._register_exits() + + # Create session + self._session = TestrunSession(root_dir=self._root_dir) + + # Register runtime parameters + if single_intf: + self._session.add_runtime_param('single_intf') + if net_only: + self._session.add_runtime_param('net_only') + if validate: + self._session.add_runtime_param('validate') + + self._net_orc = net_orc.NetworkOrchestrator(session=self._session) + self._test_orc = test_orc.TestOrchestrator(self._session, self._net_orc) + + # Load device repository + self.load_all_devices() + + # If no_ui selected and not network only mode, + # load the target device into the session + if self._no_ui and not net_only: + target_device = self._session.get_device(target_mac) + if target_device is not None: + target_device.firmware = firmware + self._session.set_target_device(target_device) + else: + print( + f'Target device specified does not exist in device registry: ' + f'{target_mac}', + file=sys.stderr) + sys.exit(1) + + # Load test modules + self._test_orc.start() + + # Start websockets server + self.start_ws() + + # Init MQTT client + self._mqtt_client = mqtt.MQTT() + + if self._no_ui: + + # Check Testrun is able to start + if self.get_net_orc().check_config() is False: + return + + # Any additional checks that need to be performed go here + + self.start() + + else: + + # Start UI container + self.start_ui() + + self._api = Api(self) + self._api.start() + + # Hold until API ends + while True: + time.sleep(1) + + def get_root_dir(self): + return self._root_dir + + def get_version(self): + return self.get_session().get_version() + + def load_all_devices(self): + self._session.clear_device_repository() + self._load_devices(device_dir=LOCAL_DEVICES_DIR) + + # Temporarily removing loading of template device + # configs (feature not required yet) + # self._load_devices(device_dir=RESOURCE_DEVICES_DIR) + return self.get_session().get_device_repository() + + def _copy_existing_reports(self, device: Device): + old_reports = self._load_test_reports(device) + device.clear_reports() + if old_reports: + for report in old_reports: + timestamp = report.get_started().strftime('%Y-%m-%dT%H:%M:%S') + report_path = os.path.join(self.get_reports_folder(device), timestamp) + if os.path.exists(report_path) and os.path.isdir(report_path): + new_report_folder_name = DEVICE_REPORT_NAME_FORMAT.format( + mac_addr=device.mac_addr.replace(':', ''), + timestamp=timestamp + ) + new_report_path = os.path.join( + self.get_common_reports_folder(), new_report_folder_name + ) + try: + shutil.copytree( + report_path, + os.path.join(self.get_common_reports_folder(), new_report_path) + ) + except (FileExistsError, shutil.Error) as e: + LOGGER.error(f'Error occurred while copying report: {e}') + report.set_report_url(new_report_folder_name) + report.set_export_url(new_report_folder_name) + device.add_report(report) + self.save_device(device) + try: + shutil.rmtree( + OLD_REPORTS_FOLDER.format(device_folder=device.device_folder) + ) + except FileNotFoundError: + LOGGER.error( + f'Old reports folder not found for device {device.model}' + ) + else: + LOGGER.info('No existing reports to copy') + + def _load_devices(self, device_dir): + LOGGER.debug('Loading devices from ' + device_dir) + + util.run_command(f'chown -R {util.get_host_user()} {device_dir}') + + for device_folder in os.listdir(device_dir): + + device_config_file_path = os.path.join(device_dir, device_folder, + DEVICE_CONFIG) + + # Check if device config file exists before loading + if not os.path.exists(device_config_file_path): + LOGGER.error('Device configuration file missing ' + + f'for device {device_folder}') + continue + + # Open device config file + with open(device_config_file_path, + encoding='utf-8') as device_config_file: + + try: + device_config_json = json.load(device_config_file) + except json.decoder.JSONDecodeError as e: + LOGGER.error('Invalid JSON found in ' + + f'device configuration {device_config_file_path}') + LOGGER.debug(e) + continue + + device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) + device_model = device_config_json.get(DEVICE_MODEL) + mac_addr = device_config_json.get(DEVICE_MAC_ADDR) + test_modules = device_config_json.get(DEVICE_TEST_MODULES) + reports = device_config_json.get('reports', []) + # Load max device reports + max_device_reports = None + if 'max_device_reports' in device_config_json: + max_device_reports = device_config_json.get(MAX_DEVICE_REPORTS_KEY) + + folder_url = os.path.join(device_dir, device_folder) + + device_reports = [] + if reports: + for report in reports: + test_report = TestReport() + test_report.from_json(report) + device_reports.append(test_report) + + static_ip = device_config_json.get(DEVICE_IP_ADDR_KEY) + if static_ip is not None: + try: + ipaddress.IPv4Address(static_ip) + except ValueError: + LOGGER.error( + f'Invalid ip_addr "{static_ip}" in config for {mac_addr}; ' + 'ignoring') + static_ip = None + + device = Device(folder_url=folder_url, + manufacturer=device_manufacturer, + model=device_model, + mac_addr=mac_addr, + test_modules=test_modules, + max_device_reports=max_device_reports, + device_folder=device_folder, + reports=device_reports, + ip_addr=static_ip + ) + + # Load in the additional fields + if DEVICE_TYPE_KEY in device_config_json: + device.type = device_config_json.get(DEVICE_TYPE_KEY) + + if DEVICE_TECHNOLOGY_KEY in device_config_json: + device.technology = device_config_json.get(DEVICE_TECHNOLOGY_KEY) + + if DEVICE_TEST_PACK_KEY in device_config_json: + device.test_pack = device_config_json.get(DEVICE_TEST_PACK_KEY) + + if DEVICE_ADDITIONAL_INFO_KEY in device_config_json: + device.additional_info = device_config_json.get( + DEVICE_ADDITIONAL_INFO_KEY) + + if None in [device.type, device.technology, device.test_pack]: + LOGGER.warning( + 'Device is outdated and requires further configuration') + device.status = 'Invalid' + + if not device.get_reports(): + self._copy_existing_reports(device) + + # Add device to device repository + self.get_session().add_device(device) + LOGGER.debug(f'Loaded device {device.manufacturer} ' + + f'{device.model} with MAC address {device.mac_addr}') + + def _load_test_reports(self, device): + + LOGGER.debug('Loading test reports for device ' + + f'{device.manufacturer} {device.model}') + + # Remove the existing reports in memory + device.clear_reports() + reports = [] + # Locate reports folder + reports_folder = self.get_reports_folder(device) + + # Check if reports folder exists (device may have no reports) + if not os.path.exists(reports_folder): + return + + for report_folder in os.listdir(reports_folder): + # 1.3 file path + report_json_file_path = os.path.join(reports_folder, report_folder, + 'test', + device.mac_addr.replace(':', ''), + 'report.json') + + if not os.path.isfile(report_json_file_path): + # Revert to pre 1.3 file path + report_json_file_path = os.path.join(reports_folder, report_folder, + 'report.json') + + if not os.path.isfile(report_json_file_path): + # Revert to pre 1.3 file path + report_json_file_path = os.path.join(reports_folder, report_folder, + 'report.json') + + # Check if the report.json file exists + if not os.path.isfile(report_json_file_path): + # Some error may have occurred during this test run + continue + + with open(report_json_file_path, encoding='utf-8') as report_json_file: + report_json = json.load(report_json_file) + test_report = TestReport() + test_report.from_json(report_json) + test_report.set_mac_addr(device.mac_addr) + device.add_report(test_report) + reports.append(test_report) + return reports + + def get_reports_folder(self, device): + """Return the reports folder path for the device""" + return os.path.join(self._root_dir, LOCAL_DEVICES_DIR, device.device_folder, + 'reports') + + def get_common_reports_folder(self): + """Return the common reports folder path for all devices""" + return os.path.join(self._root_dir, REPORTS_FOLDER) + + def delete_report(self, device: Device, report: TestReport) -> bool: + LOGGER.debug(f'Deleting test report for device {device.model} ' + + f'at {report.get_folder_name()}') + + device.remove_report(report) + return True + + def create_device(self, device: Device): + + # Define the device folder location + device_folder_path = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, + device.device_folder) + + # Create the directory + os.makedirs(device_folder_path) + + config_file_path = os.path.join(device_folder_path, DEVICE_CONFIG) + + with open(config_file_path, 'w', encoding='utf-8') as config_file: + config_file.writelines(json.dumps(device.to_config_json(), indent=4)) + + # Ensure new folder has correct permissions + util.run_command(f"chown -R {util.get_host_user()} '{device_folder_path}'") + + # Add new device to the device repository + self._session.add_device(device) + + return device.to_config_json() + + def save_device(self, device: Device): + """Edit and save an existing device config.""" + + # Obtain the config file path + config_file_path = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, + device.device_folder, DEVICE_CONFIG) + + with open(config_file_path, 'w+', encoding='utf-8') as config_file: + config_file.writelines(json.dumps(device.to_config_json(), indent=4)) + + + return device.to_config_json() + + def delete_device(self, device: Device): + + # Obtain the config file path + device_folder = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, + device.device_folder) + + # Remove device reports + device.remove_reports() + + # Delete the device directory + shutil.rmtree(device_folder) + + # Remove the device from the current session device repository + self.get_session().remove_device(device) + + def start(self): + + self.get_session().start() + + self._start_network() + + self.get_net_orc().get_listener().register_callback( + self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED]) + + if self._net_only: + LOGGER.info('Network only option configured, no tests will be run') + else: + self.get_net_orc().get_listener().register_callback( + self._device_stable, [NetworkEvent.DEVICE_STABLE]) + + self.get_net_orc().start_listener() + self.get_session().set_status(TestrunStatus.WAITING_FOR_DEVICE) + LOGGER.info('Waiting for devices on the network...') + + # Keep application running until stopped + while True: + time.sleep(5) + + async def stop(self): + + # First, change the status to stopping + self.get_session().stop() + + # First, change the status to stopping + self.get_session().stop() + + # Prevent discovering new devices whilst stopping + if self.get_net_orc().get_listener() is not None: + self.get_net_orc().get_listener().stop_listener() + + self._stop_tests() + + self.get_session().set_status(TestrunStatus.CANCELLED) + + # Disconnect before WS server stops to prevent error + self._mqtt_client.disconnect() + + self._stop_network(kill=True) + + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) + + def shutdown(self): + LOGGER.info('Shutting down Testrun') + self.stop() + self._stop_ui() + self._stop_ws() + + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug('Exit signal received: ' + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info('Exit signal received.') + self.shutdown() + sys.exit(1) + + def _get_config_abs(self, config_file=None): + if config_file is None: + # If not defined, use relative pathing to local file + config_file = os.path.join(self._root_dir, self._config_file) + + # Expand the config file to absolute pathing + return os.path.abspath(config_file) + + def get_config_file(self): + return self._get_config_abs() + + def get_net_orc(self): + return self._net_orc + + def get_test_orc(self): + return self._test_orc + + def _start_network(self): + # Start the network orchestrator + if not self.get_net_orc().start(): + self.stop() + sys.exit(1) + + def _stop_network(self, kill=True): + self.get_net_orc().stop(kill) + + def _stop_tests(self): + self._test_orc.stop() + + def get_mqtt_client(self): + return self._mqtt_client + + def get_device(self, mac_addr): + """Returns a loaded device object from the device mac address.""" + for device in self.get_session().get_device_repository(): + if device.mac_addr == mac_addr: + return device + return None + + def _device_discovered(self, mac_addr): + + device = self.get_session().get_target_device() + + if device is not None: + if mac_addr != device.mac_addr: + msg_found = f'Found device with mac addr: {mac_addr} but was ignored' + LOGGER.info(msg_found) + msg_expected = f'Expected device mac address is {device.mac_addr}' + LOGGER.info(msg_expected) + full_message = f'{msg_found}\n{msg_expected}' + self._mqtt_client.send_message( + mqtt.MQTTTopic.INFO, {'message': full_message}) + # Ignore discovered device because it is not the target device + return + else: + device = self.get_device(mac_addr) + if device is None: + return + + self.get_session().set_target_device(device) + + LOGGER.info( + f'Discovered {device.manufacturer} {device.model} on the network. ' + + 'Waiting for device to obtain IP') + + def _device_stable(self, mac_addr): + + # Do not continue testing if Testrun has cancelled during monitor phase + if self.get_session().get_status() == TestrunStatus.CANCELLED: + self._stop_network() + return + + LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') + self._set_status(TestrunStatus.IN_PROGRESS) + + # Start testrun timer + self.get_session().start_timer() + + self._test_orc.run_test_modules() + + self._stop_network() + + def get_session(self): + return self._session + + def _set_status(self, status): + self.get_session().set_status(status) + + def start_ui(self): + + self._stop_ui() + + LOGGER.info('Starting UI') + + client = docker.from_env() + + try: + client.containers.run(image='testrun/ui', + auto_remove=True, + name='tr-ui', + hostname='testrun.io', + detach=True, + ports={'80': 8080}) + except docker.errors.ImageNotFound as ie: + LOGGER.error('An error occurred whilst starting the UI. ' + + 'Please investigate and try again.') + LOGGER.error(ie) + sys.exit(1) + + # TODO: Make port configurable + LOGGER.info('User interface is ready on http://localhost:8080') + + def _stop_ui(self): + LOGGER.info('Stopping user interface') + client = docker.from_env() + try: + container = client.containers.get('tr-ui') + if container is not None: + container.kill() + # If the container has been started without auto-remove flag remove it + try: + container.remove() + except docker.errors.APIError: + pass + except docker.errors.NotFound: + pass + + def start_ws(self): + + self._stop_ws() + + LOGGER.info('Starting WS server') + + client = docker.from_env() + + try: + client.containers.run(image='testrun/ws', + auto_remove=True, + name='tr-ws', + detach=True, + ports={ + '9001': 9001, + '1883': 1883 + }) + except docker.errors.ImageNotFound as ie: + LOGGER.error('An error occurred whilst starting the websockets server. ' + + 'Please investigate and try again.') + LOGGER.error(ie) + sys.exit(1) + + def _stop_ws(self): + LOGGER.info('Stopping websockets server') + client = docker.from_env() + try: + container = client.containers.get('tr-ws') + if container is not None: + container.kill() + # If the container has been started without auto-remove flag remove it + try: + container.remove() + except docker.errors.APIError: + pass + + except docker.errors.NotFound: + pass diff --git a/framework/requirements.txt b/framework/requirements.txt index aafda3035..5361caef1 100644 --- a/framework/requirements.txt +++ b/framework/requirements.txt @@ -1,5 +1,5 @@ # Requirements for the core module -requests==2.32.5 +requests==2.33.0 # Requirements for the net_orc module docker==7.1.0 @@ -15,7 +15,7 @@ pydyf==0.11.0 fastapi==0.115.0 psutil==5.9.8 uvicorn==0.27.0 -python-multipart==0.0.19 +python-multipart==0.0.22 pydantic==2.10.0 # Requirements for testing @@ -28,7 +28,7 @@ responses==0.25.3 markdown==3.5.2 # Requirements for the session -cryptography==44.0.1 +cryptography==46.0.6 pytz==2024.1 # Requirements for the risk profile @@ -43,6 +43,4 @@ APScheduler==3.10.4 # Requirements for reports generation Jinja2==3.1.6 beautifulsoup4==4.12.3 - -# linting -pylint==3.2.6 +html5lib==1.1