diff --git a/requirements.txt b/requirements.txt index f8dca51..7b07f1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ httpx==0.27.0 +matplotlib==3.9.2 pydantic==2.7.1 # reasoner_pydantic==4.1.6 setproctitle==1.3.3 diff --git a/setup.py b/setup.py index 1f436ff..6b2e19a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="sri-test-harness", - version="0.6.4", + version="0.6.5", author="Max Wang", author_email="max@covar.com", url="https://github.com/TranslatorSRI/TestHarness", diff --git a/test_harness/main.py b/test_harness/main.py index de77933..dc323a0 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -97,6 +97,11 @@ def main(args): "json", collector.performance_stats, ) + for filename, content in collector.render_performance_artifacts(): + try: + slacker.upload_binary_file(filename, content) + except Exception as e: + logger.warning(f"Failed to upload perf artifact {filename}: {e}") logger.info("Finishing up test run...") reporter.finish_test_run() diff --git a/test_harness/perf_plots.py b/test_harness/perf_plots.py new file mode 100644 index 0000000..7178716 --- /dev/null +++ b/test_harness/perf_plots.py @@ -0,0 +1,120 @@ +"""Render Locust-style time-series charts from a stats_history snapshot.""" + +from __future__ import annotations + +import io +import logging +from datetime import datetime +from typing import Any, Dict, List, Optional, Sequence, Tuple + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.dates as mdates +import matplotlib.pyplot as plt + +logger = logging.getLogger(__name__) + + +def _parse_timestamp(value: Any) -> Optional[datetime]: + if isinstance(value, (int, float)): + return datetime.utcfromtimestamp(value) + if isinstance(value, str): + # Locust uses ISO 8601 with a trailing 'Z' for UTC. + text = value.rstrip("Z") + try: + return datetime.fromisoformat(text) + except ValueError: + return None + return None + + +def _series( + history: Sequence[Dict[str, Any]], key: str +) -> Tuple[List[datetime], List[float]]: + """Pull a (times, values) pair out of one stats_history series. + + Each entry's value is a ``[timestamp, value]`` pair, but we prefer the + snapshot-level ``time`` so all series share the same X axis. + """ + times: List[datetime] = [] + values: List[float] = [] + for row in history: + ts = _parse_timestamp(row.get("time")) + entry = row.get(key) + if ts is None or not isinstance(entry, (list, tuple)) or len(entry) < 2: + continue + value = entry[1] + if value is None: + continue + times.append(ts) + values.append(float(value)) + return times, values + + +def _percentile_keys(history: Sequence[Dict[str, Any]]) -> List[str]: + """Return percentile series keys present in the snapshots, sorted.""" + keys = set() + for row in history: + for key in row.keys(): + if key.startswith("response_time_percentile_"): + keys.add(key) + return sorted(keys, key=lambda k: float(k.split("_")[-1])) + + +def render_history_png(history: Sequence[Dict[str, Any]], title: str) -> bytes: + """Render a Locust-style three-panel chart for the given history. + + Panels (top to bottom): + 1. Total RPS, with failures/sec overlaid. + 2. Response time percentiles in milliseconds. + 3. Active user count. + + Returns the PNG bytes. Caller is responsible for handling empty history; + we raise if there isn't enough data to plot. + """ + if len(history) < 2: + raise ValueError("history too short to plot") + + fig, (ax_rps, ax_rt, ax_users) = plt.subplots( + nrows=3, ncols=1, figsize=(10, 12), sharex=True + ) + + rps_times, rps_values = _series(history, "current_rps") + fail_times, fail_values = _series(history, "current_fail_per_sec") + ax_rps.plot(rps_times, rps_values, color="#2ca02c", label="RPS") + ax_rps.plot(fail_times, fail_values, color="#d62728", label="Failures/s") + ax_rps.set_ylabel("Requests / s") + ax_rps.set_title("Total Requests per Second") + ax_rps.grid(True, alpha=0.3) + ax_rps.legend(loc="upper right") + + percentile_colors = ["#1f77b4", "#ff7f0e", "#9467bd", "#8c564b"] + for color, key in zip(percentile_colors, _percentile_keys(history)): + times, values = _series(history, key) + label = "p" + key.split("_")[-1].replace("0.", "") + ax_rt.plot(times, values, color=color, label=label) + ax_rt.set_ylabel("Milliseconds") + ax_rt.set_title("Response Times") + ax_rt.grid(True, alpha=0.3) + ax_rt.legend(loc="upper right") + + user_times, user_values = _series(history, "user_count") + ax_users.step(user_times, user_values, color="#1f77b4", where="post") + ax_users.set_ylabel("Users") + ax_users.set_title("Number of Users") + ax_users.set_xlabel("Time (UTC)") + ax_users.grid(True, alpha=0.3) + + ax_users.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M:%S")) + fig.autofmt_xdate() + fig.suptitle(title) + fig.tight_layout(rect=(0, 0, 1, 0.97)) + + buf = io.BytesIO() + try: + fig.savefig(buf, format="png", dpi=110, bbox_inches="tight") + finally: + plt.close(fig) + return buf.getvalue() diff --git a/test_harness/performance_test_runner.py b/test_harness/performance_test_runner.py index 55675c3..03f4ea0 100644 --- a/test_harness/performance_test_runner.py +++ b/test_harness/performance_test_runner.py @@ -2,12 +2,13 @@ import logging import time -from typing import Dict +from typing import Dict, List import gevent from gevent import GreenletExit from locust import HttpUser, LoadTestShape, task from locust.env import Environment +from locust.html import get_html_report from locust.stats import stats_history, stats_printer from translator_testing_model.datamodel.pydanticmodel import ( AcceptanceTestAsset, @@ -20,7 +21,6 @@ from test_harness.runner.generate_query import generate_query from test_harness.runner.query_runner import QueryRunner, env_map - # Custom request_type values used to distinguish layers of the test in stats. # Locust groups stats by (method, name); using these as the "method" lets us # pull each layer out cleanly in the result collector. @@ -32,6 +32,10 @@ # one row per parent_pk. SUBMIT_NAME = "submit_query" POLL_NAME = "poll_status" +# The /trace poll only returns status metadata; the final TRAPI message lives +# at the merged_version PK. Fetch it on completion so we can record the size +# of the actual response. +MERGED_FETCH_NAME = "fetch_merged" # Per-outcome names for the end-to-end QUERY event. Distinct names give us # a count for each outcome directly out of Locust's stats serialization. @@ -106,9 +110,7 @@ def send_query(self): len(response.content) if response.content else 0 ) else: - failure_reason = ( - f"Got a bad response: {response.status_code}" - ) + failure_reason = f"Got a bad response: {response.status_code}" response.failure(failure_reason) except GreenletExit: outcome = ARA_QUERY_FAILED @@ -125,6 +127,29 @@ def send_query(self): ) class ARSUser(HttpUser): + def _fetch_merged_size(self, merged_pk, trace_response) -> int: + """Pull the actual final-response byte size for a completed query. + + Falls back to the trace response's content length if the merged + message can't be fetched, so the QUERY event still records a size. + """ + fallback = len(trace_response.content) if trace_response.content else 0 + if not merged_pk: + return fallback + with self.client.get( + f"/ars/api/messages/{merged_pk}", + catch_response=True, + name=MERGED_FETCH_NAME, + ) as merged_res: + if merged_res.status_code != 200: + merged_res.failure( + f"Failed to fetch merged {merged_pk}: " + f"{merged_res.status_code}" + ) + return fallback + merged_res.success() + return len(merged_res.content) if merged_res.content else 0 + @task def send_query(self): query_started = time.time() @@ -162,9 +187,7 @@ def send_query(self): while True: if remaining_test_time() <= 0: outcome = OUTCOME_ABANDONED - failure_reason = ( - f"Test ended while polling {parent_pk}" - ) + failure_reason = f"Test ended while polling {parent_pk}" return with self.client.get( @@ -185,9 +208,7 @@ def send_query(self): try: res = response.json() except ValueError: - failure_reason = ( - f"Non-JSON poll body for {parent_pk}" - ) + failure_reason = f"Non-JSON poll body for {parent_pk}" outcome = OUTCOME_POLLING_FAILED return @@ -195,10 +216,9 @@ def send_query(self): if status == "Done": outcome = OUTCOME_COMPLETED failure_reason = None - response_length = ( - len(response.content) - if response.content - else 0 + response_length = self._fetch_merged_size( + res.get("merged_version"), + response, ) return if status == "Error": @@ -210,9 +230,7 @@ def send_query(self): sleep_for = min(POLL_INTERVAL_SECONDS, remaining_test_time()) if sleep_for <= 0: outcome = OUTCOME_ABANDONED - failure_reason = ( - f"Test ended while polling {parent_pk}" - ) + failure_reason = f"Test ended while polling {parent_pk}" return time.sleep(sleep_for) except GreenletExit: @@ -244,6 +262,20 @@ def send_query(self): env = Environment(user_classes=[user_class], host=host, shape_class=TestShape()) runner = env.create_local_runner() + # Capture per-query response sizes (one entry per query, keyed by + # outcome name) so the report can flag cases where queries reported the + # same status but came back with different payload sizes. + query_response_sizes: Dict[str, List[int]] = {} + + def _record_query_size( + request_type, name, response_time, response_length, exception, context, **kwargs + ): + if request_type != QUERY_TYPE: + return + query_response_sizes.setdefault(name, []).append(response_length or 0) + + env.events.request.add_listener(_record_query_size) + # Start stats printer gevent.spawn(stats_printer(env.stats)) gevent.spawn(stats_history, runner) @@ -260,12 +292,23 @@ def send_query(self): print("Done with locust testing!") + try: + summary_html = get_html_report(env, show_download_link=False) + except Exception as e: + logging.getLogger(__name__).warning( + "Failed to render Locust HTML report: %s", e + ) + summary_html = None + return { "stats": env.stats.serialize_stats(), "failures": env.stats.serialize_errors(), "test_run_time": test_run_time, "spawn_rate": spawn_rate, "target": target, + "query_response_sizes": query_response_sizes, + "stats_history": list(env.runner.stats.history), + "summary_html": summary_html, } diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index 0392e58..cff10d2 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -1,7 +1,9 @@ """The Collector of Results.""" import logging -from typing import Dict, Iterable, List, Optional, Union +import re +from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union +from urllib.parse import urlparse from translator_testing_model.datamodel.pydanticmodel import ( PathfinderTestAsset, @@ -11,9 +13,9 @@ TestEnvEnum, ) +from test_harness import perf_plots from test_harness.utils import AgentStatus, TestReport - # Stat row identifiers produced by the performance test runner. Kept in sync # with the constants in performance_test_runner.py. QUERY_TYPE = "QUERY" @@ -74,17 +76,15 @@ def _summarize_layer(stat: Optional[Dict]) -> Dict: num_none = stat.get("num_none_requests", 0) measured = max(0, num_requests - num_none) response_times = stat.get("response_times", {}) or {} - duration = ( - stat.get("last_request_timestamp", 0) or 0 - ) - (stat.get("start_time", 0) or 0) + duration = (stat.get("last_request_timestamp", 0) or 0) - ( + stat.get("start_time", 0) or 0 + ) return { "num_requests": num_requests, "num_failures": stat.get("num_failures", 0), "min_response_time": stat.get("min_response_time") or 0, "max_response_time": stat.get("max_response_time", 0), - "avg_response_time": _safe_div( - stat.get("total_response_time", 0), measured - ), + "avg_response_time": _safe_div(stat.get("total_response_time", 0), measured), "median_response_time": percentile_from_dict(measured, response_times, 0.5), "p95_response_time": percentile_from_dict(measured, response_times, 0.95), "requests_per_second": _safe_div(num_requests, duration), @@ -104,9 +104,7 @@ def _find_stat( return None -def _summarize_query_lifecycle( - stats: List[Dict], outcome_names: Iterable[str] -) -> Dict: +def _summarize_query_lifecycle(stats: List[Dict], outcome_names: Iterable[str]) -> Dict: """Aggregate end-to-end QUERY events across all outcomes.""" total_requests = 0 total_response_time = 0.0 @@ -136,7 +134,9 @@ def _summarize_query_lifecycle( completed_name = next( (name for name in outcome_names if name.endswith("_completed")), None ) - completed_stat = _find_stat(stats, completed_name, QUERY_TYPE) if completed_name else None + completed_stat = ( + _find_stat(stats, completed_name, QUERY_TYPE) if completed_name else None + ) completed_buckets = (completed_stat or {}).get("response_times", {}) or {} completed_count = (completed_stat or {}).get("num_requests", 0) completed_total_rt = (completed_stat or {}).get("total_response_time", 0) or 0 @@ -165,11 +165,39 @@ def _summarize_query_lifecycle( completed_count, completed_buckets, 0.95 ), "min_response_time": (completed_stat or {}).get("min_response_time") or 0, - "max_response_time": (completed_stat or {}).get("max_response_time", 0) or 0, + "max_response_time": (completed_stat or {}).get("max_response_time", 0) + or 0, }, } +def _slugify_host(host_url: str) -> str: + """Make a filesystem/Slack-friendly slug for a host URL.""" + netloc = urlparse(host_url).netloc or host_url + return re.sub(r"[^A-Za-z0-9._-]+", "_", netloc).strip("_") or "perf" + + +def _summarize_response_sizes( + sizes_by_outcome: Dict[str, List[int]], outcome_names: Iterable[str] +) -> Dict[str, Dict]: + """Per-outcome response-size summary, including distinct-size count so + the report can flag queries that finished with the same status but came + back with different payloads (eg an error body in place of TRAPI).""" + summary: Dict[str, Dict] = {} + for name in outcome_names: + sizes = sizes_by_outcome.get(name) or [] + if not sizes: + continue + summary[name] = { + "count": len(sizes), + "min": min(sizes), + "max": max(sizes), + "avg": sum(sizes) / len(sizes), + "distinct": len(set(sizes)), + } + return summary + + class ResultCollector: """Collect results for easy dissemination.""" @@ -196,10 +224,7 @@ def __init__(self, test_env: Optional[TestEnvEnum], logger: logging.Logger): ] self.agents = agents self.query_types = ["TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"] - self.acceptance_report = { - status_type.value: 0 - for status_type in AgentStatus - } + self.acceptance_report = {status_type.value: 0 for status_type in AgentStatus} self.acceptance_stats = {} for agent in self.agents: self.acceptance_stats[agent] = {} @@ -242,7 +267,8 @@ def collect_acceptance_result( agent_results = ",".join(agent_statuses) pk_url = ( f"https://arax.ci.transltr.io/?r={parent_pk}" - if parent_pk is not None else "" + if parent_pk is not None + else "" ) self.acceptance_csv += ( f""""{asset.name}",{url},{pk_url},{test.id},{asset.id},{agent_results}\n""" @@ -273,6 +299,11 @@ def collect_performance_result( "submit": _summarize_layer(submit_stat), "poll": _summarize_layer(poll_stat) if target == "ars" else None, "queries": lifecycle, + "response_sizes": _summarize_response_sizes( + results.get("query_response_sizes") or {}, outcome_names + ), + "history": results.get("stats_history") or [], + "summary_html": results.get("summary_html"), } self.performance_report["failures"] = results.get("failures") or {} @@ -282,6 +313,40 @@ def collect_performance_result( **results, } + def render_performance_artifacts(self) -> Iterator[Tuple[str, bytes]]: + """Yield (filename, bytes) tuples for per-target performance artifacts. + + Produces up to two files per host: + * ``_perf.png`` - matplotlib chart of stats_history + * ``_perf.html`` - Locust's own HTML report + Renderable artifacts are skipped (with a log line) when data is + missing; render exceptions are caught so one bad target doesn't + block the rest. + """ + for host_url, target_stats in self.performance_report["stats"].items(): + slug = _slugify_host(host_url) + + history = target_stats.get("history") or [] + if len(history) >= 2: + try: + png_bytes = perf_plots.render_history_png(history, title=host_url) + except Exception as e: + self.logger.warning( + f"Failed to render perf chart for {host_url}: {e}" + ) + else: + yield f"{slug}_perf.png", png_bytes + else: + self.logger.info( + f"Skipping perf chart for {host_url}: insufficient history" + ) + + summary_html = target_stats.get("summary_html") + if summary_html: + yield f"{slug}_perf.html", summary_html.encode("utf-8") + else: + self.logger.info(f"Skipping HTML report for {host_url}: not available") + def dump_result_summary(self): """Format test results summary for Slack.""" results_formatted = "" @@ -301,15 +366,15 @@ def dump_result_summary(self): results_formatted += self._format_performance_target( target_url, target_stats ) - if len(self.performance_report["failures"].keys()): - results_formatted += "\n> Failures:" - for failure_stat in self.performance_report["failures"].values(): - results_formatted += ( - f"\n> ---" - f"\n> {failure_stat.get('name', 'Unknown')}" - f"\n> {failure_stat.get('error', 'Unknown Error')}" - f"\n> occurrences: {failure_stat.get('occurrences', 0)}" - ) + failures = self.performance_report["failures"] + if failures: + total_occurrences = sum( + f.get("occurrences", 0) for f in failures.values() + ) + results_formatted += ( + f"\n> Failures: {total_occurrences} " + f"({len(failures)} distinct) - see uploaded HTML report" + ) return results_formatted @@ -332,19 +397,33 @@ def _format_performance_target(target_url: str, target_stats: Dict) -> str: by_outcome = queries.get("by_outcome", {}) or {} completed = queries.get("completed_only") or {} all_outcomes = queries.get("all_outcomes") or {} + response_sizes = target_stats.get("response_sizes") or {} run_time_seconds = run_time or 0 lines.append("> - End-to-end queries:") lines.append(f"> * Submitted (recorded): {total_queries}") for name, count in by_outcome.items(): label = name.replace("ars_query_", "").replace("ara_query_", "") - lines.append(f"> * {label}: {count}") + line = f"> * {label}: {count}" + size_summary = response_sizes.get(name) + if size_summary: + line += ( + f" [response size bytes: " + f"min={size_summary['min']}, " + f"max={size_summary['max']}, " + f"avg={size_summary['avg']:.0f}, " + f"distinct={size_summary['distinct']}]" + ) + lines.append(line) + if size_summary and size_summary["distinct"] > 1: + lines.append( + f"> WARNING: {label} responses differ in size; " + "check for partial or error payloads" + ) completed_count = completed.get("count", 0) if run_time_seconds: throughput = completed_count / (run_time_seconds / 60.0) - lines.append( - f"> * Completed throughput: {throughput:.2f} queries/minute" - ) + lines.append(f"> * Completed throughput: {throughput:.2f} queries/minute") if completed_count: lines.append( "> * Completed query time (s): " diff --git a/test_harness/run.py b/test_harness/run.py index 6bb3570..58e2d3e 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -89,7 +89,9 @@ def run_tests( result={}, test_details=None, ) - if isinstance(test, PathfinderTestCase) and isinstance(asset, PathfinderTestAsset): + if isinstance(test, PathfinderTestCase) and isinstance( + asset, PathfinderTestAsset + ): report.test_details = { "minimum_required_path_nodes": asset.minimum_required_path_nodes, "expected_path_nodes": "; ".join( @@ -142,7 +144,9 @@ def run_tests( agent_report.status = AgentStatus.NO_RESULTS agent_report.message = "No results" continue - if isinstance(test, PathfinderTestCase) and isinstance(asset, PathfinderTestAsset): + if isinstance(test, PathfinderTestCase) and isinstance( + asset, PathfinderTestAsset + ): pathfinder_pass_fail_analysis( report.result, agent, @@ -161,7 +165,11 @@ def run_tests( report.result, agent, response["response"]["message"]["results"], - normalized_curies.get(asset.output_id, "") if asset.output_id is not None else "", + ( + normalized_curies.get(asset.output_id, "") + if asset.output_id is not None + else "" + ), asset.expected_output, ) except Exception as e: @@ -199,6 +207,7 @@ def run_tests( reporter.upload_labels(test_id, labels) except Exception as e: logger.warning(f"[{test.id}] failed to upload labels: {e}") + logger.info(f"Full report: {json.dumps(asdict(report), indent=4)}") reporter.upload_log(test_id, json.dumps(asdict(report), indent=4)) else: status = AgentStatus.SKIPPED diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 8deae8e..f80968b 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -86,12 +86,17 @@ def get_ars_child_response( child_pk: str, base_url: str, infores: str, - start_time: float, ): - """Given a child pk, get response from ARS.""" + """Given a child pk, get response from ARS. + + Each call gets its own poll deadline so that one slow / timed-out ARA + doesn't cause subsequently checked ARAs to be marked as timed out + before they're even given a chance to respond. + """ self.logger.info(f"Getting response for {infores}...") - current_time = time.time() + start_time = time.time() + current_time = start_time response = None status = 500 @@ -174,7 +179,7 @@ def get_ars_responses( # add child pk pks[infores] = child_pk child_responses.append( - self.get_ars_child_response(child_pk, base_url, infores, start_time) + self.get_ars_child_response(child_pk, base_url, infores) ) for child_response in child_responses: diff --git a/test_harness/slacker.py b/test_harness/slacker.py index d470c57..bc6d9c0 100644 --- a/test_harness/slacker.py +++ b/test_harness/slacker.py @@ -1,12 +1,42 @@ """Slack notification integration class.""" import json +import logging import os import tempfile import httpx from slack_sdk import WebClient +# Slack rejects section blocks whose text exceeds 3000 chars. Leave a small +# safety margin so we never end up at the boundary. +SLACK_SECTION_TEXT_LIMIT = 2900 + + +def _chunk_text(text, limit=SLACK_SECTION_TEXT_LIMIT): + """Split text into chunks no larger than ``limit`` chars, preferring + newline boundaries so quoted-block formatting stays intact.""" + if len(text) <= limit: + return [text] + chunks = [] + current = "" + for line in text.split("\n"): + candidate = f"{current}\n{line}" if current else line + if len(candidate) <= limit: + current = candidate + continue + if current: + chunks.append(current) + current = "" + # A single line longer than the limit (rare) - hard split it. + while len(line) > limit: + chunks.append(line[:limit]) + line = line[limit:] + current = line + if current: + chunks.append(current) + return chunks + class Slacker: """Slack notification poster.""" @@ -18,21 +48,23 @@ def __init__(self, url=None, token=None, slack_channel=None): self.url = url if url is not None else os.getenv("SLACK_WEBHOOK_URL") slack_token = token if token is not None else os.getenv("SLACK_TOKEN") self.client = WebClient(slack_token) + self.logger = logging.getLogger(__name__) def post_notification(self, messages=[]): """Post a notification to Slack.""" # https://gist.github.com/mrjk/079b745c4a8a118df756b127d6499aa0 blocks = [] for message in messages: - blocks.append( - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": str(message), - }, - } - ) + for chunk in _chunk_text(str(message)): + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": chunk, + }, + } + ) with httpx.Client() as client: res = client.post( url=self.url, @@ -41,6 +73,12 @@ def post_notification(self, messages=[]): "blocks": blocks, }, ) + if res.status_code >= 300: + self.logger.warning( + "Slack webhook rejected notification: %s %s", + res.status_code, + res.text, + ) def upload_test_results_file(self, filename, extension, results): """Upload a results file to Slack.""" @@ -57,3 +95,16 @@ def upload_test_results_file(self, filename, extension, results): file=tmp_path, initial_comment="Test Results:", ) + + def upload_binary_file(self, filename, content, initial_comment=None, title=None): + """Upload a binary file (PNG, HTML, etc.) to Slack.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = os.path.join(td, filename) + with open(tmp_path, "wb") as f: + f.write(content) + self.client.files_upload_v2( + channel=self.channel, + title=title or filename, + file=tmp_path, + initial_comment=initial_comment or "Performance report:", + ) diff --git a/tests/test_run.py b/tests/test_run.py index d5ad5e3..39948ee 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -49,7 +49,7 @@ def test_run_tests(mocker, httpx_mock: HTTPXMock): reporter=MockReporter( base_url="http://test", ), - collector=MockResultCollector(logger), + collector=MockResultCollector("dev", logger), logger=logger, args={ "suite": "testing",