diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4acbd4..c0515b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.13'] steps: - uses: actions/checkout@v4 @@ -24,16 +24,19 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" - name: Install dependencies run: uv sync --python ${{ matrix.python-version }} - name: Run tests - run: uv run pytest tests/ -v --cov=agentchaos --cov-report=xml + run: uv run pytest tests/ -x --cov=agentchaos --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 - if: matrix.python-version == '3.12' + if: matrix.python-version == '3.13' with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml @@ -51,6 +54,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" - name: Install dependencies run: uv sync diff --git a/.gitignore b/.gitignore index 0a6fe04..30bfe8f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ Thumbs.db # Testing .pytest_cache/ +.coverage coverage.xml htmlcov/ .deepeval/ diff --git a/pyproject.toml b/pyproject.toml index bd76541..04095fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ select = ["E", "F", "W", "I"] [tool.coverage.run] source = ["agentchaos"] +omit = ["agentchaos/__main__.py"] [tool.coverage.report] exclude_lines = [ diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..3628e31 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,172 @@ +# tests/test_config.py — Unit tests for fault_config (pure in-memory catalog validation) +import json +import os +import tempfile + +import agentchaos +from agentchaos import EvalReport, EvalResult, fault_config +from agentchaos.fault_engine import FaultSpec + + +def test_total_65_experiments(): + assert len(fault_config.EXPERIMENTS) == 65 + + +def test_all_experiments_have_name_and_faults(): + for exp in fault_config.EXPERIMENTS: + assert "name" in exp + assert "faults" in exp + assert len(exp["faults"]) >= 1 + + +def test_all_faults_are_faultspec(): + for exp in fault_config.EXPERIMENTS: + for spec in exp["faults"]: + assert isinstance(spec, FaultSpec) + + +def test_strategies_coverage(): + names = fault_config.list_all() + for strategy in ["single", "persistent", "intermittent", "burst"]: + matching = [n for n in names if n.endswith(f"_{strategy}")] + assert len(matching) == 12 # 6 llm + 6 tool + + +def test_compound_experiments(): + names = fault_config.list_all() + compounds = [n for n in names if n.startswith("compound_")] + assert len(compounds) == 8 + + +def test_positional_experiments(): + names = fault_config.list_all() + positionals = [n for n in names if "_pos_" in n] + assert len(positionals) == 9 + + +def test_get_valid(): + exp = fault_config.get("llm_error_single") + assert exp["name"] == "llm_error_single" + assert len(exp["faults"]) == 1 + assert exp["faults"][0].action == "set" + + +def test_get_invalid_raises(): + try: + fault_config.get("nonexistent_fault") + assert False, "Should have raised ValueError" + except ValueError as e: + assert "nonexistent_fault" in str(e) + + +def test_list_all_sorted(): + names = fault_config.list_all() + assert names == sorted(names) + + +def test_list_by_category_structure(): + cats = fault_config.list_by_category() + assert "llm" in cats + assert "tool" in cats + assert "compound" in cats + assert len(cats["llm"]) == 6 + assert len(cats["tool"]) == 6 + assert len(cats["compound"]) == 8 + + +def test_list_by_category_has_experiments(): + cats = fault_config.list_by_category() + for cat_name, items in cats.items(): + for item in items: + assert "base_name" in item + assert "type" in item + assert "description" in item + assert "experiments" in item + + +def test_each_experiment_retrievable(): + for name in fault_config.list_all(): + exp = fault_config.get(name) + assert exp is not None + assert exp["name"] == name + + +# ── __init__.py coverage: EvalReport / save_trace branches ──────── + + +def test_eval_report_properties(): + r1 = EvalResult( + fault="f1", + result="ok", + error="", + passed=True, + elapsed=0.1, + faults_fired=0, + diagnosis={}, + fault_log=[], + ) + r2 = EvalResult( + fault="f2", + result="", + error="err", + passed=False, + elapsed=0.2, + faults_fired=1, + diagnosis={}, + fault_log=[], + ) + report = EvalReport(total=2, passed=1, failed=1, results=[r1, r2]) + assert report.pass_rate == 0.5 + assert report.vulnerable_to == ["f2"] + assert "1/2" in report.summary() + + +def test_eval_report_empty(): + report = EvalReport(total=0, passed=0, failed=0, results=[]) + assert report.pass_rate == 0.0 + assert report.vulnerable_to == [] + assert "none" in report.summary() + + +def test_save_trace_with_fault_data(): + """save_trace correctly writes injected_output when fault was applied.""" + engine = agentchaos.inject("llm_error_single") + # manually add a trace entry to simulate a faulted call + engine.trace.append( + { + "call_index": 0, + "request": {"model": "test", "messages": [], "tools": []}, + "response": { + "content": "original", + "tool_calls": [], + "finish_reason": "stop", + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + "http_status": 200, + "modified_content": "[ERROR] injected", + "modified_tool_calls": [], + }, + "timing": {"llm_latency_ms": 100, "total_ms": 101}, + "fault": {"applied": True}, + } + ) + agentchaos.disable() + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + agentchaos.save_trace(path) + with open(path) as f: + data = json.load(f) + assert len(data) == 1 + assert data[0]["fault_applied"] is True + assert "injected_output" in data[0] + assert data[0]["injected_output"]["content"] == "[ERROR] injected" + assert data[0]["raw_output"]["content"] == "original" + finally: + os.unlink(path) + + +def test_save_trace_no_engine(): + """save_trace with no prior engine does not crash.""" + agentchaos._last_engine = None + agentchaos.save_trace("/tmp/test_no_engine.json") # should log error, not crash diff --git a/tests/test_diagnose.py b/tests/test_diagnose.py new file mode 100644 index 0000000..f8cae9e --- /dev/null +++ b/tests/test_diagnose.py @@ -0,0 +1,134 @@ +# tests/test_diagnose.py — Unit tests for fault_diagnose (pure string matching, no network) +from agentchaos.fault_diagnose import FAULT_TYPES, REPAIR_HINTS, diagnose + +# ── Empty / None ────────────────────────────────────────────────── + + +def test_diagnose_empty_string(): + r = diagnose("") + assert r["fault_type"] == "empty" + assert r["confidence"] == "high" + + +def test_diagnose_none_input(): + r = diagnose(None) + assert r["fault_type"] == "empty" + + +def test_diagnose_whitespace_only(): + r = diagnose(" \n ") + assert r["fault_type"] == "empty" + + +# ── Error patterns ──────────────────────────────────────────────── + + +def test_diagnose_api_error(): + r = diagnose("[API ERROR] HTTP 500: Internal Server Error.") + assert r["fault_type"] == "error" + + +def test_diagnose_http_500(): + r = diagnose("HTTP 500 Internal Server Error occurred") + assert r["fault_type"] == "error" + + +# ── Timeout patterns ───────────────────────────────────────────── + + +def test_diagnose_timeout(): + r = diagnose("[TIMEOUT] The request timed out. The server did not respond within the expected time.") + assert r["fault_type"] == "timeout" + + +def test_diagnose_connection_dropped(): + r = diagnose("Connection dropped. The server did not respond.") + assert r["fault_type"] == "timeout" + + +# ── Compound patterns ──────────────────────────────────────────── + + +def test_diagnose_service_unavailable(): + r = diagnose("[SERVICE UNAVAILABLE] HTTP 503: The server is temporarily unable to handle the request.") + assert r["fault_type"] == "compound" + + +def test_diagnose_content_filtered(): + r = diagnose("[CONTENT FILTERED] This response has been blocked by the content safety filter.") + assert r["fault_type"] == "compound" + + +def test_diagnose_502_bad_gateway(): + r = diagnose("502 Bad Gateway nginx/1.24.0") + assert r["fault_type"] == "compound" + + +# ── Schema patterns ────────────────────────────────────────────── + + +def test_diagnose_content_policy_violation(): + r = diagnose('{"error": "content_policy_violation", "message": "This response has been filtered."}') + assert r["fault_type"] == "schema" + + +# ── Corrupt patterns ───────────────────────────────────────────── + + +def test_diagnose_unicode_corruption(): + # 4 consecutive symbols from Miscellaneous Symbols block + r = diagnose("Hello \u2600\u2601\u2602\u2603\u2604 world") + assert r["fault_type"] == "corrupt" + + +def test_diagnose_mojibake(): + # simulate UTF-8 bytes misread as Latin-1 + original = "Hello world" + mojibake = original.encode("utf-8").decode("latin-1") + # add multiple mojibake patterns + text = f"{mojibake} â\x80\x99 â\x80\x9c some text" + r = diagnose(text) + assert r["fault_type"] == "corrupt" + + +# ── Truncation detection ───────────────────────────────────────── + + +def test_diagnose_truncated_text(): + # text ending mid-word without punctuation + r = diagnose("The answer to the question about prime numbers is that we need to check divisibili") + assert r["fault_type"] == "truncate" + assert r["confidence"] == "medium" + + +def test_diagnose_not_truncated_with_period(): + r = diagnose("The answer is 42.") + assert r["fault_type"] == "unknown" + + +def test_diagnose_not_truncated_short(): + r = diagnose("short") + assert r["fault_type"] == "unknown" + + +# ── Normal text → unknown ──────────────────────────────────────── + + +def test_diagnose_normal_text(): + r = diagnose("The answer is 42. This is a complete response.") + assert r["fault_type"] == "unknown" + assert r["confidence"] == "low" + + +# ── FAULT_TYPES and REPAIR_HINTS coverage ──────────────────────── + + +def test_all_fault_types_have_hints(): + for ft in FAULT_TYPES: + assert ft in REPAIR_HINTS + + +def test_diagnose_returns_hint(): + r = diagnose("[API ERROR] test") + assert "hint" in r + assert len(r["hint"]) > 0 diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..ea9d55d --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,503 @@ +# tests/test_engine.py — Unit tests for FaultEngine (pure in-memory, no network) +import copy + +from agentchaos.fault_engine import FaultEngine, FaultSpec, _parse_tokens, jp_get, jp_set + +# ── JSON path helpers ───────────────────────────────────────────── + +SAMPLE_RESPONSE = { + "choices": [ + { + "message": { + "content": "The answer is 42.", + "tool_calls": [ + {"id": "tc_1", "type": "function", "function": {"name": "calc", "arguments": '{"x": 1}'}} + ], + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, +} + + +def _resp(): + return copy.deepcopy(SAMPLE_RESPONSE) + + +def test_jp_get_content(): + assert jp_get(SAMPLE_RESPONSE, "$.choices[0].message.content") == "The answer is 42." + + +def test_jp_get_nested(): + assert jp_get(SAMPLE_RESPONSE, "$.choices[0].finish_reason") == "stop" + + +def test_jp_get_missing(): + assert jp_get(SAMPLE_RESPONSE, "$.choices[0].message.nonexistent") is None + + +def test_jp_get_index_out_of_range(): + assert jp_get(SAMPLE_RESPONSE, "$.choices[5].message.content") is None + + +def test_jp_set_content(): + data = _resp() + jp_set(data, "$.choices[0].message.content", "modified") + assert data["choices"][0]["message"]["content"] == "modified" + + +def test_jp_set_unreachable(): + data = _resp() + jp_set(data, "$.nonexist[0].field", "value") # should not crash + + +# ── FaultEngine basic ───────────────────────────────────────────── + + +def test_engine_add_and_clear(): + engine = FaultEngine(seed=42) + engine.add(FaultSpec(intercept="response", action="set", target_path="$.choices[0].message.content", value="err")) + assert len(engine._faults) == 1 + engine.clear() + assert len(engine._faults) == 0 + + +def test_engine_trace_only(): + engine = FaultEngine(seed=42, trace_only=True) + assert engine.has_active_faults() is True + assert len(engine._faults) == 0 + + +def test_engine_has_active_faults_exhausted(): + engine = FaultEngine(seed=42) + spec = FaultSpec(intercept="response", action="set", target_path="$", value="x", max_count=1) + engine.add(spec) + assert engine.has_active_faults() is True + spec._count = 1 + assert engine.has_active_faults() is False + + +# ── FaultEngine.apply — action: set ─────────────────────────────── + + +def test_apply_set_content(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.content", + value="[ERROR]", + max_count=1, + skip_guard=True, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert data["choices"][0]["message"]["content"] == "[ERROR]" + assert delay == 0.0 + + +def test_apply_set_tool_calls_empty(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.tool_calls", + value=[], + max_count=1, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert data["choices"][0]["message"]["tool_calls"] == [] + + +# ── FaultEngine.apply — action: truncate ────────────────────────── + + +def test_apply_truncate_content(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="truncate", + target_path="$.choices[0].message.content", + value=0.5, + max_count=1, + skip_guard=True, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert len(data["choices"][0]["message"]["content"]) < len("The answer is 42.") + assert data["choices"][0]["finish_reason"] == "length" + + +def test_apply_truncate_tool_arguments(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="truncate", + target_path="$.choices[0].message.tool_calls[0].function.arguments", + value=0.5, + max_count=1, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + + +# ── FaultEngine.apply — action: corrupt ─────────────────────────── + + +def test_apply_corrupt_content(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="corrupt", + target_path="$.choices[0].message.content", + value="unicode", + max_count=1, + skip_guard=True, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert data["choices"][0]["message"]["content"] != "The answer is 42." + + +def test_apply_corrupt_mojibake(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="corrupt", + target_path="$.choices[0].message.content", + value="mojibake", + max_count=1, + skip_guard=True, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + + +def test_apply_corrupt_tool_arguments(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="corrupt", + target_path="$.choices[0].message.tool_calls[0].function.arguments", + value="unicode", + max_count=1, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + + +# ── FaultEngine.apply — action: delay ───────────────────────────── + + +def test_apply_delay(): + engine = FaultEngine(seed=42) + engine.add(FaultSpec(intercept="response", action="delay", value=2000, max_count=1)) + action, data, delay = engine.apply("response", _resp()) + assert delay == 2000.0 + assert action == "delay" + + +# ── FaultEngine.apply — action: drop ────────────────────────────── + + +def test_apply_drop(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="drop", + target_path="$.choices[0].message.tool_calls[0]", + max_count=1, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert data["choices"][0]["message"]["tool_calls"] == [] + + +# ── FaultEngine.apply — action: error ───────────────────────────── + + +def test_apply_error(): + engine = FaultEngine(seed=42) + engine.add(FaultSpec(intercept="response", action="error", value=503, max_count=1)) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert "503" in data["choices"][0]["message"]["content"] + + +# ── FaultEngine.apply — action: duplicate ───────────────────────── + + +def test_apply_duplicate_no_cache(): + engine = FaultEngine(seed=42) + engine.add(FaultSpec(intercept="response", action="duplicate", max_count=1)) + action, data, delay = engine.apply("response", _resp()) + # no cached response yet, so nothing happens + assert action == "pass" + + +def test_apply_duplicate_with_cache(): + engine = FaultEngine(seed=42) + engine._last_response = {"choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}]} + engine.add(FaultSpec(intercept="response", action="duplicate", max_count=1)) + action, data, delay = engine.apply("response", _resp()) + assert action == "modify" + assert data["choices"][0]["message"]["content"] == "cached" + + +# ── FaultEngine.apply — guard logic ────────────────────────────── + + +def test_guard_skips_content_fault_when_tool_calls_present(): + """Content fault should NOT fire when tool_calls are present (guard active).""" + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.content", + value="[ERROR]", + max_count=1, + skip_guard=False, + ) + ) + action, data, delay = engine.apply("response", _resp()) + assert action == "pass" # guard prevents firing + + +def test_guard_skips_tool_fault_when_no_tool_calls(): + """Tool fault should NOT fire when tool_calls are empty (guard active).""" + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.tool_calls", + value=[], + max_count=1, + skip_guard=False, + ) + ) + resp = _resp() + resp["choices"][0]["message"]["tool_calls"] = [] + action, data, delay = engine.apply("response", resp) + assert action == "pass" + + +# ── FaultEngine.apply — probability and count ───────────────────── + + +def test_max_count_exhaustion(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.content", + value="X", + max_count=1, + skip_guard=True, + ) + ) + engine.apply("response", _resp()) # fires + action, data, delay = engine.apply("response", _resp()) # exhausted + assert action == "pass" + + +def test_min_count_delayed_onset(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.content", + value="X", + max_count=2, + min_count=1, + skip_guard=True, + ) + ) + action1, _, _ = engine.apply("response", _resp()) # count=1, but min_count=1, skip + assert action1 == "pass" + action2, data2, _ = engine.apply("response", _resp()) # count=2, fires + assert action2 == "modify" + assert data2["choices"][0]["message"]["content"] == "X" + + +def test_probability_zero_never_fires(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="set", + target_path="$.choices[0].message.content", + value="X", + probability=0.0, + skip_guard=True, + ) + ) + action, _, _ = engine.apply("response", _resp()) + assert action == "pass" + + +# ── _parse_tokens edge cases ────────────────────────────────────── + + +def test_parse_tokens_root(): + assert _parse_tokens("$") == [] + assert _parse_tokens("") == [] + + +def test_parse_tokens_numeric_segment(): + tokens = _parse_tokens("$.items.0.name") + assert 0 in tokens + assert "name" in tokens + + +# ── _truncate_json_values / _corrupt_json_values edge cases ─────── + + +def test_truncate_json_values_invalid_json(): + engine = FaultEngine(seed=42) + result = engine._truncate_json_values("not valid json{{{", 0.5) + assert "_fault_truncated" in result + + +def test_truncate_json_values_nested(): + engine = FaultEngine(seed=42) + result = engine._truncate_json_values('{"key": "longvalue", "nested": {"a": "hello"}}', 0.5) + data = __import__("json").loads(result) + assert len(data["key"]) <= len("longvalue") + + +def test_corrupt_json_values_invalid_json(): + engine = FaultEngine(seed=42) + result = engine._corrupt_json_values("not valid json{{{") + assert "_fault_corrupted" in result + + +def test_corrupt_json_values_nested(): + engine = FaultEngine(seed=42) + result = engine._corrupt_json_values('{"key": "hello world"}') + data = __import__("json").loads(result) + assert data["key"] != "hello world" + + +# ── corrupt broken_json mode ───────────────────────────────────── + + +def test_apply_corrupt_broken_json(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="corrupt", + target_path="$.choices[0].message.content", + value="broken_json", + max_count=1, + skip_guard=True, + ) + ) + action, data, _ = engine.apply("response", _resp()) + assert action == "modify" + assert "\x00" in data["choices"][0]["message"]["content"] + + +# ── truncate on list target ─────────────────────────────────────── + + +def test_apply_truncate_list(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="truncate", + target_path="$.choices[0].message.tool_calls", + value=0.5, + max_count=1, + ) + ) + resp = _resp() + resp["choices"][0]["message"]["tool_calls"] = [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}] + action, data, _ = engine.apply("response", resp) + assert action == "modify" + assert len(data["choices"][0]["message"]["tool_calls"]) == 2 + + +# ── corrupt on empty string target (no-op) ──────────────────────── + + +def test_apply_corrupt_empty_string(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="corrupt", + target_path="$.choices[0].message.content", + value="unicode", + max_count=1, + skip_guard=True, + ) + ) + resp = _resp() + resp["choices"][0]["message"]["content"] = "" + action, data, _ = engine.apply("response", resp) + # empty string can't be corrupted, action stays pass + assert action == "pass" + + +# ── truncate on empty string (no-op) ───────────────────────────── + + +def test_apply_truncate_empty_string(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="response", + action="truncate", + target_path="$.choices[0].message.content", + value=0.3, + max_count=1, + skip_guard=True, + ) + ) + resp = _resp() + resp["choices"][0]["message"]["content"] = "" + action, data, _ = engine.apply("response", resp) + assert action == "pass" + + +# ── intercept mismatch (request spec on response) ──────────────── + + +def test_apply_intercept_mismatch(): + engine = FaultEngine(seed=42) + engine.add( + FaultSpec( + intercept="request", + action="set", + target_path="$.choices[0].message.content", + value="X", + max_count=1, + skip_guard=True, + ) + ) + action, _, _ = engine.apply("response", _resp()) + assert action == "pass"