test/coverage and ci speed

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.10', '3.13']
 
     steps:
       - uses: actions/checkout@v4
@@ -24,16 +24,19 @@ jobs:
 
       - name: Install uv
         uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
 
       - name: Install dependencies
         run: uv sync --python ${{ matrix.python-version }}
 
       - name: Run tests
-        run: uv run pytest tests/ -v --cov=agentchaos --cov-report=xml
+        run: uv run pytest tests/ -x --cov=agentchaos --cov-report=xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
-        if: matrix.python-version == '3.12'
+        if: matrix.python-version == '3.13'
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml
@@ -51,6 +54,9 @@ jobs:
 
       - name: Install uv
         uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
 
       - name: Install dependencies
         run: uv sync

.gitignore

@@ -26,6 +26,7 @@ Thumbs.db
 
 # Testing
 .pytest_cache/
+.coverage
 coverage.xml
 htmlcov/
 .deepeval/

pyproject.toml

@@ -71,6 +71,7 @@ select = ["E", "F", "W", "I"]
 
 [tool.coverage.run]
 source = ["agentchaos"]
+omit = ["agentchaos/__main__.py"]
 
 [tool.coverage.report]
 exclude_lines = [

tests/test_config.py

@@ -0,0 +1,172 @@
+# tests/test_config.py — Unit tests for fault_config (pure in-memory catalog validation)
+import json
+import os
+import tempfile
+
+import agentchaos
+from agentchaos import EvalReport, EvalResult, fault_config
+from agentchaos.fault_engine import FaultSpec
+
+
+def test_total_65_experiments():
+    assert len(fault_config.EXPERIMENTS) == 65
+
+
+def test_all_experiments_have_name_and_faults():
+    for exp in fault_config.EXPERIMENTS:
+        assert "name" in exp
+        assert "faults" in exp
+        assert len(exp["faults"]) >= 1
+
+
+def test_all_faults_are_faultspec():
+    for exp in fault_config.EXPERIMENTS:
+        for spec in exp["faults"]:
+            assert isinstance(spec, FaultSpec)
+
+
+def test_strategies_coverage():
+    names = fault_config.list_all()
+    for strategy in ["single", "persistent", "intermittent", "burst"]:
+        matching = [n for n in names if n.endswith(f"_{strategy}")]
+        assert len(matching) == 12  # 6 llm + 6 tool
+
+
+def test_compound_experiments():
+    names = fault_config.list_all()
+    compounds = [n for n in names if n.startswith("compound_")]
+    assert len(compounds) == 8
+
+
+def test_positional_experiments():
+    names = fault_config.list_all()
+    positionals = [n for n in names if "_pos_" in n]
+    assert len(positionals) == 9
+
+
+def test_get_valid():
+    exp = fault_config.get("llm_error_single")
+    assert exp["name"] == "llm_error_single"
+    assert len(exp["faults"]) == 1
+    assert exp["faults"][0].action == "set"
+
+
+def test_get_invalid_raises():
+    try:
+        fault_config.get("nonexistent_fault")
+        assert False, "Should have raised ValueError"
+    except ValueError as e:
+        assert "nonexistent_fault" in str(e)
+
+
+def test_list_all_sorted():
+    names = fault_config.list_all()
+    assert names == sorted(names)
+
+
+def test_list_by_category_structure():
+    cats = fault_config.list_by_category()
+    assert "llm" in cats
+    assert "tool" in cats
+    assert "compound" in cats
+    assert len(cats["llm"]) == 6
+    assert len(cats["tool"]) == 6
+    assert len(cats["compound"]) == 8
+
+
+def test_list_by_category_has_experiments():
+    cats = fault_config.list_by_category()
+    for cat_name, items in cats.items():
+        for item in items:
+            assert "base_name" in item
+            assert "type" in item
+            assert "description" in item
+            assert "experiments" in item
+
+
+def test_each_experiment_retrievable():
+    for name in fault_config.list_all():
+        exp = fault_config.get(name)
+        assert exp is not None
+        assert exp["name"] == name
+
+
+# ── __init__.py coverage: EvalReport / save_trace branches ────────
+
+
+def test_eval_report_properties():
+    r1 = EvalResult(
+        fault="f1",
+        result="ok",
+        error="",
+        passed=True,
+        elapsed=0.1,
+        faults_fired=0,
+        diagnosis={},
+        fault_log=[],
+    )
+    r2 = EvalResult(
+        fault="f2",
+        result="",
+        error="err",
+        passed=False,
+        elapsed=0.2,
+        faults_fired=1,
+        diagnosis={},
+        fault_log=[],
+    )
+    report = EvalReport(total=2, passed=1, failed=1, results=[r1, r2])
+    assert report.pass_rate == 0.5
+    assert report.vulnerable_to == ["f2"]
+    assert "1/2" in report.summary()
+
+
+def test_eval_report_empty():
+    report = EvalReport(total=0, passed=0, failed=0, results=[])
+    assert report.pass_rate == 0.0
+    assert report.vulnerable_to == []
+    assert "none" in report.summary()
+
+
+def test_save_trace_with_fault_data():
+    """save_trace correctly writes injected_output when fault was applied."""
+    engine = agentchaos.inject("llm_error_single")
+    # manually add a trace entry to simulate a faulted call
+    engine.trace.append(
+        {
+            "call_index": 0,
+            "request": {"model": "test", "messages": [], "tools": []},
+            "response": {
+                "content": "original",
+                "tool_calls": [],
+                "finish_reason": "stop",
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+                "http_status": 200,
+                "modified_content": "[ERROR] injected",
+                "modified_tool_calls": [],
+            },
+            "timing": {"llm_latency_ms": 100, "total_ms": 101},
+            "fault": {"applied": True},
+        }
+    )
+    agentchaos.disable()
+
+    with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f:
+        path = f.name
+    try:
+        agentchaos.save_trace(path)
+        with open(path) as f:
+            data = json.load(f)
+        assert len(data) == 1
+        assert data[0]["fault_applied"] is True
+        assert "injected_output" in data[0]
+        assert data[0]["injected_output"]["content"] == "[ERROR] injected"
+        assert data[0]["raw_output"]["content"] == "original"
+    finally:
+        os.unlink(path)
+
+
+def test_save_trace_no_engine():
+    """save_trace with no prior engine does not crash."""
+    agentchaos._last_engine = None
+    agentchaos.save_trace("/tmp/test_no_engine.json")  # should log error, not crash

tests/test_diagnose.py

@@ -0,0 +1,134 @@
+# tests/test_diagnose.py — Unit tests for fault_diagnose (pure string matching, no network)
+from agentchaos.fault_diagnose import FAULT_TYPES, REPAIR_HINTS, diagnose
+
+# ── Empty / None ──────────────────────────────────────────────────
+
+
+def test_diagnose_empty_string():
+    r = diagnose("")
+    assert r["fault_type"] == "empty"
+    assert r["confidence"] == "high"
+
+
+def test_diagnose_none_input():
+    r = diagnose(None)
+    assert r["fault_type"] == "empty"
+
+
+def test_diagnose_whitespace_only():
+    r = diagnose("   \n  ")
+    assert r["fault_type"] == "empty"
+
+
+# ── Error patterns ────────────────────────────────────────────────
+
+
+def test_diagnose_api_error():
+    r = diagnose("[API ERROR] HTTP 500: Internal Server Error.")
+    assert r["fault_type"] == "error"
+
+
+def test_diagnose_http_500():
+    r = diagnose("HTTP 500 Internal Server Error occurred")
+    assert r["fault_type"] == "error"
+
+
+# ── Timeout patterns ─────────────────────────────────────────────
+
+
+def test_diagnose_timeout():
+    r = diagnose("[TIMEOUT] The request timed out. The server did not respond within the expected time.")
+    assert r["fault_type"] == "timeout"
+
+
+def test_diagnose_connection_dropped():
+    r = diagnose("Connection dropped. The server did not respond.")
+    assert r["fault_type"] == "timeout"
+
+
+# ── Compound patterns ────────────────────────────────────────────
+
+
+def test_diagnose_service_unavailable():
+    r = diagnose("[SERVICE UNAVAILABLE] HTTP 503: The server is temporarily unable to handle the request.")
+    assert r["fault_type"] == "compound"
+
+
+def test_diagnose_content_filtered():
+    r = diagnose("[CONTENT FILTERED] This response has been blocked by the content safety filter.")
+    assert r["fault_type"] == "compound"
+
+
+def test_diagnose_502_bad_gateway():
+    r = diagnose("<!DOCTYPE html><html><body>502 Bad Gateway nginx/1.24.0</body></html>")
+    assert r["fault_type"] == "compound"
+
+
+# ── Schema patterns ──────────────────────────────────────────────
+
+
+def test_diagnose_content_policy_violation():
+    r = diagnose('{"error": "content_policy_violation", "message": "This response has been filtered."}')
+    assert r["fault_type"] == "schema"
+
+
+# ── Corrupt patterns ─────────────────────────────────────────────
+
+
+def test_diagnose_unicode_corruption():
+    # 4 consecutive symbols from Miscellaneous Symbols block
+    r = diagnose("Hello \u2600\u2601\u2602\u2603\u2604 world")
+    assert r["fault_type"] == "corrupt"
+
+
+def test_diagnose_mojibake():
+    # simulate UTF-8 bytes misread as Latin-1
+    original = "Hello world"
+    mojibake = original.encode("utf-8").decode("latin-1")
+    # add multiple mojibake patterns
+    text = f"{mojibake} â\x80\x99 â\x80\x9c some text"
+    r = diagnose(text)
+    assert r["fault_type"] == "corrupt"
+
+
+# ── Truncation detection ─────────────────────────────────────────
+
+
+def test_diagnose_truncated_text():
+    # text ending mid-word without punctuation
+    r = diagnose("The answer to the question about prime numbers is that we need to check divisibili")
+    assert r["fault_type"] == "truncate"
+    assert r["confidence"] == "medium"
+
+
+def test_diagnose_not_truncated_with_period():
+    r = diagnose("The answer is 42.")
+    assert r["fault_type"] == "unknown"
+
+
+def test_diagnose_not_truncated_short():
+    r = diagnose("short")
+    assert r["fault_type"] == "unknown"
+
+
+# ── Normal text → unknown ────────────────────────────────────────
+
+
+def test_diagnose_normal_text():
+    r = diagnose("The answer is 42. This is a complete response.")
+    assert r["fault_type"] == "unknown"
+    assert r["confidence"] == "low"
+
+
+# ── FAULT_TYPES and REPAIR_HINTS coverage ────────────────────────
+
+
+def test_all_fault_types_have_hints():
+    for ft in FAULT_TYPES:
+        assert ft in REPAIR_HINTS
+
+
+def test_diagnose_returns_hint():
+    r = diagnose("[API ERROR] test")
+    assert "hint" in r
+    assert len(r["hint"]) > 0