Close the dotnet-test skill-coverage gaps from the PR 945 report

.gitignore

@@ -168,6 +168,11 @@ coverage*.json
 coverage*.xml
 coverage*.info
 
+# ...but eval fixtures ship checked-in coverage reports as test inputs.
+!tests/**/fixtures/**/coverage*.json
+!tests/**/fixtures/**/coverage*.xml
+!tests/**/fixtures/**/coverage*.info
+
 # Visual Studio code coverage results
 *.coverage
 *.coveragexml

eng/skill-coverage/Measure-SkillCoverage.ps1

@@ -293,89 +293,169 @@ function Get-SignificantTerms([string]$text) {
 #  eval.yaml Parsing - extract test evidence
 # ═══════════════════════════════════════════════════════════
 
+function Get-LineIndent([string]$line) {
+    if ($line -match '^(\s*)\S') { return $Matches[1].Length }
+    return -1   # blank or whitespace-only
+}
+
+function Remove-YamlQuoting([string]$text) {
+    $t = $text.Trim()
+    if ($t.Length -ge 2 -and $t.StartsWith('"') -and $t.EndsWith('"')) {
+        return ($t.Substring(1, $t.Length - 2) -replace '\\\\', '\')
+    }
+    if ($t.Length -ge 2 -and $t.StartsWith("'") -and $t.EndsWith("'")) {
+        return $t.Substring(1, $t.Length - 2)
+    }
+    return $t
+}
+
+# Reads a YAML scalar that may be a block scalar (| or >) or a plain/quoted
+# scalar folded across several more-indented continuation lines. Returns the
+# joined value plus the index of the last line consumed.
+function Read-YamlScalar {
+    param([string[]]$Lines, [int]$Index, [int]$KeyIndent, [string]$Inline)
+
+    $inline = $Inline.Trim()
+    $buffer = @()
+    $last = $Index
+
+    if ($inline -match '^[|>][+-]?\d*$') { $inline = '' }
+    elseif ($inline) { $buffer += $inline }
+
+    for ($j = $Index + 1; $j -lt $Lines.Count; $j++) {
+        $candidate = $Lines[$j].TrimEnd()
+        $indent = Get-LineIndent $candidate
+        if ($indent -lt 0) { break }
+        if ($indent -le $KeyIndent) { break }
+
+        $trimmed = $candidate.Trim()
+        if ($trimmed.StartsWith('#')) { break }
+        if ($trimmed -match '^-(\s|$)') { break }
+        if ($trimmed -match '^[A-Za-z_][\w.-]*:(\s|$)') { break }
+
+        $buffer += $trimmed
+        $last = $j
+    }
+
+    [PSCustomObject]@{
+        Value     = ($buffer -join ' ').Trim()
+        LastIndex = $last
+    }
+}
+
+<#
+.SYNOPSIS
+    Extracts test evidence from an eval.yaml spec.
+
+.DESCRIPTION
+    Evidence is anything the eval actually checks: grader configuration
+    (regex patterns, literal substrings, expected values, commands) and
+    LLM-judged rubric items. Both are matched against SKILL.md teaching
+    points by the coverage engine.
+
+    The parser is indentation-driven so it handles the real spec shape:
+    graders are a list of `- type:` entries each carrying a nested `config:`
+    map, and rubric items are plain (usually unquoted) scalars that commonly
+    wrap across several lines.
+#>
 function Get-EvalEvidence([string]$yamlContent) {
+    # Grader config keys whose value is a regular expression.
+    $regexKeys = @('pattern', 'stdout_matches')
+    # Grader config keys whose value is a literal string.
+    $literalKeys = @('substring', 'value', 'stdout_contains', 'command')
+
     $evidence = @()
     $scenarioCount = 0
     $scenario = '(unknown)'
     $section = 'none'
-    $assertType = $null
+    $sectionIndent = -1
+    $graderType = $null
 
-    foreach ($rawLine in $yamlContent -split "`n") {
-        $line = $rawLine.TrimEnd()
+    $lines = $yamlContent -split "`n"
 
-        # Scenario name
-        if ($line -match '^\s+-?\s*name:\s*[''"]?(.+?)[''"]?\s*$') {
-            $scenario = $Matches[1]
-            $scenarioCount++
+    for ($i = 0; $i -lt $lines.Count; $i++) {
+        $line = $lines[$i].TrimEnd()
+        $indent = Get-LineIndent $line
+        if ($indent -lt 0) { continue }
+        $trimmed = $line.Trim()
+        if ($trimmed.StartsWith('#')) { continue }
+
+        # Leaving the current block ends the section.
+        if ($section -ne 'none' -and $indent -le $sectionIndent) {
             $section = 'none'
-            $assertType = $null
+            $sectionIndent = -1
+            $graderType = $null
         }
 
-        # Section transitions
-        if ($line -match '^\s{2,6}assertions:\s*$')  { $section = 'assertions'; continue }
-        if ($line -match '^\s{2,6}rubric:\s*$')       { $section = 'rubric'; continue }
-        if ($line -match '^\s{2,6}(prompt|setup|timeout|reject_tools|expect_tools|expect_activation):' -and $section -ne 'none') {
+        # Stimulus boundary: `- name: <scenario>` inside the stimuli list.
+        if ($indent -ge 2 -and $trimmed -match '^-\s+name:\s*(.+)$') {
+            $scenario = Remove-YamlQuoting $Matches[1]
+            $scenarioCount++
             $section = 'none'
+            $sectionIndent = -1
+            $graderType = $null
             continue
         }
 
-        # Assertion fields
-        if ($section -eq 'assertions') {
-            if ($line -match '^\s+-\s*type:\s*[''"]?(\S+?)[''"]?\s*$') {
-                $assertType = $Matches[1]
-                if ($assertType -eq 'exit_success') {
+        if ($trimmed -eq 'graders:') {
+            $section = 'graders'; $sectionIndent = $indent; $graderType = $null; continue
+        }
+        if ($trimmed -eq 'rubric:') {
+            $section = 'rubric'; $sectionIndent = $indent; continue
+        }
+
+        if ($section -eq 'graders') {
+            if ($trimmed -match '^-\s+type:\s*(.+)$') {
+                $graderType = (Remove-YamlQuoting $Matches[1]) -replace '[^\w.-]', ''
+                if ($graderType -eq 'exit-success') {
                     $evidence += [PSCustomObject]@{
                         Scenario     = $scenario
-                        EvidenceType = 'assertion:exit_success'
-                        Content      = 'exit_success: project builds and tests pass'
+                        EvidenceType = 'assertion:exit-success'
+                        Content      = 'exit-success: project builds and tests pass'
                         RawPattern   = $null
                     }
                 }
+                continue
             }
-            if ($line -match '^\s+pattern:\s*"(.+?)"\s*$') {
-                $regex = $Matches[1] -replace '\\\\', '\'
-                $evidence += [PSCustomObject]@{
-                    Scenario     = $scenario
-                    EvidenceType = "assertion:$assertType"
-                    Content      = $regex
-                    RawPattern   = $regex
-                }
-            }
-            elseif ($line -match "^\s+pattern:\s*'(.+?)'\s*$") {
-                $regex = $Matches[1]
-                $evidence += [PSCustomObject]@{
-                    Scenario     = $scenario
-                    EvidenceType = "assertion:$assertType"
-                    Content      = $regex
-                    RawPattern   = $regex
-                }
-            }
-            # Unquoted pattern (bare YAML scalar)
-            elseif ($line -match '^\s+pattern:\s*([^''"\s].+?)\s*$') {
-                $regex = $Matches[1]
-                $evidence += [PSCustomObject]@{
-                    Scenario     = $scenario
-                    EvidenceType = "assertion:$assertType"
-                    Content      = $regex
-                    RawPattern   = $regex
-                }
-            }
-            if ($line -match '^\s+value:\s*[''"](.+?)[''"]') {
+
+            if ($trimmed -match '^([A-Za-z_][\w.-]*):\s*(.*)$') {
+                $key = $Matches[1]
+                $rest = $Matches[2]
+
+                $isRegex = $regexKeys -contains $key
+                $isLiteral = $literalKeys -contains $key
+                # For file-exists style graders the path *is* the assertion.
+                $isPath = ($key -eq 'path' -and $graderType -match '^file-(not-)?exists$')
+
+                if (-not ($isRegex -or $isLiteral -or $isPath)) { continue }
+
+                $scalar = Read-YamlScalar -Lines $lines -Index $i -KeyIndent $indent -Inline $rest
+                $i = $scalar.LastIndex
+                $value = Remove-YamlQuoting $scalar.Value
+                if (-not $value) { continue }
+
                 $evidence += [PSCustomObject]@{
                     Scenario     = $scenario
-                    EvidenceType = "assertion:$assertType"
-                    Content      = $Matches[1]
-                    RawPattern   = $null
+                    EvidenceType = "assertion:$graderType"
+                    Content      = $value
+                    # Literal values are escaped so the code-pattern matcher
+                    # treats them as text rather than as a regex.
+                    RawPattern   = if ($isRegex) { $value } else { [regex]::Escape($value) }
                 }
             }
+            continue
         }
 
-        # Rubric items
-        if ($section -eq 'rubric' -and $line -match '^\s+-\s*[''"](.+?)[''"]') {
+        if ($section -eq 'rubric' -and $trimmed -match '^-\s*(.*)$') {
+            $scalar = Read-YamlScalar -Lines $lines -Index $i -KeyIndent $indent -Inline $Matches[1]
+            $i = $scalar.LastIndex
+            $value = Remove-YamlQuoting $scalar.Value
+            if (-not $value) { continue }
+
             $evidence += [PSCustomObject]@{
                 Scenario     = $scenario
                 EvidenceType = 'rubric'
-                Content      = $Matches[1]
+                Content      = $value
                 RawPattern   = $null
             }
         }

tests/dotnet-test/assertion-quality/eval.yaml

@@ -68,6 +68,8 @@ stimuli:
       - Recommended specific meaningful assertions for at least some of the assertion-free tests
       - Distinguished between intentional smoke tests (acceptable if labeled as such) and tests masquerading as real
         verification
+      - Metric counts add up correctly - the 5 assertion-free tests plus the 3 trivial-only tests reconcile to the
+        8 test methods in the file, and any percentage reported matches those counts
   - name: Recognize well-diversified assertion usage
     prompt: |
       Can you evaluate the assertion quality in my UserService.Tests project?
@@ -92,6 +94,8 @@ stimuli:
         Assert.AreNotEqual for role change)
       - Acknowledged the exception assertions with specific exception types and parameter name verification
       - If any gaps were identified, they were presented as minor enhancement opportunities rather than problems
+      - The report acknowledges outright that this suite has good assertion diversity, instead of manufacturing
+        problems so it has something to flag
   - name: Identify self-referential assertions in identity and round-trip tests
     prompt: |
       I want an assertion quality analysis of my Config tests. They all pass

tests/dotnet-test/crap-score/eval.yaml

@@ -98,8 +98,8 @@ stimuli:
         "Complexity: 4" comment above it'
       - Reported a complexity in the low-to-mid teens, counting the && and || operators, the ?? and the two ternaries
         as decision points — not 4
-      - Used the 55% coverage recorded for ApplySurcharges in the Cobertura XML
-      - Reported a CRAP score in the high twenties and classified it as high risk
+      - Used the 60% coverage recorded for ApplySurcharges in the Cobertura XML
+      - Reported a CRAP score in the mid-to-high twenties (roughly 26) and classified it as high risk
       - Stated the coverage level required to bring the method under a CRAP threshold of 15
 
   - name: Recognize when complexity alone blocks the CRAP threshold

tests/dotnet-test/crap-score/fixtures/refactor-required/coverage.cobertura.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<coverage line-rate="0.6486" branch-rate="0.5690" version="1.0" timestamp="1700000000" lines-covered="24" lines-valid="37" branches-covered="33" branches-valid="58">
+  <sources>
+    <source>.</source>
+  </sources>
+  <packages>
+    <package name="Billing" line-rate="0.6486" branch-rate="0.5690" complexity="34">
+      <classes>
+        <class name="Billing.InvoiceEngine" filename="InvoiceEngine.cs" line-rate="0.6486" branch-rate="0.5690" complexity="34">
+          <methods>
+            <method name="ApplySurcharges" signature="(Billing.Invoice,System.String,System.String)" line-rate="0.6000" branch-rate="0.5000" complexity="14">
+              <lines>
+                <line number="19" hits="24" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="20" hits="0"/>
+                <line number="22" hits="24"/>
+                <line number="24" hits="24" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="25" hits="0"/>
+                <line number="26" hits="24" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="27" hits="0"/>
+                <line number="29" hits="24" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="30" hits="0"/>
+                <line number="31" hits="24" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="32" hits="0"/>
+                <line number="34" hits="24" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="35" hits="0"/>
+                <line number="37" hits="24" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="38" hits="24" branch="true" condition-coverage="50% (1/2)"/>
+              </lines>
+            </method>
+            <method name="ClassifyAccount" signature="(Billing.Account)" line-rate="0.6316" branch-rate="0.5625" complexity="17">
+              <lines>
+                <line number="45" hits="18" branch="true" condition-coverage="100% (2/2)"/>
+                <line number="46" hits="2"/>
+                <line number="48" hits="16" branch="true" condition-coverage="100% (2/2)"/>
+                <line number="50" hits="5" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="51" hits="3" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="52" hits="0" branch="true" condition-coverage="0% (0/2)"/>
+                <line number="53" hits="0"/>
+                <line number="54" hits="0"/>
+                <line number="57" hits="11" branch="true" condition-coverage="100% (2/2)"/>
+                <line number="58" hits="4" branch="true" condition-coverage="50% (1/2)"/>
+                <line number="60" hits="7" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="61" hits="0"/>
+                <line number="63" hits="7" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="64" hits="0"/>
+                <line number="66" hits="7" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="67" hits="0"/>
+                <line number="69" hits="7" branch="true" condition-coverage="50% (2/4)"/>
+                <line number="70" hits="0"/>
+                <line number="72" hits="7" branch="true" condition-coverage="50% (1/2)"/>
+              </lines>
+            </method>
+            <method name="RoundToCurrency" signature="(System.Decimal,System.Int32)" line-rate="1.0000" branch-rate="1.0000" complexity="3">
+              <lines>
+                <line number="77" hits="9" branch="true" condition-coverage="100% (2/2)"/>
+                <line number="78" hits="2"/>
+                <line number="80" hits="7" branch="true" condition-coverage="100% (2/2)"/>
+              </lines>
+            </method>
+          </methods>
+        </class>
+      </classes>
+    </package>
+  </packages>
+</coverage>

tests/dotnet-test/detect-static-dependencies/eval.yaml

@@ -152,6 +152,48 @@ stimuli:
       - Provided a structured summary or report of findings with categories or counts
       - Recommended wrapping or abstracting the detected static dependencies
 
+  - name: Keep one authoritative total with file line locations and no seam for pure helpers
+    prompt: >
+      Scan StaticHeavy/ and give me an audit I can act on directly. For every
+      dependency you report I need the file and the line number so I can jump
+      straight to it, and I need the arithmetic to hold: one grand total that
+      the category table, the top-pattern table and the per-file table all
+      agree with. Also tell me explicitly which calls you decided are NOT
+      worth putting behind a seam, and why.
+    environment:
+      files:
+        - src: ./fixtures/static-heavy-project
+          dest: StaticHeavy
+    graders:
+      - type: output-matches
+        config:
+          pattern: (\w+\.cs:\d+|\w+\.cs.{0,20}[Ll]ine\s*\d+)
+      - type: output-matches
+        config:
+          pattern: Path\.Combine
+      - type: output-matches
+        config:
+          pattern: (Guid\.NewGuid|random|identity)
+      - type: output-matches
+        config:
+          pattern: (JsonSerializer|serializ)
+      - type: exit-success
+      - type: prompt
+    rubric:
+      - Every reported occurrence carries a file and line location, not just a file name
+      - Reported a single authoritative grand total and did not park real call sites in a trailing
+        "additional observations" or "also noticed" section that the totals exclude
+      - The category totals, the top-patterns ranking, and the per-file counts all reconcile to the same
+        grand total
+      - Classified Path.Combine as a deterministic pure helper that needs no seam, and did not recommend
+        wrapping, abstracting, or injecting it
+      - Counted Path.GetTempPath as a File System dependency (it reads ambient machine state) while keeping
+        Path.Combine out of the needs-wrapping total
+      - Reported the Guid.NewGuid call site in OrderProcessor.CreateOrder under a randomness or identity
+        category
+      - Reported the JsonSerializer.Serialize call site in OrderProcessor.ArchiveOrder under a
+        culture or serialization category
+
   - name: Detect statics inside lambda expressions and LINQ queries
     prompt: >
       My ReportGenerator class in LambdaStatics/ uses a lot of LINQ and