diff --git a/.gitignore b/.gitignore index 9fdf6b977d..c773088eb9 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,11 @@ coverage*.json coverage*.xml coverage*.info +# ...but eval fixtures ship checked-in coverage reports as test inputs. +!tests/**/fixtures/**/coverage*.json +!tests/**/fixtures/**/coverage*.xml +!tests/**/fixtures/**/coverage*.info + # Visual Studio code coverage results *.coverage *.coveragexml diff --git a/eng/skill-coverage/Measure-SkillCoverage.ps1 b/eng/skill-coverage/Measure-SkillCoverage.ps1 index c856667040..b6cc0e9bb8 100644 --- a/eng/skill-coverage/Measure-SkillCoverage.ps1 +++ b/eng/skill-coverage/Measure-SkillCoverage.ps1 @@ -293,89 +293,169 @@ function Get-SignificantTerms([string]$text) { # eval.yaml Parsing - extract test evidence # ═══════════════════════════════════════════════════════════ +function Get-LineIndent([string]$line) { + if ($line -match '^(\s*)\S') { return $Matches[1].Length } + return -1 # blank or whitespace-only +} + +function Remove-YamlQuoting([string]$text) { + $t = $text.Trim() + if ($t.Length -ge 2 -and $t.StartsWith('"') -and $t.EndsWith('"')) { + return ($t.Substring(1, $t.Length - 2) -replace '\\\\', '\') + } + if ($t.Length -ge 2 -and $t.StartsWith("'") -and $t.EndsWith("'")) { + return $t.Substring(1, $t.Length - 2) + } + return $t +} + +# Reads a YAML scalar that may be a block scalar (| or >) or a plain/quoted +# scalar folded across several more-indented continuation lines. Returns the +# joined value plus the index of the last line consumed. +function Read-YamlScalar { + param([string[]]$Lines, [int]$Index, [int]$KeyIndent, [string]$Inline) + + $inline = $Inline.Trim() + $buffer = @() + $last = $Index + + if ($inline -match '^[|>][+-]?\d*$') { $inline = '' } + elseif ($inline) { $buffer += $inline } + + for ($j = $Index + 1; $j -lt $Lines.Count; $j++) { + $candidate = $Lines[$j].TrimEnd() + $indent = Get-LineIndent $candidate + if ($indent -lt 0) { break } + if ($indent -le $KeyIndent) { break } + + $trimmed = $candidate.Trim() + if ($trimmed.StartsWith('#')) { break } + if ($trimmed -match '^-(\s|$)') { break } + if ($trimmed -match '^[A-Za-z_][\w.-]*:(\s|$)') { break } + + $buffer += $trimmed + $last = $j + } + + [PSCustomObject]@{ + Value = ($buffer -join ' ').Trim() + LastIndex = $last + } +} + +<# +.SYNOPSIS + Extracts test evidence from an eval.yaml spec. + +.DESCRIPTION + Evidence is anything the eval actually checks: grader configuration + (regex patterns, literal substrings, expected values, commands) and + LLM-judged rubric items. Both are matched against SKILL.md teaching + points by the coverage engine. + + The parser is indentation-driven so it handles the real spec shape: + graders are a list of `- type:` entries each carrying a nested `config:` + map, and rubric items are plain (usually unquoted) scalars that commonly + wrap across several lines. +#> function Get-EvalEvidence([string]$yamlContent) { + # Grader config keys whose value is a regular expression. + $regexKeys = @('pattern', 'stdout_matches') + # Grader config keys whose value is a literal string. + $literalKeys = @('substring', 'value', 'stdout_contains', 'command') + $evidence = @() $scenarioCount = 0 $scenario = '(unknown)' $section = 'none' - $assertType = $null + $sectionIndent = -1 + $graderType = $null - foreach ($rawLine in $yamlContent -split "`n") { - $line = $rawLine.TrimEnd() + $lines = $yamlContent -split "`n" - # Scenario name - if ($line -match '^\s+-?\s*name:\s*[''"]?(.+?)[''"]?\s*$') { - $scenario = $Matches[1] - $scenarioCount++ + for ($i = 0; $i -lt $lines.Count; $i++) { + $line = $lines[$i].TrimEnd() + $indent = Get-LineIndent $line + if ($indent -lt 0) { continue } + $trimmed = $line.Trim() + if ($trimmed.StartsWith('#')) { continue } + + # Leaving the current block ends the section. + if ($section -ne 'none' -and $indent -le $sectionIndent) { $section = 'none' - $assertType = $null + $sectionIndent = -1 + $graderType = $null } - # Section transitions - if ($line -match '^\s{2,6}assertions:\s*$') { $section = 'assertions'; continue } - if ($line -match '^\s{2,6}rubric:\s*$') { $section = 'rubric'; continue } - if ($line -match '^\s{2,6}(prompt|setup|timeout|reject_tools|expect_tools|expect_activation):' -and $section -ne 'none') { + # Stimulus boundary: `- name: ` inside the stimuli list. + if ($indent -ge 2 -and $trimmed -match '^-\s+name:\s*(.+)$') { + $scenario = Remove-YamlQuoting $Matches[1] + $scenarioCount++ $section = 'none' + $sectionIndent = -1 + $graderType = $null continue } - # Assertion fields - if ($section -eq 'assertions') { - if ($line -match '^\s+-\s*type:\s*[''"]?(\S+?)[''"]?\s*$') { - $assertType = $Matches[1] - if ($assertType -eq 'exit_success') { + if ($trimmed -eq 'graders:') { + $section = 'graders'; $sectionIndent = $indent; $graderType = $null; continue + } + if ($trimmed -eq 'rubric:') { + $section = 'rubric'; $sectionIndent = $indent; continue + } + + if ($section -eq 'graders') { + if ($trimmed -match '^-\s+type:\s*(.+)$') { + $graderType = (Remove-YamlQuoting $Matches[1]) -replace '[^\w.-]', '' + if ($graderType -eq 'exit-success') { $evidence += [PSCustomObject]@{ Scenario = $scenario - EvidenceType = 'assertion:exit_success' - Content = 'exit_success: project builds and tests pass' + EvidenceType = 'assertion:exit-success' + Content = 'exit-success: project builds and tests pass' RawPattern = $null } } + continue } - if ($line -match '^\s+pattern:\s*"(.+?)"\s*$') { - $regex = $Matches[1] -replace '\\\\', '\' - $evidence += [PSCustomObject]@{ - Scenario = $scenario - EvidenceType = "assertion:$assertType" - Content = $regex - RawPattern = $regex - } - } - elseif ($line -match "^\s+pattern:\s*'(.+?)'\s*$") { - $regex = $Matches[1] - $evidence += [PSCustomObject]@{ - Scenario = $scenario - EvidenceType = "assertion:$assertType" - Content = $regex - RawPattern = $regex - } - } - # Unquoted pattern (bare YAML scalar) - elseif ($line -match '^\s+pattern:\s*([^''"\s].+?)\s*$') { - $regex = $Matches[1] - $evidence += [PSCustomObject]@{ - Scenario = $scenario - EvidenceType = "assertion:$assertType" - Content = $regex - RawPattern = $regex - } - } - if ($line -match '^\s+value:\s*[''"](.+?)[''"]') { + + if ($trimmed -match '^([A-Za-z_][\w.-]*):\s*(.*)$') { + $key = $Matches[1] + $rest = $Matches[2] + + $isRegex = $regexKeys -contains $key + $isLiteral = $literalKeys -contains $key + # For file-exists style graders the path *is* the assertion. + $isPath = ($key -eq 'path' -and $graderType -match '^file-(not-)?exists$') + + if (-not ($isRegex -or $isLiteral -or $isPath)) { continue } + + $scalar = Read-YamlScalar -Lines $lines -Index $i -KeyIndent $indent -Inline $rest + $i = $scalar.LastIndex + $value = Remove-YamlQuoting $scalar.Value + if (-not $value) { continue } + $evidence += [PSCustomObject]@{ Scenario = $scenario - EvidenceType = "assertion:$assertType" - Content = $Matches[1] - RawPattern = $null + EvidenceType = "assertion:$graderType" + Content = $value + # Literal values are escaped so the code-pattern matcher + # treats them as text rather than as a regex. + RawPattern = if ($isRegex) { $value } else { [regex]::Escape($value) } } } + continue } - # Rubric items - if ($section -eq 'rubric' -and $line -match '^\s+-\s*[''"](.+?)[''"]') { + if ($section -eq 'rubric' -and $trimmed -match '^-\s*(.*)$') { + $scalar = Read-YamlScalar -Lines $lines -Index $i -KeyIndent $indent -Inline $Matches[1] + $i = $scalar.LastIndex + $value = Remove-YamlQuoting $scalar.Value + if (-not $value) { continue } + $evidence += [PSCustomObject]@{ Scenario = $scenario EvidenceType = 'rubric' - Content = $Matches[1] + Content = $value RawPattern = $null } } diff --git a/tests/dotnet-test/assertion-quality/eval.yaml b/tests/dotnet-test/assertion-quality/eval.yaml index f708a7b1d9..5eac5b63be 100644 --- a/tests/dotnet-test/assertion-quality/eval.yaml +++ b/tests/dotnet-test/assertion-quality/eval.yaml @@ -68,6 +68,8 @@ stimuli: - Recommended specific meaningful assertions for at least some of the assertion-free tests - Distinguished between intentional smoke tests (acceptable if labeled as such) and tests masquerading as real verification + - Metric counts add up correctly - the 5 assertion-free tests plus the 3 trivial-only tests reconcile to the + 8 test methods in the file, and any percentage reported matches those counts - name: Recognize well-diversified assertion usage prompt: | Can you evaluate the assertion quality in my UserService.Tests project? @@ -92,6 +94,8 @@ stimuli: Assert.AreNotEqual for role change) - Acknowledged the exception assertions with specific exception types and parameter name verification - If any gaps were identified, they were presented as minor enhancement opportunities rather than problems + - The report acknowledges outright that this suite has good assertion diversity, instead of manufacturing + problems so it has something to flag - name: Identify self-referential assertions in identity and round-trip tests prompt: | I want an assertion quality analysis of my Config tests. They all pass diff --git a/tests/dotnet-test/crap-score/eval.yaml b/tests/dotnet-test/crap-score/eval.yaml index bd71e071ec..84da474dc8 100644 --- a/tests/dotnet-test/crap-score/eval.yaml +++ b/tests/dotnet-test/crap-score/eval.yaml @@ -98,8 +98,8 @@ stimuli: "Complexity: 4" comment above it' - Reported a complexity in the low-to-mid teens, counting the && and || operators, the ?? and the two ternaries as decision points — not 4 - - Used the 55% coverage recorded for ApplySurcharges in the Cobertura XML - - Reported a CRAP score in the high twenties and classified it as high risk + - Used the 60% coverage recorded for ApplySurcharges in the Cobertura XML + - Reported a CRAP score in the mid-to-high twenties (roughly 26) and classified it as high risk - Stated the coverage level required to bring the method under a CRAP threshold of 15 - name: Recognize when complexity alone blocks the CRAP threshold diff --git a/tests/dotnet-test/crap-score/fixtures/refactor-required/coverage.cobertura.xml b/tests/dotnet-test/crap-score/fixtures/refactor-required/coverage.cobertura.xml new file mode 100644 index 0000000000..f607aaf6ad --- /dev/null +++ b/tests/dotnet-test/crap-score/fixtures/refactor-required/coverage.cobertura.xml @@ -0,0 +1,65 @@ + + + + . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/dotnet-test/detect-static-dependencies/eval.yaml b/tests/dotnet-test/detect-static-dependencies/eval.yaml index 20f72d4d63..777355761a 100644 --- a/tests/dotnet-test/detect-static-dependencies/eval.yaml +++ b/tests/dotnet-test/detect-static-dependencies/eval.yaml @@ -152,6 +152,48 @@ stimuli: - Provided a structured summary or report of findings with categories or counts - Recommended wrapping or abstracting the detected static dependencies + - name: Keep one authoritative total with file line locations and no seam for pure helpers + prompt: > + Scan StaticHeavy/ and give me an audit I can act on directly. For every + dependency you report I need the file and the line number so I can jump + straight to it, and I need the arithmetic to hold: one grand total that + the category table, the top-pattern table and the per-file table all + agree with. Also tell me explicitly which calls you decided are NOT + worth putting behind a seam, and why. + environment: + files: + - src: ./fixtures/static-heavy-project + dest: StaticHeavy + graders: + - type: output-matches + config: + pattern: (\w+\.cs:\d+|\w+\.cs.{0,20}[Ll]ine\s*\d+) + - type: output-matches + config: + pattern: Path\.Combine + - type: output-matches + config: + pattern: (Guid\.NewGuid|random|identity) + - type: output-matches + config: + pattern: (JsonSerializer|serializ) + - type: exit-success + - type: prompt + rubric: + - Every reported occurrence carries a file and line location, not just a file name + - Reported a single authoritative grand total and did not park real call sites in a trailing + "additional observations" or "also noticed" section that the totals exclude + - The category totals, the top-patterns ranking, and the per-file counts all reconcile to the same + grand total + - Classified Path.Combine as a deterministic pure helper that needs no seam, and did not recommend + wrapping, abstracting, or injecting it + - Counted Path.GetTempPath as a File System dependency (it reads ambient machine state) while keeping + Path.Combine out of the needs-wrapping total + - Reported the Guid.NewGuid call site in OrderProcessor.CreateOrder under a randomness or identity + category + - Reported the JsonSerializer.Serialize call site in OrderProcessor.ArchiveOrder under a + culture or serialization category + - name: Detect statics inside lambda expressions and LINQ queries prompt: > My ReportGenerator class in LambdaStatics/ uses a lot of LINQ and diff --git a/tests/dotnet-test/migrate-static-to-wrapper/eval.yaml b/tests/dotnet-test/migrate-static-to-wrapper/eval.yaml index 0d01665345..baf1dc11c0 100644 --- a/tests/dotnet-test/migrate-static-to-wrapper/eval.yaml +++ b/tests/dotnet-test/migrate-static-to-wrapper/eval.yaml @@ -132,3 +132,46 @@ stimuli: - Defaulted the seam to TimeProvider.System so existing callers behave exactly as before - Replaced all three DateTime.UtcNow call sites with the seam - Did not convert the static class into an instance class or change any public method signature + + - name: Add the required using directive and update tests with a test double + prompt: > + Billing/Services/SubscriptionService.cs still calls DateTime.UtcNow directly. I already + have an IClock seam under Billing/Abstractions and it is registered in my composition + root. Migrate SubscriptionService to take IClock through its constructor and fix + Billing.Tests so the project still builds and the tests still pass. Only touch + SubscriptionService and its tests — the rest of the project is a later pass. + environment: + files: + - src: ./fixtures/with-tests + dest: WithTests + graders: + - type: file-contains + config: + path: "**/SubscriptionService.cs" + value: Billing.Abstractions + - type: file-not-contains + config: + path: "**/SubscriptionService.cs" + value: DateTime.UtcNow + - type: file-contains + config: + path: "**/SubscriptionServiceTests.cs" + value: Clock + - type: file-contains + config: + path: "**/InvoiceService.cs" + value: DateTime.UtcNow + - type: exit-success + - type: prompt + rubric: + - Added the required using directive for the Billing.Abstractions namespace to SubscriptionService.cs so + the IClock type resolves without fully qualifying it + - Injected IClock through the SubscriptionService constructor and stored it in a readonly field that follows + the existing naming convention + - Replaced every DateTime.UtcNow call site in SubscriptionService with the injected clock + - Test files were updated with an appropriate test double - a hand-rolled fake IClock or a mock - rather than + the production SystemClock or the real system time + - Ran the build and the tests, and reported the result that the commands actually produced + - Kept the change to the single class the user scoped it to instead of migrating the whole project or + namespace in one run, and left InvoiceService on DateTime.UtcNow + - Reported which production files and which test files were modified diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing.Tests/Billing.Tests.csproj b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing.Tests/Billing.Tests.csproj new file mode 100644 index 0000000000..4f98e9e990 --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing.Tests/Billing.Tests.csproj @@ -0,0 +1,14 @@ + + + net10.0 + enable + enable + false + + + + + + + + diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing.Tests/SubscriptionServiceTests.cs b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing.Tests/SubscriptionServiceTests.cs new file mode 100644 index 0000000000..f4def7ca06 --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing.Tests/SubscriptionServiceTests.cs @@ -0,0 +1,46 @@ +using Billing.Services; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Billing.Tests; + +[TestClass] +public sealed class SubscriptionServiceTests +{ + [TestMethod] + public void Start_RenewsAtIsStartPlusTerm() + { + var service = new SubscriptionService(); + + var subscription = service.Start("pro", 30); + + Assert.AreEqual(30d, (subscription.RenewsAt - subscription.StartedAt).TotalDays, 0.001); + } + + [TestMethod] + public void IsActive_ReturnsFalse_WhenRenewalDateHasPassed() + { + var service = new SubscriptionService(); + var subscription = new Subscription + { + PlanId = "pro", + StartedAt = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc), + RenewsAt = new DateTime(2020, 2, 1, 0, 0, 0, DateTimeKind.Utc) + }; + + Assert.IsFalse(service.IsActive(subscription)); + } + + [TestMethod] + public void DaysUntilRenewal_ReturnsZero_WhenRenewalDateHasPassed() + { + var service = new SubscriptionService(); + var subscription = new Subscription + { + PlanId = "pro", + StartedAt = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc), + RenewsAt = new DateTime(2020, 2, 1, 0, 0, 0, DateTimeKind.Utc) + }; + + Assert.AreEqual(0, service.DaysUntilRenewal(subscription)); + } +} diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Abstractions/IClock.cs b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Abstractions/IClock.cs new file mode 100644 index 0000000000..612fd6a39a --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Abstractions/IClock.cs @@ -0,0 +1,9 @@ +namespace Billing.Abstractions; + +/// +/// Ambient clock seam. Registered as a singleton in the composition root. +/// +public interface IClock +{ + DateTime UtcNow { get; } +} diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Abstractions/SystemClock.cs b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Abstractions/SystemClock.cs new file mode 100644 index 0000000000..1f63e4ec0b --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Abstractions/SystemClock.cs @@ -0,0 +1,6 @@ +namespace Billing.Abstractions; + +public sealed class SystemClock : IClock +{ + public DateTime UtcNow => DateTime.UtcNow; +} diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Billing.csproj b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Billing.csproj new file mode 100644 index 0000000000..6c3a88719f --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Billing.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Services/InvoiceService.cs b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Services/InvoiceService.cs new file mode 100644 index 0000000000..ca4ea04193 --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Services/InvoiceService.cs @@ -0,0 +1,15 @@ +namespace Billing.Services; + +/// +/// Deliberately left on the static clock — this class is scheduled for a later migration pass. +/// +public sealed class InvoiceService +{ + public DateTime NextInvoiceDate(int dayOfMonth) + { + var today = DateTime.UtcNow.Date; + var candidate = new DateTime(today.Year, today.Month, dayOfMonth, 0, 0, 0, DateTimeKind.Utc); + + return candidate > today ? candidate : candidate.AddMonths(1); + } +} diff --git a/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Services/SubscriptionService.cs b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Services/SubscriptionService.cs new file mode 100644 index 0000000000..e867852e67 --- /dev/null +++ b/tests/dotnet-test/migrate-static-to-wrapper/fixtures/with-tests/Billing/Services/SubscriptionService.cs @@ -0,0 +1,33 @@ +namespace Billing.Services; + +public sealed class SubscriptionService +{ + public Subscription Start(string planId, int termDays) + { + var now = DateTime.UtcNow; + + return new Subscription + { + PlanId = planId, + StartedAt = now, + RenewsAt = now.AddDays(termDays) + }; + } + + public bool IsActive(Subscription subscription) => DateTime.UtcNow < subscription.RenewsAt; + + public int DaysUntilRenewal(Subscription subscription) + { + var remaining = subscription.RenewsAt - DateTime.UtcNow; + return remaining > TimeSpan.Zero ? (int)Math.Ceiling(remaining.TotalDays) : 0; + } +} + +public sealed class Subscription +{ + public string PlanId { get; set; } = ""; + + public DateTime StartedAt { get; set; } + + public DateTime RenewsAt { get; set; } +} diff --git a/tests/dotnet-test/test-anti-patterns/eval.yaml b/tests/dotnet-test/test-anti-patterns/eval.yaml index b6c682d681..2b4bbc7df1 100644 --- a/tests/dotnet-test/test-anti-patterns/eval.yaml +++ b/tests/dotnet-test/test-anti-patterns/eval.yaml @@ -51,6 +51,11 @@ stimuli: ArgumentNullException - Identified that the static HttpClient field is never disposed and appears unused - Provided concrete fixes for the critical and high severity findings + - Every finding names a specific location — the test method it applies to — rather than issuing a general + warning about the suite + - The severity summary counts match the findings actually enumerated in the report; nothing is counted that + is not listed, and nothing listed is left out of the counts + - Recommendations are prioritized by severity, telling the user which findings to fix first constraints: reject_tools: - bash diff --git a/tests/dotnet-test/test-gap-analysis/eval.yaml b/tests/dotnet-test/test-gap-analysis/eval.yaml index 2afc92b29a..da61c76f6c 100644 --- a/tests/dotnet-test/test-gap-analysis/eval.yaml +++ b/tests/dotnet-test/test-gap-analysis/eval.yaml @@ -42,6 +42,14 @@ stimuli: - Identified that the heavy item fee boundary (weight > 10kg) is not tested - Identified that the coupon minimum floor (total cannot go below 1.00) is not tested - Recommended concrete test cases that would kill the survived mutations + - Ran the test suite and empirically confirmed every reported survivor by applying the mutation and re-running + the covering tests, or explicitly labelled the findings as unverified static reasoning + - Every survived mutation is labeled with its correct mutation category (boundary, relational operator, + arithmetic, return value, null check) + - Findings are prioritized by risk rather than listed in source order, with the discount and coupon-floor + survivors ahead of lower-impact ones + - Published a settled verdict — the report contains no in-flight reasoning such as "wait, this is actually + killed, let me recalibrate" - name: Find logic and null-check mutation gaps in access control code prompt: | I have an access control system and I'm worried the tests might not catch diff --git a/tests/dotnet-test/test-smell-detection/eval.yaml b/tests/dotnet-test/test-smell-detection/eval.yaml index 181688439b..d58622963b 100644 --- a/tests/dotnet-test/test-smell-detection/eval.yaml +++ b/tests/dotnet-test/test-smell-detection/eval.yaml @@ -98,6 +98,52 @@ stimuli: - Identified BulkInsert_RunsWithoutErrors as having no assertions — it exercises code but verifies nothing - Distinguished between the legitimate integration patterns (setup/teardown, multi-step persistence) and the actual smells (sleep, conditional, assertion-free) + + - name: Separate reasoned skips and self-documenting numbers from real smells + prompt: | + Audit `Inventory.Tests/InventoryServiceTests.cs` against the + testsmells.org catalog. Name each finding using the academic + taxonomy, and for every finding give me the severity plus the + reason you assigned that severity, and the fixed code — I don't + want "fix this", I want the replacement. If something looks like a + smell but isn't, tell me that too and say why. + environment: + files: + - src: fixtures/skip-and-magic/Inventory/Inventory.csproj + dest: Inventory/Inventory.csproj + - src: fixtures/skip-and-magic/Inventory/InventoryService.cs + dest: Inventory/InventoryService.cs + - src: fixtures/skip-and-magic/Inventory.Tests/Inventory.Tests.csproj + dest: Inventory.Tests/Inventory.Tests.csproj + - src: fixtures/skip-and-magic/Inventory.Tests/InventoryServiceTests.cs + dest: Inventory.Tests/InventoryServiceTests.cs + graders: + - type: output-matches + config: + pattern: (Ignored Test|skip|Skip) + - type: output-matches + config: + pattern: (Sleepy|Thread\.Sleep|sleep) + - type: output-matches + config: + pattern: (Assertion.?Free|no assert|without.*assert) + - type: output-matches + config: + pattern: (High|Medium|Low|severity) + - type: exit-success + - type: prompt + rubric: + - Flagged the Thread.Sleep(3000) in Replenish_AsyncJob_Completes as a High-severity Sleepy Test + - Flagged Audit_RunsWithoutError as an Assertion-Free Test — it exercises the service but verifies nothing + - Reported both skipped tests as Ignored Test, but treated the bare `Skip = "skip"` on + Restock_FromSupplier_UpdatesQuantities as more concerning than the reasoned skip on + Reserve_AcrossWarehouses_BalancesStock, which documents why and links a tracking issue + - Did not flag the contextually obvious numbers as Magic Number Test — asserting a count of 3 right after + adding three items, and a count of 1 after removing one of two, is self-documenting + - Severity levels are justified with a stated rationale rather than assigned arbitrarily + - Every finding includes a concrete fix example — the rewritten code — rather than a bare "fix this" + instruction + - Used xUnit terminology and APIs (Fact, Skip, Assert.Equal) rather than describing the suite in MSTest terms - name: Decline request to write new tests from scratch prompt: | I need to write unit tests for my ShoppingCart class. It supports diff --git a/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory.Tests/Inventory.Tests.csproj b/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory.Tests/Inventory.Tests.csproj index 45c3c9ede4..0ce26f6c7b 100644 --- a/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory.Tests/Inventory.Tests.csproj +++ b/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory.Tests/Inventory.Tests.csproj @@ -11,4 +11,7 @@ + + + diff --git a/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory/Inventory.csproj b/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory/Inventory.csproj new file mode 100644 index 0000000000..6c3a88719f --- /dev/null +++ b/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory/Inventory.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory/InventoryService.cs b/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory/InventoryService.cs new file mode 100644 index 0000000000..db7ae35812 --- /dev/null +++ b/tests/dotnet-test/test-smell-detection/fixtures/skip-and-magic/Inventory/InventoryService.cs @@ -0,0 +1,40 @@ +namespace Inventory; + +public sealed class InventoryService +{ + private readonly Dictionary _quantities = []; + private readonly HashSet _replenished = []; + + public int ItemCount => _quantities.Count; + + public void Add(string sku, int quantity) => _quantities[sku] = quantity; + + public void Remove(string sku) => _quantities.Remove(sku); + + public int QuantityOf(string sku) => _quantities.TryGetValue(sku, out var q) ? q : 0; + + public bool Reserve(string sku, int quantity) + { + if (QuantityOf(sku) < quantity) + { + return false; + } + + _quantities[sku] -= quantity; + return true; + } + + public void Restock(string sku, int quantity) => _quantities[sku] = QuantityOf(sku) + quantity; + + public void ReplenishAsync(string sku) => _replenished.Add(sku); + + public bool WasReplenished(string sku) => _replenished.Contains(sku); + + public void Audit() + { + foreach (var sku in _quantities.Keys) + { + _ = QuantityOf(sku); + } + } +} diff --git a/tests/dotnet-test/test-tagging/eval.yaml b/tests/dotnet-test/test-tagging/eval.yaml index 58cc562fe2..ba7558d28d 100644 --- a/tests/dotnet-test/test-tagging/eval.yaml +++ b/tests/dotnet-test/test-tagging/eval.yaml @@ -97,6 +97,9 @@ stimuli: config: path: OrderService.Tests/OrderProcessorTests.cs value: "[Category(" + - type: output-matches + config: + pattern: \[Category\b - type: output-matches config: pattern: positive|negative @@ -281,3 +284,41 @@ stimuli: changes - Used only valid trait values from the taxonomy — no invented category names that would cause logical errors - Every test method received at least one positive or negative tag + + - name: Classify a Go suite in a report without modifying any source files + prompt: My Go package at calculator/ has four tests and no build tags. Classify each one for me so I can see the + positive/negative split and where the gaps are. + environment: + files: + - src: fixtures/go-report-only/calculator/go.mod + dest: calculator/go.mod + - src: fixtures/go-report-only/calculator/calculator.go + dest: calculator/calculator.go + - src: fixtures/go-report-only/calculator/calculator_test.go + dest: calculator/calculator_test.go + graders: + - type: output-matches + config: + pattern: positive|negative + - type: output-matches + config: + pattern: (boundary|edge|zero) + - type: file-not-contains + config: + path: calculator/calculator_test.go + value: t.Skip + - type: file-not-contains + config: + path: calculator/calculator_test.go + value: "//go:build" + - type: prompt + rubric: + - Recognized that Go's testing package is a report-only framework — there is no trait attribute to add, so the + classification is delivered as a report + - For this report-only framework, no source files were modified — calculator_test.go is byte-for-byte unchanged + and no build tags, t.Skip calls, or comment markers were injected + - Classified TestAdd_ValidInputs_ReturnsSum and TestDivide_ValidInputs_ReturnsQuotient as positive + - Classified TestDivide_ByZero_ReturnsError as negative + - Classified TestAdd_ZeroOperands_ReturnsZero as boundary alongside its positive/negative classification + - Produced the trait summary table showing the distribution across the four tests + - Did not invent trait values outside the taxonomy