From dce8756e67d864a0b22229de2f95a93a96874417 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 18 Jul 2026 14:28:35 +0100 Subject: [PATCH 1/6] docs(evaluation): define routing quality gate [issue:#23] --- evaluation/QUALITY_POLICY.md | 36 ++++++++++++++++++++++++++++++++++ evaluation/quality-policy.json | 24 +++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 evaluation/QUALITY_POLICY.md create mode 100644 evaluation/quality-policy.json diff --git a/evaluation/QUALITY_POLICY.md b/evaluation/QUALITY_POLICY.md new file mode 100644 index 0000000..f2e48b7 --- /dev/null +++ b/evaluation/QUALITY_POLICY.md @@ -0,0 +1,36 @@ +# Routing quality policy + +Issue #23 uses this policy to decide whether a routing change is good enough to ship. The policy is committed before corpus corrections, prompt changes, or final measurements so the acceptance bar cannot move after results are known. + +## Ground truth + +- Expected folders follow the filing rules exactly. +- `YYYY` and `YYYY-MM` resolve from a reliable date stated in file content when one exists; otherwise they resolve from the evaluation date supplied to the prompt. +- An ambiguous case expects an empty folder only when its content and metadata do not support a safe classification and descriptive rename. +- Corpus corrections must include a reason and must be made before candidate measurements. Correcting demonstrably inconsistent ground truth is not counted as model improvement. +- Private documents, raw outputs, and identifying values remain outside the repository. + +## Comparable experiment + +- Use at least 12 cases and include receipts, scans, screenshots, PDFs, office documents, and ambiguous inputs. +- Run the unchanged PR #22 prompt against corrected ground truth to establish a new baseline. +- Evaluate the candidate on the same corpus, machine, OS, model, use case, extraction path, scorer, and generation settings. +- Publish at least three baseline and three candidate runs. Infrastructure-unavailable runs are reported but do not become quality scores. +- Keep Apple PCC outside the on-device acceptance gate. + +## Ship gate + +The candidate must meet every threshold in [`quality-policy.json`](quality-policy.json): + +- at least 80% aggregate exact decisions and at least 75% in every comparable run; +- at least 90% folder, 85% filename, and 90% tag accuracy; +- 100% safe abstention on genuinely ambiguous cases; +- no more than 5% generation failures; +- zero unsafe or invalid decisions reaching filesystem application; and +- no more than 25% mean-latency regression against the corrected baseline. + +A candidate that misses any threshold remains experimental. Negative and inconclusive results are published without weakening the gate. + +## Safety invariants + +Quality work must not weaken relative-path enforcement, traversal rejection, extension preservation, collision handling, opaque batch identity validation, explicit PCC consent, or the manual-review path. The model proposes; validated Swift code mutates. diff --git a/evaluation/quality-policy.json b/evaluation/quality-policy.json new file mode 100644 index 0000000..a7e5b82 --- /dev/null +++ b/evaluation/quality-policy.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "minimum_corpus_cases": 12, + "minimum_comparable_runs": 3, + "required_kinds": [ + "receipt", + "scan", + "screenshot", + "pdf", + "office_document", + "ambiguous" + ], + "thresholds": { + "aggregate_exact_accuracy": 0.8, + "minimum_run_exact_accuracy": 0.75, + "folder_accuracy": 0.9, + "filename_accuracy": 0.85, + "tag_accuracy": 0.9, + "ambiguous_abstention_accuracy": 1.0, + "maximum_generation_failure_rate": 0.05, + "maximum_unsafe_or_invalid_decisions": 0, + "maximum_mean_latency_regression": 0.25 + } +} From 2831277e270ec74c4ab6d996364e0b7bbfd10128 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 18 Jul 2026 14:57:38 +0100 Subject: [PATCH 2/6] feat(routing): resolve configured destinations safely [issue:#23] --- Sources/SortingHat/CLI.swift | 3 +- Sources/SortingHatCore/LiveEvaluation.swift | 71 +++-- Sources/SortingHatCore/Organizer.swift | 20 +- .../RoutingDecisionResolver.swift | 254 ++++++++++++++++++ .../live-evaluation-corpus.example.json | 2 +- Tests/SortingHatTests/SortingHatTests.swift | 178 +++++++++++- 6 files changed, 503 insertions(+), 25 deletions(-) create mode 100644 Sources/SortingHatCore/RoutingDecisionResolver.swift diff --git a/Sources/SortingHat/CLI.swift b/Sources/SortingHat/CLI.swift index 9bfe4df..dbd7687 100644 --- a/Sources/SortingHat/CLI.swift +++ b/Sources/SortingHat/CLI.swift @@ -56,7 +56,8 @@ enum SortingHatCLI { guardrails: config.appleGuardrails, pccAllowed: config.allowApplePCC) let configuration = EvaluationConfiguration(model: model.rawValue, useCase: config.appleUseCase.rawValue, guardrails: config.appleGuardrails.rawValue, pccAllowed: config.allowApplePCC, - promptVersion: FMAnalyzer.promptVersion, operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString) + promptVersion: FMAnalyzer.promptVersion, operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString, + routingPolicyVersion: RoutingDecisionResolver.version) let baseline: EvaluationArtifact? if let path = value(after: "--baseline", in: args) { let decoder = JSONDecoder(); decoder.dateDecodingStrategy = .iso8601 diff --git a/Sources/SortingHatCore/LiveEvaluation.swift b/Sources/SortingHatCore/LiveEvaluation.swift index 7eda430..c353384 100644 --- a/Sources/SortingHatCore/LiveEvaluation.swift +++ b/Sources/SortingHatCore/LiveEvaluation.swift @@ -46,14 +46,24 @@ public struct EvaluationConfiguration: Codable, Sendable { public let pccAllowed: Bool public let promptVersion: String public let operatingSystem: String + public let routingPolicyVersion: String? - public init(model: String, useCase: String, guardrails: String, pccAllowed: Bool, promptVersion: String, operatingSystem: String) { + public init( + model: String, + useCase: String, + guardrails: String, + pccAllowed: Bool, + promptVersion: String, + operatingSystem: String, + routingPolicyVersion: String? = nil + ) { self.model = model self.useCase = useCase self.guardrails = guardrails self.pccAllowed = pccAllowed self.promptVersion = promptVersion self.operatingSystem = operatingSystem + self.routingPolicyVersion = routingPolicyVersion } enum CodingKeys: String, CodingKey { @@ -62,6 +72,7 @@ public struct EvaluationConfiguration: Codable, Sendable { case pccAllowed = "pcc_allowed" case promptVersion = "prompt_version" case operatingSystem = "operating_system" + case routingPolicyVersion = "routing_policy_version" } } @@ -98,6 +109,7 @@ public struct EvaluationResult: Codable, Sendable { public let id: String public let kind: String public let latencyMilliseconds: Double + public let rawDecision: Decision? public let decision: Decision? public let error: String? public let folderCorrect: Bool @@ -108,6 +120,7 @@ public struct EvaluationResult: Codable, Sendable { enum CodingKeys: String, CodingKey { case id, kind, decision, error, abstained + case rawDecision = "raw_decision" case latencyMilliseconds = "latency_ms" case folderCorrect = "folder_correct" case filenameCorrect = "filename_correct" @@ -163,31 +176,35 @@ public enum LiveEvaluator { let results = manifest.cases.map { item -> EvaluationResult in let file = root.appending(path: item.path).standardizedFileURL let started = ContinuousClock.now + var rawDecision: Decision? do { - let decision = try analyzer.analyze(file: file, rules: manifest.rules) + let raw = try analyzer.analyze(file: file, rules: manifest.rules) + rawDecision = raw + let decision = try RoutingDecisionResolver.resolve(file: file, decision: raw, rules: manifest.rules) let elapsed = milliseconds(since: started) let abstained = decision.folder.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - let folderCorrect = item.expected.abstain ? abstained : item.expected.folders.contains(decision.folder) + let validationError = validate(file: file, decision: decision, rules: manifest.rules) + let valid = validationError == nil + let folderCorrect = valid && (item.expected.abstain ? abstained : item.expected.folders.contains(decision.folder)) let loweredName = decision.filename.lowercased() - let filenameCorrect = item.expected.filenameContains.allSatisfy { loweredName.contains($0.lowercased()) } + let filenameCorrect = valid && item.expected.filenameContains.allSatisfy { loweredName.contains($0.lowercased()) } let loweredTags = Set(decision.tags.map { $0.lowercased() }) - let tagsCorrect = item.expected.tags.allSatisfy { loweredTags.contains($0.lowercased()) } - let invalid = validate(file: file, decision: decision, rules: manifest.rules) != nil - return EvaluationResult(id: item.id, kind: item.kind, latencyMilliseconds: elapsed, decision: decision, - error: nil, folderCorrect: folderCorrect, filenameCorrect: filenameCorrect, - tagsCorrect: tagsCorrect, abstained: abstained, unsafeOrInvalid: invalid) + let tagsCorrect = valid && item.expected.tags.allSatisfy { loweredTags.contains($0.lowercased()) } + return EvaluationResult(id: item.id, kind: item.kind, latencyMilliseconds: elapsed, rawDecision: rawDecision, decision: decision, + error: validationError?.localizedDescription, folderCorrect: folderCorrect, filenameCorrect: filenameCorrect, + tagsCorrect: tagsCorrect, abstained: abstained, unsafeOrInvalid: !valid) } catch { return EvaluationResult(id: item.id, kind: item.kind, latencyMilliseconds: milliseconds(since: started), - decision: nil, error: error.localizedDescription, folderCorrect: false, + rawDecision: rawDecision, decision: nil, error: error.localizedDescription, folderCorrect: false, filenameCorrect: false, tagsCorrect: false, abstained: false, unsafeOrInvalid: isUnsafeOrInvalid(error)) } } let metrics = metrics(for: results) - return EvaluationArtifact(schemaVersion: 1, corpusName: manifest.name, createdAt: Date(), configuration: configuration, + return EvaluationArtifact(schemaVersion: 2, corpusName: manifest.name, createdAt: Date(), configuration: configuration, metrics: metrics, results: results, thresholdFailures: thresholdFailures(metrics, manifest.thresholds), - regressions: regressions(metrics, baseline?.metrics)) + regressions: regressions(metrics, baseline: baseline, corpusName: manifest.name, configuration: configuration)) } public static func write(_ artifact: EvaluationArtifact, to outputDirectory: URL) throws { @@ -208,6 +225,7 @@ public enum LiveEvaluator { Corpus: \(artifact.corpusName) Model: \(artifact.configuration.model) (\(artifact.configuration.useCase)) Prompt: \(artifact.configuration.promptVersion) + Routing policy: \(artifact.configuration.routingPolicyVersion ?? "legacy") OS: \(artifact.configuration.operatingSystem) | Metric | Result | @@ -262,12 +280,33 @@ public enum LiveEvaluator { return failures } - private static func regressions(_ metrics: EvaluationMetrics, _ baseline: EvaluationMetrics?) -> [String] { + private static func regressions( + _ metrics: EvaluationMetrics, + baseline: EvaluationArtifact?, + corpusName: String, + configuration: EvaluationConfiguration + ) -> [String] { guard let baseline else { return [] } + guard baseline.schemaVersion == 2 else { + return ["baseline artifact schema \(baseline.schemaVersion) is not comparable with schema 2"] + } + guard baseline.corpusName == corpusName, baseline.metrics.total == metrics.total else { + return ["baseline corpus does not match this evaluation"] + } + let baselineConfiguration = baseline.configuration + guard baselineConfiguration.model == configuration.model, + baselineConfiguration.useCase == configuration.useCase, + baselineConfiguration.guardrails == configuration.guardrails, + baselineConfiguration.pccAllowed == configuration.pccAllowed, + baselineConfiguration.operatingSystem == configuration.operatingSystem else { + return ["baseline model environment does not match this evaluation"] + } + var values: [String] = [] - if metrics.accuracy < baseline.accuracy { values.append("accuracy regressed from \(percent(baseline.accuracy)) to \(percent(metrics.accuracy))") } - if metrics.generationFailureRate > baseline.generationFailureRate { values.append("generation failure rate regressed") } - if metrics.unsafeDecisionRate > baseline.unsafeDecisionRate { values.append("unsafe/invalid decision rate regressed") } + let baselineMetrics = baseline.metrics + if metrics.accuracy < baselineMetrics.accuracy { values.append("accuracy regressed from \(percent(baselineMetrics.accuracy)) to \(percent(metrics.accuracy))") } + if metrics.generationFailureRate > baselineMetrics.generationFailureRate { values.append("generation failure rate regressed") } + if metrics.unsafeDecisionRate > baselineMetrics.unsafeDecisionRate { values.append("unsafe/invalid decision rate regressed") } return values } diff --git a/Sources/SortingHatCore/Organizer.swift b/Sources/SortingHatCore/Organizer.swift index d611d87..872f9c9 100644 --- a/Sources/SortingHatCore/Organizer.swift +++ b/Sources/SortingHatCore/Organizer.swift @@ -5,11 +5,22 @@ public struct Organizer { public let output: URL public let rules: [String] public let analyzer: any FileAnalyzing + public let referenceDate: Date public var fileManager = FileManager.default - public init(inbox: URL, output: URL? = nil, rules: [String], analyzer: any FileAnalyzing, fileManager: FileManager = .default) { + public init( + inbox: URL, + output: URL? = nil, + rules: [String], + analyzer: any FileAnalyzing, + referenceDate: Date = .now, + fileManager: FileManager = .default + ) { self.inbox = inbox self.output = output ?? inbox.deletingLastPathComponent() - self.rules = rules; self.analyzer = analyzer; self.fileManager = fileManager + self.rules = rules + self.analyzer = analyzer + self.referenceDate = referenceDate + self.fileManager = fileManager } public func candidates() throws -> [URL] { @@ -28,6 +39,10 @@ public struct Organizer { return try plan(file, decision: decision) } + public func resolve(_ file: URL, decision: Decision) throws -> Decision { + try RoutingDecisionResolver.resolve(file: file, decision: decision, rules: rules, referenceDate: referenceDate) + } + public func planAll(_ files: [URL]) -> [PlanningOutcome] { guard let batchAnalyzer = analyzer as? any BatchFileAnalyzing else { return files.map { file in @@ -60,6 +75,7 @@ public struct Organizer { } private func plan(_ file: URL, decision: Decision) throws -> PlannedMove { + let decision = try resolve(file, decision: decision) let proposedFolder = decision.folder.trimmingCharacters(in: .whitespacesAndNewlines) guard !proposedFolder.isEmpty else { throw HatError.needsReview(decision.reason) diff --git a/Sources/SortingHatCore/RoutingDecisionResolver.swift b/Sources/SortingHatCore/RoutingDecisionResolver.swift new file mode 100644 index 0000000..bcc8c9f --- /dev/null +++ b/Sources/SortingHatCore/RoutingDecisionResolver.swift @@ -0,0 +1,254 @@ +import Foundation + +/// Resolves model suggestions against the destinations the person actually configured. +/// The model still interprets content; this layer owns deterministic file and path contracts. +public enum RoutingDecisionResolver { + public static let version = "routing-rules-v1" + + public static func resolve( + file: URL, + decision: Decision, + rules: [String], + referenceDate: Date = .now + ) throws -> Decision { + let filename = preservingOriginalExtension(in: decision.filename, for: file) + let proposedFolder = decision.folder.trimmingCharacters(in: .whitespacesAndNewlines) + + guard !proposedFolder.isEmpty else { + return Decision(filename: filename, folder: "", tags: decision.tags, reason: decision.reason) + } + guard isSafeFolderShape(proposedFolder) else { throw HatError.unsafePath(proposedFolder) } + + let routes = rules.compactMap(CompiledRoutingRule.init) + guard !routes.isEmpty else { + return Decision(filename: filename, folder: proposedFolder, tags: decision.tags, reason: decision.reason) + } + + if let route = strongestSourceMatch(for: file, in: routes) { + let folder = try route.render(referenceDate: referenceDate, proposedFolder: proposedFolder) + return Decision( + filename: filename, + folder: folder, + tags: mergedTags(decision.tags, route.staticTags), + reason: decision.reason + ) + } + + guard let match = routes.lazy.compactMap({ route in + route.canonicalFolder(for: proposedFolder).map { (route, $0) } + }).first else { + throw HatError.invalidDecision("folder is not one of the configured destinations: \(proposedFolder)") + } + + if match.0.isCatchAll, shouldKeepForReview(decision) { + return Decision(filename: filename, folder: "", tags: decision.tags, reason: decision.reason) + } + + return Decision(filename: filename, folder: match.1, tags: decision.tags, reason: decision.reason) + } + + private static func preservingOriginalExtension(in proposed: String, for file: URL) -> String { + let filename = proposed.trimmingCharacters(in: .whitespacesAndNewlines) + guard isSafeFilenameShape(filename), !file.pathExtension.isEmpty else { return filename } + + let proposedURL = URL(fileURLWithPath: filename) + if proposedURL.pathExtension.isEmpty { return "\(filename).\(file.pathExtension)" } + guard proposedURL.pathExtension.caseInsensitiveCompare(file.pathExtension) != .orderedSame else { return filename } + + let stem = proposedURL.deletingPathExtension().lastPathComponent + guard !stem.isEmpty, stem != ".", stem != ".." else { return filename } + return "\(stem).\(file.pathExtension)" + } + + private static func isSafeFilenameShape(_ value: String) -> Bool { + !value.isEmpty && value != "." && value != ".." && + !value.contains("/") && !value.contains(":") && !value.hasPrefix("~") + } + + private static func isSafeFolderShape(_ value: String) -> Bool { + guard !value.hasPrefix("/"), !value.hasPrefix("~") else { return false } + let parts = value.split(separator: "/", omittingEmptySubsequences: false) + return parts.allSatisfy { !$0.isEmpty && $0 != "." && $0 != ".." } + } + + private static func strongestSourceMatch(for file: URL, in routes: [CompiledRoutingRule]) -> CompiledRoutingRule? { + var best: (route: CompiledRoutingRule, score: Int)? + for route in routes where !route.isCatchAll { + let score = route.sourceMatchScore(for: file) + if score > (best?.score ?? 0) { best = (route, score) } + } + return best?.route + } + + private static func shouldKeepForReview(_ decision: Decision) -> Bool { + let reason = decision.reason.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current) + let uncertaintyPhrases = [ + "insufficient", "not enough", "no clear", "lacks clear", "unclear", "ambiguous", + "cannot determine", "unable to determine", "unknown", "no dates or document type", + "no date or document type", "no dates or file-specific context", + ] + guard uncertaintyPhrases.contains(where: reason.contains) else { return false } + + let genericTags: Set = [ + "file", "files", "document", "documents", "general", "note", "other", "review", + "text", "uncategorized", "unknown", + ] + return decision.tags.allSatisfy { + genericTags.contains($0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()) + } + } + + private static func mergedTags(_ proposed: [String], _ configured: [String]) -> [String] { + var result = proposed + for tag in configured where !result.contains(where: { $0.caseInsensitiveCompare(tag) == .orderedSame }) { + result.append(tag) + } + return result + } +} + +struct CompiledRoutingRule: Equatable, Sendable { + let subject: String + let destinationTemplate: String + let staticTags: [String] + let isCatchAll: Bool + + init?(_ rule: String) { + let value = rule.trimmingCharacters(in: .whitespacesAndNewlines) + guard let prefix = value.range(of: "Put ", options: [.anchored, .caseInsensitive]) else { return nil } + let body = value[prefix.upperBound...] + guard let separator = body.range(of: " in ", options: .caseInsensitive) else { return nil } + + let subject = String(body[.. String? { + let templateParts = destinationTemplate.split(separator: "/", omittingEmptySubsequences: false).map(String.init) + let proposedParts = proposed.split(separator: "/", omittingEmptySubsequences: false).map(String.init) + guard templateParts.count == proposedParts.count else { return nil } + + var canonical: [String] = [] + for (template, value) in zip(templateParts, proposedParts) { + switch template.uppercased() { + case "YYYY": + guard Self.isYear(value) else { return nil } + canonical.append(value) + case "YYYY-MM": + guard Self.isYearMonth(value) else { return nil } + canonical.append(value) + default: + guard template.caseInsensitiveCompare(value) == .orderedSame else { return nil } + canonical.append(template) + } + } + return canonical.joined(separator: "/") + } + + func render(referenceDate: Date, proposedFolder: String) throws -> String { + let proposedDate = Self.folderDate(in: proposedFolder) + let reference = Calendar(identifier: .gregorian).dateComponents([.year, .month], from: referenceDate) + guard let referenceYear = reference.year, let referenceMonth = reference.month else { + throw HatError.invalidDecision("could not resolve destination date") + } + + let components = destinationTemplate.split(separator: "/", omittingEmptySubsequences: false).map { part -> String in + switch part.uppercased() { + case "YYYY": + return String(proposedDate.year ?? referenceYear) + case "YYYY-MM": + let year = proposedDate.year ?? referenceYear + let month = proposedDate.month ?? referenceMonth + return String(format: "%04d-%02d", year, month) + default: + return String(part) + } + } + return components.joined(separator: "/") + } + + func sourceMatchScore(for file: URL) -> Int { + let subjectTokens = Set(Self.tokens(subject).map(Self.singular).filter { !Self.subjectStopWords.contains($0) }) + guard subjectTokens.count == 1, let subjectToken = subjectTokens.first else { return 0 } + let filenameTokens = Set(Self.tokens(file.deletingPathExtension().lastPathComponent).map(Self.singular)) + if filenameTokens.contains(subjectToken) { return 101 } + + let fileExtension = Self.singular(file.pathExtension.lowercased()) + return !fileExtension.isEmpty && subjectToken == fileExtension ? 10 : 0 + } + + private static func isSupportedTemplate(_ value: String) -> Bool { + guard !value.isEmpty, !value.hasPrefix("/"), !value.hasPrefix("~") else { return false } + let parts = value.split(separator: "/", omittingEmptySubsequences: false).map(String.init) + guard parts.allSatisfy({ !$0.isEmpty && $0 != "." && $0 != ".." }) else { return false } + return parts.allSatisfy { part in + !part.uppercased().contains("YYYY") || part.uppercased() == "YYYY" || part.uppercased() == "YYYY-MM" + } + } + + private static func parseStaticTags(from tail: String) -> [String] { + guard let marker = tail.range(of: "tag them ", options: .caseInsensitive) else { return [] } + let value = String(tail[marker.upperBound...]).trimmingCharacters( + in: .whitespacesAndNewlines.union(CharacterSet(charactersIn: ".")) + ) + return value.components(separatedBy: " and ").compactMap { candidate in + let tag = candidate.trimmingCharacters(in: .whitespacesAndNewlines.union(CharacterSet(charactersIn: ","))) + let lowered = tag.lowercased() + guard !tag.isEmpty, !lowered.hasPrefix("the "), !lowered.hasPrefix("a "), !lowered.hasPrefix("an ") else { return nil } + return tag + } + } + + private static func folderDate(in folder: String) -> (year: Int?, month: Int?) { + for part in folder.split(separator: "/").map(String.init) { + if isYearMonth(part) { + let values = part.split(separator: "-").compactMap { Int($0) } + return (values[0], values[1]) + } + } + for part in folder.split(separator: "/").map(String.init) where isYear(part) { + return (Int(part), nil) + } + return (nil, nil) + } + + private static func isYear(_ value: String) -> Bool { + value.count == 4 && value.allSatisfy(\.isNumber) && (1900...2999).contains(Int(value) ?? 0) + } + + private static func isYearMonth(_ value: String) -> Bool { + let parts = value.split(separator: "-", omittingEmptySubsequences: false) + guard parts.count == 2, isYear(String(parts[0])), parts[1].count == 2, let month = Int(parts[1]) else { return false } + return (1...12).contains(month) + } + + private static func tokens(_ value: String) -> [String] { + value.lowercased().components(separatedBy: CharacterSet.alphanumerics.inverted).filter { !$0.isEmpty } + } + + private static func singular(_ value: String) -> String { + if value.hasSuffix("ies"), value.count > 3 { return String(value.dropLast(3)) + "y" } + if value.hasSuffix("s"), !value.hasSuffix("ss"), value.count > 1 { return String(value.dropLast()) } + return value + } + + private static func normalizedPhrase(_ value: String) -> String { tokens(value).map(singular).joined(separator: " ") } + + private static let catchAllSubjects: Set = [ + "all file", "all other file", "anything else", "every file", "everything else", "other file", + ] + private static let subjectStopWords: Set = [ + "a", "all", "an", "and", "any", "document", "else", "every", "file", "item", "my", "or", "other", "the", + ] +} diff --git a/Tests/SortingHatTests/Fixtures/live-evaluation-corpus.example.json b/Tests/SortingHatTests/Fixtures/live-evaluation-corpus.example.json index 157e0f6..10af011 100644 --- a/Tests/SortingHatTests/Fixtures/live-evaluation-corpus.example.json +++ b/Tests/SortingHatTests/Fixtures/live-evaluation-corpus.example.json @@ -28,7 +28,7 @@ "id": "scan-001", "path": "documents/scan-001.pdf", "kind": "scan", - "expected": { "folders": ["Files/2026-07"], "filename_contains": ["form"], "tags": [], "abstain": false } + "expected": { "folders": ["Files/2026-07"], "filename_contains": ["registration"], "tags": [], "abstain": false } }, { "id": "screenshot-001", diff --git a/Tests/SortingHatTests/SortingHatTests.swift b/Tests/SortingHatTests/SortingHatTests.swift index 3635341..9203610 100644 --- a/Tests/SortingHatTests/SortingHatTests.swift +++ b/Tests/SortingHatTests/SortingHatTests.swift @@ -15,10 +15,32 @@ struct EvaluationAnalyzer: FileAnalyzing { if file.lastPathComponent == "unsafe.txt" { return Decision(filename: "unsafe-renamed.txt", folder: "../Escape", tags: [], reason: "unsafe") } + if file.lastPathComponent == "unchanged.txt" { + return Decision(filename: "unchanged.txt", folder: "Files/2026-07", tags: [], reason: "unchanged") + } return Decision(filename: "tesco-receipt.txt", folder: "Receipts/2026", tags: ["receipt", "tesco"], reason: "receipt") } } +struct RoutingEvaluationAnalyzer: FileAnalyzing { + func analyze(file: URL, rules: [String]) throws -> Decision { + if file.lastPathComponent.hasPrefix("screenshot") { + return Decision( + filename: "settings.txt", + folder: "Files/2026-07", + tags: ["settings"], + reason: "No dates or file-specific context to classify this text" + ) + } + return Decision( + filename: "follow-up-note.txt", + folder: "Files/2026-07", + tags: ["note"], + reason: "No dates or document types identified in text" + ) + } +} + struct OCRRequiringAnalyzer: FileAnalyzing { func analyze(file: URL, rules: [String]) throws -> Decision { _ = try DocumentTextExtractor.extractContent(from: file) @@ -125,24 +147,32 @@ struct SortingHatTests { try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) let receipt = root.appending(path: "receipt.txt") let unsafe = root.appending(path: "unsafe.txt") + let unchanged = root.appending(path: "unchanged.txt") try "TESCO total GBP 42.18".write(to: receipt, atomically: true, encoding: .utf8) try "untrusted".write(to: unsafe, atomically: true, encoding: .utf8) + try "unchanged".write(to: unchanged, atomically: true, encoding: .utf8) let originalReceipt = try Data(contentsOf: receipt) let manifest = EvaluationManifest(version: 1, name: "synthetic", rules: ["File receipts"], cases: [ EvaluationCase(id: "receipt", path: "receipt.txt", kind: "receipt", expected: ExpectedDecision( folders: ["Receipts/2026"], filenameContains: ["tesco", "receipt"], tags: ["receipt"], abstain: false)), EvaluationCase(id: "unsafe", path: "unsafe.txt", kind: "ambiguous", expected: ExpectedDecision( folders: ["Files/2026-07"], filenameContains: [], tags: [], abstain: false)), + EvaluationCase(id: "unchanged", path: "unchanged.txt", kind: "ambiguous", expected: ExpectedDecision( + folders: ["Files/2026-07"], filenameContains: ["unchanged"], tags: [], abstain: false)), ], thresholds: EvaluationThresholds(minimumAccuracy: 0.5, maximumGenerationFailureRate: 0, maximumUnsafeDecisionRate: 0)) let configuration = EvaluationConfiguration(model: "system", useCase: "general", guardrails: "default", pccAllowed: false, promptVersion: "test", operatingSystem: "testOS") let artifact = LiveEvaluator.run(manifest: manifest, corpusRoot: root, analyzer: EvaluationAnalyzer(), configuration: configuration) - #expect(artifact.metrics.total == 2) + #expect(artifact.metrics.total == 3) #expect(artifact.metrics.correct == 1) - #expect(artifact.metrics.unsafeOrInvalidDecisions == 1) + #expect(artifact.metrics.unsafeOrInvalidDecisions == 2) #expect(artifact.thresholdFailures.contains { $0.contains("unsafe/invalid") }) + #expect(artifact.results[2].error?.contains("original filename unchanged") == true) + #expect(!artifact.results[2].folderCorrect) + #expect(!artifact.results[2].filenameCorrect) + #expect(!artifact.results[2].tagsCorrect) #expect(try Data(contentsOf: receipt) == originalReceipt) #expect(FileManager.default.fileExists(atPath: unsafe.path)) } @@ -161,7 +191,7 @@ struct SortingHatTests { let baselineMetrics = EvaluationMetrics(total: 1, correct: 1, folderCorrect: 1, filenameCorrect: 1, tagsCorrect: 1, generationFailures: 0, schemaFailures: 0, unsafeOrInvalidDecisions: 0, abstentions: 0, accuracy: 1, generationFailureRate: 0, unsafeDecisionRate: 0, averageLatencyMilliseconds: 1) - let baseline = EvaluationArtifact(schemaVersion: 1, corpusName: "synthetic", createdAt: Date(), configuration: configuration, + let baseline = EvaluationArtifact(schemaVersion: 2, corpusName: "synthetic", createdAt: Date(), configuration: configuration, metrics: baselineMetrics, results: [], thresholdFailures: [], regressions: []) let artifact = LiveEvaluator.run(manifest: manifest, corpusRoot: root, analyzer: EvaluationAnalyzer(), @@ -175,6 +205,67 @@ struct SortingHatTests { #expect(summary.contains("accuracy regressed")) } + @Test func refusesAutomaticRegressionComparisonAcrossArtifactSchemas() throws { + let root = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + try "TESCO".write(to: root.appending(path: "receipt.txt"), atomically: true, encoding: .utf8) + let manifest = EvaluationManifest(version: 1, name: "synthetic", rules: ["File receipts"], cases: [ + EvaluationCase(id: "receipt", path: "receipt.txt", kind: "receipt", expected: ExpectedDecision( + folders: ["Receipts/2026"], filenameContains: ["receipt"], tags: ["receipt"], abstain: false)), + ], thresholds: nil) + let configuration = EvaluationConfiguration(model: "system", useCase: "general", guardrails: "default", + pccAllowed: false, promptVersion: "test", operatingSystem: "testOS") + let metrics = EvaluationMetrics(total: 1, correct: 0, folderCorrect: 0, filenameCorrect: 0, tagsCorrect: 0, + generationFailures: 0, schemaFailures: 0, unsafeOrInvalidDecisions: 0, abstentions: 0, accuracy: 0, + generationFailureRate: 0, unsafeDecisionRate: 0, averageLatencyMilliseconds: 1) + let legacy = EvaluationArtifact(schemaVersion: 1, corpusName: "synthetic", createdAt: Date(), configuration: configuration, + metrics: metrics, results: [], thresholdFailures: [], regressions: []) + + let artifact = LiveEvaluator.run(manifest: manifest, corpusRoot: root, analyzer: EvaluationAnalyzer(), + configuration: configuration, baseline: legacy) + + #expect(artifact.regressions == ["baseline artifact schema 1 is not comparable with schema 2"]) + } + + @Test func liveEvaluationScoresResolvedShippingDecisionAndRetainsRawDiagnostics() throws { + let root = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let screenshot = root.appending(path: "screenshot-settings.png") + let ambiguous = root.appending(path: "unclear.txt") + FileManager.default.createFile(atPath: screenshot.path, contents: Data()) + try "Review later. Put it with the other one.".write(to: ambiguous, atomically: true, encoding: .utf8) + let screenshotData = try Data(contentsOf: screenshot) + let rules = [ + "Put screenshots in Screenshots/YYYY-MM and tag them screenshot.", + "Put everything else in Files/YYYY-MM.", + ] + let manifest = EvaluationManifest(version: 1, name: "routing", rules: rules, cases: [ + EvaluationCase(id: "screenshot", path: screenshot.lastPathComponent, kind: "screenshot", expected: ExpectedDecision( + folders: ["Screenshots/2026-07"], filenameContains: ["settings"], tags: ["screenshot"], abstain: false)), + EvaluationCase(id: "ambiguous", path: ambiguous.lastPathComponent, kind: "ambiguous", expected: ExpectedDecision( + folders: [], filenameContains: [], tags: [], abstain: true)), + ], thresholds: nil) + let configuration = EvaluationConfiguration(model: "system", useCase: "general", guardrails: "default", + pccAllowed: false, promptVersion: "test", operatingSystem: "testOS", routingPolicyVersion: RoutingDecisionResolver.version) + + let artifact = LiveEvaluator.run( + manifest: manifest, + corpusRoot: root, + analyzer: RoutingEvaluationAnalyzer(), + configuration: configuration + ) + + #expect(artifact.schemaVersion == 2) + #expect(artifact.metrics.correct == 2) + #expect(artifact.results[0].rawDecision?.folder == "Files/2026-07") + #expect(artifact.results[0].decision?.folder == "Screenshots/2026-07") + #expect(artifact.results[0].decision?.filename == "settings.png") + #expect(artifact.results[1].rawDecision?.folder == "Files/2026-07") + #expect(artifact.results[1].decision?.folder == "") + #expect(try Data(contentsOf: screenshot) == screenshotData) + #expect(FileManager.default.fileExists(atPath: ambiguous.path)) + } + @Test func parsesHumanReadableConfig() throws { let url = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) try """ @@ -412,13 +503,16 @@ struct SortingHatTests { #expect(throws: HatError.self) { try Organizer(inbox: inbox, rules: ["Rename files"], analyzer: analyzer).plan(source) } } - @Test func rejectsChangedFileExtension() throws { + @Test func repairsChangedFileExtensionWithoutChangingTheContainer() throws { let root = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) let source = root.appending(path: "receipt.pdf") FileManager.default.createFile(atPath: source.path, contents: Data()) let analyzer = StubAnalyzer(decision: Decision(filename: "tesco-receipt.jpg", folder: "Receipts", tags: [], reason: "receipt")) - #expect(throws: HatError.self) { try Organizer(inbox: root, rules: ["Rename files"], analyzer: analyzer).plan(source) } + + let move = try Organizer(inbox: root, rules: ["Rename files"], analyzer: analyzer).plan(source) + + #expect(move.destination.lastPathComponent == "tesco-receipt.pdf") } @Test func restoresMissingOriginalFileExtension() throws { @@ -451,6 +545,80 @@ struct SortingHatTests { #expect(FileManager.default.fileExists(atPath: source.path)) } + @Test func compilesOnlyControlledPutRoutesWithUnicodeDestinations() throws { + #expect(CompiledRoutingRule("Rename files to lowercase") == nil) + let route = try #require(CompiledRoutingRule("Put client reports in Client Files/Årsrapporter/YYYY.")) + let catchAll = try #require(CompiledRoutingRule("Put all other files in Files/YYYY-MM.")) + #expect(route.subject == "client reports") + #expect(route.destinationTemplate == "Client Files/Årsrapporter/YYYY") + #expect(route.canonicalFolder(for: "client files/årsrapporter/2026") == "Client Files/Årsrapporter/2026") + #expect(route.sourceMatchScore(for: URL(fileURLWithPath: "/tmp/report.pdf")) == 0) + #expect(catchAll.isCatchAll) + } + + @Test func resolvesStrongSourceRouteCanonicalFolderExtensionAndTags() throws { + let root = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let source = root.appending(path: "Screenshot 42.png") + FileManager.default.createFile(atPath: source.path, contents: Data()) + let rules = [ + "Put screenshots in Screenshots/YYYY-MM and tag them screenshot.", + "Put everything else in Files/YYYY-MM.", + ] + let analyzer = StubAnalyzer(decision: Decision( + filename: "sorting-hat-settings.txt", + folder: "files/2026-07", + tags: ["settings"], + reason: "No dates or file-specific context to classify this text" + )) + + let move = try Organizer(inbox: root, output: root.appending(path: "Filed"), rules: rules, analyzer: analyzer).plan(source) + + #expect(move.destination.path.hasSuffix("Filed/Screenshots/2026-07/sorting-hat-settings.png")) + #expect(move.tags == ["settings", "screenshot"]) + } + + @Test func keepsExplicitlyUncertainCatchAllDecisionForReview() throws { + let file = URL(fileURLWithPath: "/tmp/unclear.txt") + let rules = ["Put receipts in Receipts/YYYY.", "Put everything else in Files/YYYY-MM."] + let ambiguous = Decision( + filename: "follow-up-note.txt", + folder: "Files/2026-07", + tags: ["note"], + reason: "No dates or document types identified in text" + ) + let useful = Decision( + filename: "accessibility-checklist.txt", + folder: "files/2026-07", + tags: ["accessibility"], + reason: "No dates or document type keywords found in content" + ) + + let held = try RoutingDecisionResolver.resolve(file: file, decision: ambiguous, rules: rules) + let filed = try RoutingDecisionResolver.resolve(file: file, decision: useful, rules: rules) + + #expect(held.folder == "") + #expect(filed.folder == "Files/2026-07") + } + + @Test func rejectsUnconfiguredAndUnresolvedControlledDestinations() throws { + let file = URL(fileURLWithPath: "/tmp/note.txt") + let rules = ["Put everything else in Files/YYYY-MM."] + let unknown = Decision(filename: "planning-note.txt", folder: "Archive/2026-07", tags: [], reason: "planning") + let unresolved = Decision(filename: "planning-note.txt", folder: "Files/YYYY-MM", tags: [], reason: "planning") + + #expect(throws: HatError.self) { try RoutingDecisionResolver.resolve(file: file, decision: unknown, rules: rules) } + #expect(throws: HatError.self) { try RoutingDecisionResolver.resolve(file: file, decision: unresolved, rules: rules) } + } + + @Test func doesNotRepairUnsafeFolderEvenWhenSourceNameMatchesARoute() throws { + let file = URL(fileURLWithPath: "/tmp/screenshot.png") + let rules = ["Put screenshots in Screenshots/YYYY-MM."] + let unsafe = Decision(filename: "settings.png", folder: "../Escape", tags: [], reason: "screenshot") + + #expect(throws: HatError.self) { try RoutingDecisionResolver.resolve(file: file, decision: unsafe, rules: rules) } + } + @Test func boundsExtractedDocumentText() throws { let file = FileManager.default.temporaryDirectory.appending(path: "\(UUID().uuidString).txt") try String(repeating: "receipt ", count: 100).write(to: file, atomically: true, encoding: .utf8) From fa5f2683f75762933165b88f5d801c6500d39085 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 18 Jul 2026 16:04:10 +0100 Subject: [PATCH 3/6] fix(evaluation): harden routing artifact fidelity [issue:#23] --- Sources/SortingHatCore/LiveEvaluation.swift | 43 ++++++++++--- Tests/SortingHatTests/SortingHatTests.swift | 68 ++++++++++++++++++++- 2 files changed, 102 insertions(+), 9 deletions(-) diff --git a/Sources/SortingHatCore/LiveEvaluation.swift b/Sources/SortingHatCore/LiveEvaluation.swift index c353384..66f3351 100644 --- a/Sources/SortingHatCore/LiveEvaluation.swift +++ b/Sources/SortingHatCore/LiveEvaluation.swift @@ -173,6 +173,16 @@ public enum LiveEvaluator { baseline: EvaluationArtifact? = nil ) -> EvaluationArtifact { let root = corpusRoot.standardizedFileURL + let referenceDate = Date() + let recordedConfiguration = EvaluationConfiguration( + model: configuration.model, + useCase: configuration.useCase, + guardrails: configuration.guardrails, + pccAllowed: configuration.pccAllowed, + promptVersion: configuration.promptVersion, + operatingSystem: configuration.operatingSystem, + routingPolicyVersion: RoutingDecisionResolver.version + ) let results = manifest.cases.map { item -> EvaluationResult in let file = root.appending(path: item.path).standardizedFileURL let started = ContinuousClock.now @@ -180,10 +190,20 @@ public enum LiveEvaluator { do { let raw = try analyzer.analyze(file: file, rules: manifest.rules) rawDecision = raw - let decision = try RoutingDecisionResolver.resolve(file: file, decision: raw, rules: manifest.rules) + let decision = try RoutingDecisionResolver.resolve( + file: file, + decision: raw, + rules: manifest.rules, + referenceDate: referenceDate + ) let elapsed = milliseconds(since: started) let abstained = decision.folder.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - let validationError = validate(file: file, decision: decision, rules: manifest.rules) + let validationError = validate( + file: file, + decision: decision, + rules: manifest.rules, + referenceDate: referenceDate + ) let valid = validationError == nil let folderCorrect = valid && (item.expected.abstain ? abstained : item.expected.folders.contains(decision.folder)) let loweredName = decision.filename.lowercased() @@ -201,10 +221,10 @@ public enum LiveEvaluator { } } let metrics = metrics(for: results) - return EvaluationArtifact(schemaVersion: 2, corpusName: manifest.name, createdAt: Date(), configuration: configuration, + return EvaluationArtifact(schemaVersion: 2, corpusName: manifest.name, createdAt: referenceDate, configuration: recordedConfiguration, metrics: metrics, results: results, thresholdFailures: thresholdFailures(metrics, manifest.thresholds), - regressions: regressions(metrics, baseline: baseline, corpusName: manifest.name, configuration: configuration)) + regressions: regressions(metrics, baseline: baseline, corpusName: manifest.name, configuration: recordedConfiguration)) } public static func write(_ artifact: EvaluationArtifact, to outputDirectory: URL) throws { @@ -246,13 +266,21 @@ public enum LiveEvaluator { """ } - private static func validate(file: URL, decision: Decision, rules: [String]) -> Error? { + private static func validate(file: URL, decision: Decision, rules: [String], referenceDate: Date) -> Error? { if decision.folder.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return nil } struct FixedAnalyzer: FileAnalyzing { let decision: Decision func analyze(file: URL, rules: [String]) throws -> Decision { decision } } - do { _ = try Organizer(inbox: file.deletingLastPathComponent(), rules: rules, analyzer: FixedAnalyzer(decision: decision)).plan(file); return nil } + do { + _ = try Organizer( + inbox: file.deletingLastPathComponent(), + rules: rules, + analyzer: FixedAnalyzer(decision: decision), + referenceDate: referenceDate + ).plan(file) + return nil + } catch { return error } } @@ -298,7 +326,8 @@ public enum LiveEvaluator { baselineConfiguration.useCase == configuration.useCase, baselineConfiguration.guardrails == configuration.guardrails, baselineConfiguration.pccAllowed == configuration.pccAllowed, - baselineConfiguration.operatingSystem == configuration.operatingSystem else { + baselineConfiguration.operatingSystem == configuration.operatingSystem, + baselineConfiguration.routingPolicyVersion == configuration.routingPolicyVersion else { return ["baseline model environment does not match this evaluation"] } diff --git a/Tests/SortingHatTests/SortingHatTests.swift b/Tests/SortingHatTests/SortingHatTests.swift index 9203610..53e0463 100644 --- a/Tests/SortingHatTests/SortingHatTests.swift +++ b/Tests/SortingHatTests/SortingHatTests.swift @@ -187,7 +187,7 @@ struct SortingHatTests { folders: ["Wrong"], filenameContains: ["receipt"], tags: ["receipt"], abstain: false)), ], thresholds: nil) let configuration = EvaluationConfiguration(model: "system", useCase: "general", guardrails: "default", - pccAllowed: false, promptVersion: "test", operatingSystem: "testOS") + pccAllowed: false, promptVersion: "test", operatingSystem: "testOS", routingPolicyVersion: RoutingDecisionResolver.version) let baselineMetrics = EvaluationMetrics(total: 1, correct: 1, folderCorrect: 1, filenameCorrect: 1, tagsCorrect: 1, generationFailures: 0, schemaFailures: 0, unsafeOrInvalidDecisions: 0, abstentions: 0, accuracy: 1, generationFailureRate: 0, unsafeDecisionRate: 0, averageLatencyMilliseconds: 1) @@ -227,6 +227,69 @@ struct SortingHatTests { #expect(artifact.regressions == ["baseline artifact schema 1 is not comparable with schema 2"]) } + @Test func decodesSchemaOneArtifactWithoutRoutingPolicyOrRawDecision() throws { + let data = Data(#""" + { + "schema_version": 1, + "corpus_name": "legacy-synthetic", + "created_at": "2026-07-18T12:00:00Z", + "configuration": { + "model": "system", + "use_case": "general", + "guardrails": "default", + "pcc_allowed": false, + "prompt_version": "sorting-decision-v2", + "operating_system": "macOS 27" + }, + "metrics": { + "total": 1, + "correct": 1, + "folder_correct": 1, + "filename_correct": 1, + "tags_correct": 1, + "generation_failures": 0, + "schema_failures": 0, + "unsafe_or_invalid_decisions": 0, + "abstentions": 0, + "accuracy": 1, + "generation_failure_rate": 0, + "unsafe_decision_rate": 0, + "average_latency_ms": 12 + }, + "results": [ + { + "id": "receipt", + "kind": "receipt", + "latency_ms": 12, + "decision": { + "filename": "tesco-receipt.txt", + "folder": "Receipts/2026", + "tags": ["receipt"], + "reason": "receipt" + }, + "error": null, + "folder_correct": true, + "filename_correct": true, + "tags_correct": true, + "abstained": false, + "unsafe_or_invalid": false + } + ], + "threshold_failures": [], + "regressions": [] + } + """#.utf8) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + + let artifact = try decoder.decode(EvaluationArtifact.self, from: data) + + #expect(artifact.schemaVersion == 1) + #expect(artifact.configuration.routingPolicyVersion == nil) + #expect(artifact.results[0].rawDecision == nil) + #expect(artifact.results[0].decision?.folder == "Receipts/2026") + } + @Test func liveEvaluationScoresResolvedShippingDecisionAndRetainsRawDiagnostics() throws { let root = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) @@ -246,7 +309,7 @@ struct SortingHatTests { folders: [], filenameContains: [], tags: [], abstain: true)), ], thresholds: nil) let configuration = EvaluationConfiguration(model: "system", useCase: "general", guardrails: "default", - pccAllowed: false, promptVersion: "test", operatingSystem: "testOS", routingPolicyVersion: RoutingDecisionResolver.version) + pccAllowed: false, promptVersion: "test", operatingSystem: "testOS") let artifact = LiveEvaluator.run( manifest: manifest, @@ -256,6 +319,7 @@ struct SortingHatTests { ) #expect(artifact.schemaVersion == 2) + #expect(artifact.configuration.routingPolicyVersion == RoutingDecisionResolver.version) #expect(artifact.metrics.correct == 2) #expect(artifact.results[0].rawDecision?.folder == "Files/2026-07") #expect(artifact.results[0].decision?.folder == "Screenshots/2026-07") From 61ab33a1e1942b8acf1279de706f999678e73416 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 18 Jul 2026 16:05:47 +0100 Subject: [PATCH 4/6] docs(evaluation): publish routing quality result [issue:#23] --- README.md | 10 +++-- docs/hacker-news-draft.md | 8 ++-- docs/wwdc26-comparison.md | 80 +++++++++++++++++++---------------- evaluation/README.md | 4 +- evaluation/ROUTING_RESULTS.md | 71 +++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 44 deletions(-) create mode 100644 evaluation/ROUTING_RESULTS.md diff --git a/README.md b/README.md index 0ac6833..df8aa21 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ./script/build_and_run.sh ``` -[See exactly how the product extends Apple’s WWDC26 file-sorting demo, including the reproducible negative benchmark results.](docs/wwdc26-comparison.md) +[See exactly how the product extends Apple’s WWDC26 file-sorting demo, including the passing shipping-path benchmark and its limitations.](docs/wwdc26-comparison.md) > [!WARNING] > The current `v0.1.0` build is an experimental pre-release. Its app bundle is ad-hoc signed, not signed with an Apple Developer ID, and it is not notarized. Gatekeeper may block the first launch; managed Macs may prohibit it entirely. Homebrew installation confirms the archive and cask are valid, but does not bypass these macOS security checks. @@ -69,7 +69,7 @@ Sorting Hat reads searchable PDFs, plain-text formats, RTF, Word, and OpenDocume When Apple `fm` is selected, compatible non-image files are analyzed in batches of at most 8 files and 24,000 extracted characters. Each file is identified by an opaque request-local ID, and every returned decision is independently validated before any move. Missing, duplicate, unexpected, or unsafe decisions cannot be applied. Images remain on the individual multimodal path, and a failed batch does not prevent files in other batches from being processed. A deterministic process benchmark verifies that 8 compatible files use 1 `fm respond` invocation instead of 8. -The Inbox is intake-only. Sorting Hat renames each file and moves it to a rule-specific folder under `output` (for example, `~/SortingHat/Receipts/2026`). It creates those destination folders as needed, rejects absolute paths and traversal, preserves existing files with numbered names, and writes tags as Finder metadata. +The Inbox is intake-only. Sorting Hat renames each file and moves it to a rule-specific folder under `output` (for example, `~/SortingHat/Receipts/2026`). For the rule builder's controlled `Put ... in ...` routes, deterministic Swift code canonicalises the model's proposal against configured destinations and rejects unknown or unresolved folders. It creates destination folders as needed, rejects absolute paths and traversal, preserves the source file type, protects existing files with numbered names, and writes tags as Finder metadata. If a model returns the uploaded filename unchanged, Sorting Hat leaves the source in the Inbox and reports an invalid sorting decision instead of silently skipping the requested rename. @@ -97,9 +97,11 @@ Live quality checks are deliberately opt-in and read-only. Create a private, ano The manifest is versioned and contains rules plus cases with a relative source path, content kind (`receipt`, `scan`, `screenshot`, `pdf`, `office_document`, or `ambiguous`), accepted folders, required filename terms, required tags, and whether an empty-folder abstention is expected. The checked-in template defines all six required classes but deliberately includes no documents. Corpus files must sit beside or below the manifest; output inside that corpus directory is rejected. The evaluator calls Apple `fm` but never calls `Organizer.apply`, so sources are not renamed, tagged, or moved. -Each run writes `evaluation.json` and `summary.md`. The artifact records model, OS, prompt version, use case, guardrails, per-case decisions and latency, accuracy, generation failures, unsafe/invalid decisions, and abstentions. Pass a previous `evaluation.json` with `--baseline` to expose regressions in the Markdown summary. Exit status `2` means a threshold or baseline regressed; `1` means the evaluation could not run. Do not commit corpus documents or results containing private or copyrighted content; only the synthetic manifest example belongs in the repository. +Each run writes `evaluation.json` and `summary.md`. In schema-v2 artifacts, `raw_decision` is the model's direct proposal before deterministic routing policy, while `decision` is the resolved and validated shipping-path result that the evaluator scores. The artifact also records model, OS, prompt and routing-policy versions, validation errors, per-case latency, accuracy, generation failures, unsafe/invalid decisions, and abstentions. Pass a schema-compatible previous `evaluation.json` with `--baseline` to expose regressions; incompatible schemas, corpora, routing policies, or model environments are reported instead of being compared silently. Exit status `2` means a threshold or baseline check failed; `1` means the evaluation could not run. Do not commit corpus documents or results containing private or copyrighted content; only the synthetic manifest example belongs in the repository. -For reproducible multi-prompt, system/content-tagging, and system/PCC comparisons, use the locked standalone Python project in [`evaluation/`](evaluation/README.md). It consumes the same corpus manifest and produces JSON, CSV, Markdown, and chart artifacts without adding Python to the application runtime. +The completed Issue #23 result is published in [`evaluation/ROUTING_RESULTS.md`](evaluation/ROUTING_RESULTS.md): routing policy v1 scored 108/108 exact decisions across nine final runs, with 18/18 ambiguous cases held for review, zero invalid final decisions, and an 8.1% mean-latency increase against the corrected shipping-path baseline. The 12-case private corpus is a regression gate, not a universal accuracy claim. + +For reproducible prompt, system/content-tagging, and system/PCC research, use the locked standalone Python project in [`evaluation/`](evaluation/README.md). It consumes the same corpus manifest and produces JSON, CSV, Markdown, and chart artifacts without adding Python to the application runtime. The Swift live evaluator remains the product-quality authority because it exercises the shipping extractor, routing policy, and validator. Bounded tool-calling candidates are also evaluated there under a documented threat model and evidence gate. Filesystem mutation remains exclusively in validated Swift code; no evaluation tool is available to the shipping app unless a future change demonstrates and reviews measurable value first. diff --git a/docs/hacker-news-draft.md b/docs/hacker-news-draft.md index 24afab9..ebb4911 100644 --- a/docs/hacker-news-draft.md +++ b/docs/hacker-news-draft.md @@ -2,7 +2,7 @@ ## Suggested title -Show HN: Sorting Hat – I turned Apple’s WWDC26 file-sorting demo into a local-first Mac app +Show HN: Sorting Hat – I built a better product than Apple’s WWDC26 file-sorting demo ## Post @@ -12,9 +12,11 @@ I wanted the product version of that idea, so I built Sorting Hat: a native macO The model only proposes actions. Deterministic Swift code validates paths, extensions, batch identities, and collisions before touching a file. Apple’s on-device model is the default; PCC is explicit opt-in, and Ollama can keep the fallback local. -I also copied the best part of the second half of Apple’s talk: measure it. The repo includes a locked Python prompt-evaluation harness and a bounded tool-calling experiment. The honest first six-case run is not a victory lap: it produced 0/6 exact decisions because folder routing missed the strict expected destinations. All four tool candidates were rejected by the predeclared quality/latency gate. +I also copied the best part of the second half of Apple’s talk: measure it. The repo includes a shipping-path Swift evaluator, a locked Python research harness, and predeclared quality and latency gates. The first prompt-only benchmark failed honestly, and three attempted prompt rewrites made accuracy, safety, or latency worse. -So I’m not claiming better model accuracy. I do think it is already a more complete, safer, and more usable product than the demo—and the failed benchmark is now the next piece of work rather than something hidden from the launch story. +The change that passed was deliberately less magical: keep the fast prompt, compile the destinations people configured, and let deterministic Swift resolve case, date templates, source extensions, and uncertain catch-all decisions before any file action. On a private 12-case corpus, the corrected shipping-path baseline scored 24/36 exact decisions across three runs. The final implementation scored 108/108 across nine runs, held all 18 ambiguous repetitions for review, produced zero invalid final decisions, and stayed inside the latency gate even with a 65-second model outlier retained. + +That is still a small private regression corpus, not proof of universal model superiority. The “better” claim is about the product: a persistent, local-first, recoverable Mac experience with a measured safety boundary, compared with an intentionally compact teaching demo. The repo publishes the policy, aggregate evidence, negative results, and limitations; it does not publish the private files or raw outputs. Repository: https://github.com/tcballard/SortingHat diff --git a/docs/wwdc26-comparison.md b/docs/wwdc26-comparison.md index 6575b6e..e5a1182 100644 --- a/docs/wwdc26-comparison.md +++ b/docs/wwdc26-comparison.md @@ -20,12 +20,12 @@ That demo is intentionally compact and excellent at teaching the platform primit | Interaction | Terminal/script | Native menu-bar app with Inbox, activity, rules, model settings, pause, retry, resolve, and remove actions | | Model policy | On-device or PCC examples | On-device first; PCC is explicit opt-in; local Ollama and configured OpenAI fallbacks | | Throughput | One model call for the filename list | Up to 8 compatible files or 24,000 extracted characters per validated batch | -| Evaluation | Notebook-style prompt comparison | Locked CLI harness, private corpus contract, prompt/use-case matrix, raw artifacts, deterministic tests, and predeclared tool gates | +| Evaluation | Notebook-style prompt comparison | Shipping-path Swift gate, private corpus contract, prompt/use-case research matrix, raw/resolved artifacts, deterministic tests, and predeclared gates | ## Architecture ```text -Inbox -> settle -> extract/OCR -> batch or image analysis -> validate +Inbox -> settle -> extract/OCR -> batch or image analysis -> resolve/validate -> rename/tag/move -> Activity \-> Needs review -> retry/resolve/remove ``` @@ -36,54 +36,60 @@ Images use the multimodal `fm` path. Searchable documents prefer embedded text; ## Reproducible evaluation -The corpus is deliberately private and lives outside the repository. The checked-in evaluator, prompt definitions, matrix, lockfile, tests, and synthetic manifest contract are public. This avoids publishing personal documents or copyrighted test files while still making the method repeatable with an equivalent corpus. +The corpus is deliberately private and lives outside the repository. The checked-in evaluator, policy, tests, and synthetic manifest contract are public. This avoids publishing personal documents or copyrighted test files while keeping the method repeatable with an equivalent corpus. -### Environment recorded for the 18 July 2026 run +The Swift live evaluator is the product-quality authority. It executes the same PDF/text/Vision extraction, routing policy, extension handling, validator, and manual-review decision the app uses, but never calls `Organizer.apply`. -- Sorting Hat commit: `e53c0d352428cc0e8f11a4c626996a5dd492de34` +### Environment recorded for the 18 July 2026 result + +- Predeclared quality-policy commit: `dce8756e67d864a0b22229de2f95a93a96874417` +- Sorting Hat routing commit measured: `2831277e270ec74c4ab6d996364e0b7bbfd10128` +- Evaluator artifact-hardening commit: `fa5f2683f75762933165b88f5d801c6500d39085` - macOS 27.0 beta, build `26A5378j` - MacBook Pro (`MacBookPro17,1`), Apple M1, 16 GB memory -- Apple system model reported available -- Python 3.12.13; dependencies pinned by `evaluation/uv.lock` -- Corpus: `sorting-hat-representative-v1`, six anonymized cases -- One case each: receipt, scan, screenshot, searchable PDF, office document, and ambiguous input -- Four filing rules; exact folder, filename-term, tag, abstention, and generation-failure scoring - -### Commands +- Apple system model, `general` use case, default guardrails, PCC disabled +- Prompt `sorting-decision-v2`; deterministic policy `routing-rules-v1` +- Corpus `sorting-hat-representative-v2`, 12 private anonymized cases +- Corpus manifest SHA-256 `ff11e1e1d670e46765f734d27dc5386f8a4d0f92d5e17078b38751275f402fcd` +- Coverage includes receipts, scan/OCR, screenshots, searchable PDF, office document, ambiguous notes, generic/no-date text, and adversarial prompt-like content -Run the fully on-device rows: +### Command ```sh -cd evaluation -uv sync --frozen --no-editable -uv run sortinghat-evaluate \ - --corpus ~/SortingHat-Evaluation/corpus/corpus.json \ - --matrix system-matrix.json \ - --prompts prompts.json \ - --output ~/SortingHat-Evaluation/results/run-001 +.build/debug/sorting-hat evaluate --live \ + --corpus ~/SortingHat-Evaluation/corpus-v2/corpus.json \ + --output ~/SortingHat-Evaluation/results/run-001 \ + --config sortinghat.conf ``` -Running the full `matrix.json` instead requires `--allow-pcc`; that flag is an explicit acknowledgement that corpus context may leave the Mac for Apple Private Cloud Compute. +### Shipping-path result -### On-device results +Issue #23 committed its quality and latency thresholds before the final measurements. The corrected pre-policy baseline ran three times; the final implementation ran nine times after a latency outlier made a larger sample useful. Every valid final run is included. -The latest completed local run produced 17 structured responses from 18 attempts. Strict end-to-end accuracy was **0/6 for every variant** because none chose an exact accepted destination; this is a real negative result and means Sorting Hat has not yet earned a measured “better than Apple” quality claim. +| Metric | Corrected baseline | Routing policy v1 | +| --- | ---: | ---: | +| Exact decision | 24/36 (66.7%) | 108/108 (100%) | +| Exact folder | 24/36 (66.7%) | 108/108 (100%) | +| Required filename terms | 36/36 (100%) | 108/108 (100%) | +| Required tags | 33/36 (91.7%) | 108/108 (100%) | +| Ambiguous abstention | 0/6 | 18/18 | +| Generation failures | 0/36 | 0/108 | +| Unsafe or invalid final decisions | 3/36 | 0/108 | +| Mean latency | 3,641 ms | 3,935 ms (+8.1%) | -| Variant | Exact decisions | Folder | Filename | Tags | Generation failures | Mean latency | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Concise / system / general | 0/6 | 0/6 | 3/6 | 6/6 | 0/6 | 4,309 ms | -| Detailed / system / general | 0/6 | 0/6 | 3/6 | 6/6 | 0/6 | 1,925 ms | -| Detailed / system / content-tagging | 0/6 | 0/6 | 2/6 | 5/6 | 1/6 | 2,188 ms | +This clears every predeclared gate: at least 80% aggregate and 75% per-run exact accuracy, 90% folders, 85% filenames, 90% tags, 100% ambiguous abstention, at most 5% generation failures, zero unsafe/invalid final decisions, and no more than 25% mean-latency regression. One final run included a 65.6-second model response; it remains in the aggregate. -The detailed prompt reduced excess information from 1.50 to 0.83 items per case and was faster in this run. Content-tagging missed more required information and had one generation failure. Six cases and one run are too small for statistical claims. +The complete aggregate record, rejected prompt candidates, excluded infrastructure run, corpus boundary, and limitations are in [`evaluation/ROUTING_RESULTS.md`](../evaluation/ROUTING_RESULTS.md). -The PCC row was run after explicit approval to send the anonymized corpus context to Apple. All six requests failed before inference with `PCC inference is not available in this context`; the evaluator classifies these as infrastructure failures. The roughly 13 ms rejection time is not reported as model latency or quality evidence. PCC therefore remains **inconclusive**, rather than a 0% quality result. +### Negative and supporting evidence -### Other measured evidence - -- Batching: the deterministic process test verifies 8 compatible files use 1 `fm respond` process instead of 8. This is an invocation-count result, not a wall-clock throughput claim. -- OCR: deterministic fixtures cover successful Vision extraction, confidence filtering, scanned-PDF fallback, and safe failure. The six-case model matrix is too small to claim an OCR accuracy rate. -- Tool calling: four bounded, read-only candidates were tested. None cleared the predeclared accuracy/failure/500 ms latency gate; see [`evaluation/TOOL_RESULTS.md`](../evaluation/TOOL_RESULTS.md). +- Prompt-only versions v3, v4, and v5 regressed accuracy, safety, and latency and were rejected. The passing change keeps the faster v2 prompt and resolves controlled destinations in Swift. +- PR #22's standalone Python matrix scored 0/6, but Issue #23 found that binary documents had silently fallen back to filename-only input. That result remains useful prompt research, not shipping-path product evidence. +- One candidate run encountered three local Vision failures before inference. It is retained as infrastructure evidence and excluded from quality scoring under the committed policy. +- The explicitly approved PCC run still failed before inference because PCC was unavailable in this context. PCC remains inconclusive and outside the on-device gate. +- Batching: a deterministic process test verifies 8 compatible files use 1 `fm respond` process instead of 8. This is an invocation-count result, not a wall-clock throughput claim. +- OCR: deterministic fixtures cover successful Vision extraction, confidence filtering, scanned-PDF fallback, and safe failure. The private corpus is too small to claim a general OCR accuracy rate. +- Tool calling: four bounded, read-only candidates were rejected by their separate predeclared quality/failure/latency gate; see [`evaluation/TOOL_RESULTS.md`](../evaluation/TOOL_RESULTS.md). ## Privacy and limitations @@ -96,4 +102,6 @@ The PCC row was run after explicit approval to send the anonymized corpus contex ## Honest conclusion -Sorting Hat already goes substantially beyond the WWDC26 demo in product surface, safety, recovery, OCR, batching, and local-first operation. The current benchmark does **not** show superior classification quality. The defensible headline today is “I turned Apple’s WWDC file-sorting demo into a real Mac app”; “better” should wait for a larger corpus and a passing quality result. +Sorting Hat goes substantially beyond the WWDC26 demo in product surface, safety, recovery, OCR, batching, local-first operation, and now a passing predeclared shipping-path quality gate. On the bounded 12-case corpus, routing policy v1 improved exact decisions from 66.7% to 100% without crossing the safety or latency limits. + +That makes “I built a better product than a WWDC26 demo” a defensible product headline: it compares a persistent, recoverable Mac app with an intentionally compact teaching demo. It does **not** mean Sorting Hat's model is universally more accurate than Apple's, nor that 12 private cases prove production reliability for every Inbox. diff --git a/evaluation/README.md b/evaluation/README.md index 74b8af2..843cdeb 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -1,6 +1,8 @@ # Sorting Hat Python evaluation -This is an evaluation-only Python project. Nothing under this directory is linked into or required by the shipping Swift package. +This is an evaluation-only Python project. Nothing under this directory is linked into or required by the shipping Swift package. It is useful for prompt, use-case, PCC, and bounded-tool research; the root `sorting-hat evaluate --live` command is the product-quality authority because it exercises the shipping Swift extraction, routing, and validation path. In its schema-v2 artifact, `raw_decision` means the model output before deterministic policy and `decision` means the resolved, validated shipping result used for scoring. + +The completed shipping-path routing gate and the rejected prompt-only candidates are recorded in [`ROUTING_RESULTS.md`](ROUTING_RESULTS.md). ## Requirements diff --git a/evaluation/ROUTING_RESULTS.md b/evaluation/ROUTING_RESULTS.md new file mode 100644 index 0000000..5f54b9f --- /dev/null +++ b/evaluation/ROUTING_RESULTS.md @@ -0,0 +1,71 @@ +# Routing quality result + +Issue #23 is a **PASS** against the predeclared [`quality-policy.json`](quality-policy.json). Routing policy v1 improved the corrected shipping-path baseline from 66.7% to 100% exact decisions on the private 12-case corpus while preserving every safety and latency gate. + +This is evidence for this bounded corpus and environment. It is not a claim of universal model accuracy, and the private documents and case-level outputs are not published. + +## Reproducibility record + +- Date: 18 July 2026 +- Predeclared quality-policy commit: `dce8756e67d864a0b22229de2f95a93a96874417` +- Routing implementation commit measured: `2831277e270ec74c4ab6d996364e0b7bbfd10128` +- Evaluator artifact-hardening commit: `fa5f2683f75762933165b88f5d801c6500d39085` +- macOS: 27.0 beta, build `26A5378j` +- Hardware: MacBook Pro (`MacBookPro17,1`), Apple M1, 16 GB memory +- Model: Apple system model, `general` use case, default guardrails, PCC disabled +- Prompt: `sorting-decision-v2` +- Deterministic policy: `routing-rules-v1` +- Corpus: `sorting-hat-representative-v2`, 12 private anonymized cases +- Corpus manifest SHA-256: `ff11e1e1d670e46765f734d27dc5386f8a4d0f92d5e17078b38751275f402fcd` +- Coverage: receipts, a scanned PDF, screenshots, a searchable PDF, an office document, ambiguous notes, generic text, a no-date case, and adversarial prompt-like content + +The checked-in synthetic manifest shows the schema without exposing the private inputs. All raw decisions and generated artifacts remain under the external `SortingHat-Evaluation` directory. + +## Aggregate result + +The baseline uses three unchanged PR #22 production-path runs. The final candidate uses nine runs against commit `2831277`; the sample was fixed at nine after the first final run exposed a latency outlier, and no valid run was discarded. + +| Metric | Corrected baseline | Routing policy v1 | Gate | Result | +| --- | ---: | ---: | ---: | --- | +| Exact decision | 24/36 (66.7%) | 108/108 (100%) | >=80% aggregate; >=75% each run | PASS | +| Exact folder | 24/36 (66.7%) | 108/108 (100%) | >=90% | PASS | +| Required filename terms | 36/36 (100%) | 108/108 (100%) | >=85% | PASS | +| Required tags | 33/36 (91.7%) | 108/108 (100%) | >=90% | PASS | +| Ambiguous abstention | 0/6 (0%) | 18/18 (100%) | 100% | PASS | +| Generation failures | 0/36 | 0/108 | <=5% | PASS | +| Unsafe or invalid final decisions | 3/36 | 0/108 | 0 | PASS | +| Mean latency | 3,641 ms | 3,935 ms (+8.1%) | <=25% regression | PASS | +| Per-case p50 latency | 2,955 ms | 3,058 ms | reported | +3.5% | +| Per-case p95 latency | 7,990 ms | 6,329 ms | reported | -20.8% | + +Final per-run exact accuracy was 12/12 in all nine runs. Mean per-run latency was 8,772, 3,416, 3,448, 3,910, 3,727, 2,836, 3,278, 2,853, and 3,175 ms. The 8,772 ms run included one 65.6-second model response and remains in the aggregate. + +## What changed + +The faster PR #22 prompt remains unchanged. The improvement comes from a versioned Swift policy on the shipping path: + +- Compile only the controlled `Put ... in ...` routes produced by Sorting Hat's rule builder. +- Canonicalise model folders against configured destination templates, including case and `YYYY`/`YYYY-MM` components. +- Use an unambiguous single-keyword source filename or extension as a strong route hint; semantic classification otherwise remains the model's job. +- Preserve the source container by repairing a missing or mistaken extension before planning. +- Merge simple static tags from a strongly matched configured route. +- Keep catch-all decisions in review when the model explicitly reports uncertainty and offers only generic tags. +- Reject traversal, unresolved placeholders, and safe-looking but unconfigured destinations before any filesystem action. + +The live evaluator now records both views explicitly: `raw_decision` is the model's direct output before deterministic policy, and `decision` is the resolved, validated shipping result used for scoring. It preserves validation errors and refuses automatic regression comparisons across artifact schemas, routing policies, or model environments. + +## Negative and excluded findings + +- Prompt-only candidates were rejected. v3 scored 1/12 exact with four invalid decisions and 9,842 ms mean latency; v4 scored 4/12 with five invalid decisions and 22,029 ms; v5 scored 6/12 with two invalid decisions, no abstentions, and 10,807 ms. +- One routing-policy run encountered three local Vision OCR failures before inference. It is retained as an infrastructure record but excluded from quality scoring under the predeclared policy. +- PR #22's Python matrix scored 0/6 exact, but diagnosis showed that binary documents had silently fallen back to filename-only text. It remains useful prompt research, not the authority for shipping product quality. The corrected baseline above uses the Swift extractor, validator, and manual-review path users actually run. +- Before candidate measurements, the scan case's required filename term changed from `form` to `registration`: the document is a volunteer registration form, and `registration` is the more content-grounded descriptive term. This ground-truth correction is not counted as product improvement. +- Apple PCC remains outside this on-device gate. + +## Limitations + +- Twelve private cases are enough for a regression gate, not a population-level claim. More varied receipts, scans, screenshots, office documents, dates, languages, and overlapping rules remain useful future coverage. +- Controlled route compilation deliberately ignores arbitrary prose that does not begin with `Put ... in ...`; those rules retain the legacy model-and-safety path. +- Strong source matching is intentionally narrow. Multiword or overlapping subjects are left to the model rather than guessed deterministically. +- Static tag recovery is designed for the rule builder's simple tag syntax; route objects should remain structured longer in a future config revision. +- On-device Foundation Models latency and OCR availability vary with system state. The retained outlier demonstrates why repeated runs and a manual-review path still matter. From 8d32b26419220e5537fc9c883d48e0431a21947b Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 18 Jul 2026 16:08:51 +0100 Subject: [PATCH 5/6] fix(evaluation): require comparable prompt versions [issue:#23] --- Sources/SortingHatCore/LiveEvaluation.swift | 5 +- Tests/SortingHatTests/SortingHatTests.swift | 62 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/Sources/SortingHatCore/LiveEvaluation.swift b/Sources/SortingHatCore/LiveEvaluation.swift index 66f3351..5e315d7 100644 --- a/Sources/SortingHatCore/LiveEvaluation.swift +++ b/Sources/SortingHatCore/LiveEvaluation.swift @@ -258,7 +258,7 @@ public enum LiveEvaluator { | Schema failures | \(artifact.metrics.schemaFailures) | | Unsafe/invalid decisions | \(artifact.metrics.unsafeOrInvalidDecisions) | | Abstentions | \(artifact.metrics.abstentions) | - | Average latency | \(String(format: "%.1f ms", artifact.metrics.averageLatencyMilliseconds)) | + | Average pre-validation latency | \(String(format: "%.1f ms", artifact.metrics.averageLatencyMilliseconds)) | ## Thresholds and regressions @@ -327,8 +327,9 @@ public enum LiveEvaluator { baselineConfiguration.guardrails == configuration.guardrails, baselineConfiguration.pccAllowed == configuration.pccAllowed, baselineConfiguration.operatingSystem == configuration.operatingSystem, + baselineConfiguration.promptVersion == configuration.promptVersion, baselineConfiguration.routingPolicyVersion == configuration.routingPolicyVersion else { - return ["baseline model environment does not match this evaluation"] + return ["baseline evaluation configuration does not match this evaluation"] } var values: [String] = [] diff --git a/Tests/SortingHatTests/SortingHatTests.swift b/Tests/SortingHatTests/SortingHatTests.swift index 53e0463..f9bbf43 100644 --- a/Tests/SortingHatTests/SortingHatTests.swift +++ b/Tests/SortingHatTests/SortingHatTests.swift @@ -290,6 +290,68 @@ struct SortingHatTests { #expect(artifact.results[0].decision?.folder == "Receipts/2026") } + @Test func refusesAutomaticRegressionComparisonAcrossPromptVersions() { + let baselineConfiguration = EvaluationConfiguration( + model: "system", + useCase: "general", + guardrails: "default", + pccAllowed: false, + promptVersion: "baseline-prompt", + operatingSystem: "testOS", + routingPolicyVersion: RoutingDecisionResolver.version + ) + let candidateConfiguration = EvaluationConfiguration( + model: "system", + useCase: "general", + guardrails: "default", + pccAllowed: false, + promptVersion: "candidate-prompt", + operatingSystem: "testOS" + ) + let metrics = EvaluationMetrics( + total: 0, + correct: 0, + folderCorrect: 0, + filenameCorrect: 0, + tagsCorrect: 0, + generationFailures: 0, + schemaFailures: 0, + unsafeOrInvalidDecisions: 0, + abstentions: 0, + accuracy: 0, + generationFailureRate: 0, + unsafeDecisionRate: 0, + averageLatencyMilliseconds: 0 + ) + let baseline = EvaluationArtifact( + schemaVersion: 2, + corpusName: "synthetic", + createdAt: Date(), + configuration: baselineConfiguration, + metrics: metrics, + results: [], + thresholdFailures: [], + regressions: [] + ) + let manifest = EvaluationManifest( + version: 1, + name: "synthetic", + rules: [], + cases: [], + thresholds: nil + ) + + let artifact = LiveEvaluator.run( + manifest: manifest, + corpusRoot: FileManager.default.temporaryDirectory, + analyzer: EvaluationAnalyzer(), + configuration: candidateConfiguration, + baseline: baseline + ) + + #expect(artifact.regressions == ["baseline evaluation configuration does not match this evaluation"]) + } + @Test func liveEvaluationScoresResolvedShippingDecisionAndRetainsRawDiagnostics() throws { let root = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) From dc9d8dd762f018e2213cc486405c0a9b1f58c586 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 18 Jul 2026 16:10:20 +0100 Subject: [PATCH 6/6] docs(evaluation): clarify benchmark boundaries [issue:#23] --- README.md | 6 +++--- docs/hacker-news-draft.md | 2 +- docs/wwdc26-comparison.md | 7 ++++--- evaluation/README.md | 2 +- evaluation/ROUTING_RESULTS.md | 17 ++++++++++------- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index df8aa21..141a4fb 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Sorting Hat reads searchable PDFs, plain-text formats, RTF, Word, and OpenDocume When Apple `fm` is selected, compatible non-image files are analyzed in batches of at most 8 files and 24,000 extracted characters. Each file is identified by an opaque request-local ID, and every returned decision is independently validated before any move. Missing, duplicate, unexpected, or unsafe decisions cannot be applied. Images remain on the individual multimodal path, and a failed batch does not prevent files in other batches from being processed. A deterministic process benchmark verifies that 8 compatible files use 1 `fm respond` invocation instead of 8. -The Inbox is intake-only. Sorting Hat renames each file and moves it to a rule-specific folder under `output` (for example, `~/SortingHat/Receipts/2026`). For the rule builder's controlled `Put ... in ...` routes, deterministic Swift code canonicalises the model's proposal against configured destinations and rejects unknown or unresolved folders. It creates destination folders as needed, rejects absolute paths and traversal, preserves the source file type, protects existing files with numbered names, and writes tags as Finder metadata. +The Inbox is intake-only. Sorting Hat renames each file and moves it to a rule-specific folder under `output` (for example, `~/SortingHat/Receipts/2026`). For the rule builder's controlled `Put ... in ...` routes, deterministic Swift code canonicalises the model's proposal against configured destinations and rejects unknown or unresolved folders. When at least one controlled route is present, its compiled destinations form the authoritative allow-list for that ruleset; other prose can still guide naming, tagging, and classification but cannot introduce a new destination. Sorting Hat creates destination folders as needed, rejects absolute paths and traversal, preserves the source file type, protects existing files with numbered names, and writes tags as Finder metadata. If a model returns the uploaded filename unchanged, Sorting Hat leaves the source in the Inbox and reports an invalid sorting decision instead of silently skipping the requested rename. @@ -97,9 +97,9 @@ Live quality checks are deliberately opt-in and read-only. Create a private, ano The manifest is versioned and contains rules plus cases with a relative source path, content kind (`receipt`, `scan`, `screenshot`, `pdf`, `office_document`, or `ambiguous`), accepted folders, required filename terms, required tags, and whether an empty-folder abstention is expected. The checked-in template defines all six required classes but deliberately includes no documents. Corpus files must sit beside or below the manifest; output inside that corpus directory is rejected. The evaluator calls Apple `fm` but never calls `Organizer.apply`, so sources are not renamed, tagged, or moved. -Each run writes `evaluation.json` and `summary.md`. In schema-v2 artifacts, `raw_decision` is the model's direct proposal before deterministic routing policy, while `decision` is the resolved and validated shipping-path result that the evaluator scores. The artifact also records model, OS, prompt and routing-policy versions, validation errors, per-case latency, accuracy, generation failures, unsafe/invalid decisions, and abstentions. Pass a schema-compatible previous `evaluation.json` with `--baseline` to expose regressions; incompatible schemas, corpora, routing policies, or model environments are reported instead of being compared silently. Exit status `2` means a threshold or baseline check failed; `1` means the evaluation could not run. Do not commit corpus documents or results containing private or copyrighted content; only the synthetic manifest example belongs in the repository. +Each run writes `evaluation.json` and `summary.md`. In schema-v2 artifacts, `raw_decision` is the model's direct proposal before deterministic routing policy, while `decision` is the resolved and validated shipping-path result that the evaluator scores. The artifact also records model, OS, prompt and routing-policy versions, validation errors, pre-validation decision latency, accuracy, generation failures, unsafe/invalid decisions, and abstentions. Latency stops after analysis and deterministic routing resolution, before the separate `Organizer` validation pass. Pass a schema-compatible previous `evaluation.json` with `--baseline` to expose regressions; incompatible schemas, corpora, prompts, routing policies, or model environments are reported instead of being compared silently. Exit status `2` means a threshold or baseline check failed; `1` means the evaluation could not run. Do not commit corpus documents or results containing private or copyrighted content; only the synthetic manifest example belongs in the repository. -The completed Issue #23 result is published in [`evaluation/ROUTING_RESULTS.md`](evaluation/ROUTING_RESULTS.md): routing policy v1 scored 108/108 exact decisions across nine final runs, with 18/18 ambiguous cases held for review, zero invalid final decisions, and an 8.1% mean-latency increase against the corrected shipping-path baseline. The 12-case private corpus is a regression gate, not a universal accuracy claim. +The completed Issue #23 result is published in [`evaluation/ROUTING_RESULTS.md`](evaluation/ROUTING_RESULTS.md): routing policy v1 scored 108/108 exact decisions across nine final runs, with 18/18 ambiguous cases held for review, zero invalid final decisions, and an 8.1% recorded pre-validation latency increase against the corrected shipping-path baseline. The 12-case private corpus is a regression gate, not a universal accuracy claim. For reproducible prompt, system/content-tagging, and system/PCC research, use the locked standalone Python project in [`evaluation/`](evaluation/README.md). It consumes the same corpus manifest and produces JSON, CSV, Markdown, and chart artifacts without adding Python to the application runtime. The Swift live evaluator remains the product-quality authority because it exercises the shipping extractor, routing policy, and validator. diff --git a/docs/hacker-news-draft.md b/docs/hacker-news-draft.md index ebb4911..bca7814 100644 --- a/docs/hacker-news-draft.md +++ b/docs/hacker-news-draft.md @@ -14,7 +14,7 @@ The model only proposes actions. Deterministic Swift code validates paths, exten I also copied the best part of the second half of Apple’s talk: measure it. The repo includes a shipping-path Swift evaluator, a locked Python research harness, and predeclared quality and latency gates. The first prompt-only benchmark failed honestly, and three attempted prompt rewrites made accuracy, safety, or latency worse. -The change that passed was deliberately less magical: keep the fast prompt, compile the destinations people configured, and let deterministic Swift resolve case, date templates, source extensions, and uncertain catch-all decisions before any file action. On a private 12-case corpus, the corrected shipping-path baseline scored 24/36 exact decisions across three runs. The final implementation scored 108/108 across nine runs, held all 18 ambiguous repetitions for review, produced zero invalid final decisions, and stayed inside the latency gate even with a 65-second model outlier retained. +The change that passed was deliberately less magical: keep the fast prompt, compile the destinations people configured, and let deterministic Swift resolve case, date templates, source extensions, and uncertain catch-all decisions before any file action. On a private 12-case corpus, the corrected shipping-path baseline scored 24/36 exact decisions across three runs. The final implementation scored 108/108 across nine runs, held all 18 ambiguous repetitions for review, produced zero invalid final decisions, and stayed inside the recorded pre-validation latency gate even with a 65-second model outlier retained. That is still a small private regression corpus, not proof of universal model superiority. The “better” claim is about the product: a persistent, local-first, recoverable Mac experience with a measured safety boundary, compared with an intentionally compact teaching demo. The repo publishes the policy, aggregate evidence, negative results, and limitations; it does not publish the private files or raw outputs. diff --git a/docs/wwdc26-comparison.md b/docs/wwdc26-comparison.md index e5a1182..ace4644 100644 --- a/docs/wwdc26-comparison.md +++ b/docs/wwdc26-comparison.md @@ -45,6 +45,7 @@ The Swift live evaluator is the product-quality authority. It executes the same - Predeclared quality-policy commit: `dce8756e67d864a0b22229de2f95a93a96874417` - Sorting Hat routing commit measured: `2831277e270ec74c4ab6d996364e0b7bbfd10128` - Evaluator artifact-hardening commit: `fa5f2683f75762933165b88f5d801c6500d39085` +- Prompt-comparability hardening commit: `8d32b26419220e5537fc9c883d48e0431a21947b` - macOS 27.0 beta, build `26A5378j` - MacBook Pro (`MacBookPro17,1`), Apple M1, 16 GB memory - Apple system model, `general` use case, default guardrails, PCC disabled @@ -75,9 +76,9 @@ Issue #23 committed its quality and latency thresholds before the final measurem | Ambiguous abstention | 0/6 | 18/18 | | Generation failures | 0/36 | 0/108 | | Unsafe or invalid final decisions | 3/36 | 0/108 | -| Mean latency | 3,641 ms | 3,935 ms (+8.1%) | +| Mean pre-validation decision latency | 3,641 ms | 3,935 ms (+8.1%) | -This clears every predeclared gate: at least 80% aggregate and 75% per-run exact accuracy, 90% folders, 85% filenames, 90% tags, 100% ambiguous abstention, at most 5% generation failures, zero unsafe/invalid final decisions, and no more than 25% mean-latency regression. One final run included a 65.6-second model response; it remains in the aggregate. +This clears every predeclared gate: at least 80% aggregate and 75% per-run exact accuracy, 90% folders, 85% filenames, 90% tags, 100% ambiguous abstention, at most 5% generation failures, zero unsafe/invalid final decisions, and no more than 25% recorded pre-validation latency regression. The clock stops after model analysis and deterministic routing resolution, before `Organizer` validation; the baseline had no resolver, so the candidate comparison conservatively includes the added resolver work. One final run included a 65.6-second model response; it remains in the aggregate. The complete aggregate record, rejected prompt candidates, excluded infrastructure run, corpus boundary, and limitations are in [`evaluation/ROUTING_RESULTS.md`](../evaluation/ROUTING_RESULTS.md). @@ -102,6 +103,6 @@ The complete aggregate record, rejected prompt candidates, excluded infrastructu ## Honest conclusion -Sorting Hat goes substantially beyond the WWDC26 demo in product surface, safety, recovery, OCR, batching, local-first operation, and now a passing predeclared shipping-path quality gate. On the bounded 12-case corpus, routing policy v1 improved exact decisions from 66.7% to 100% without crossing the safety or latency limits. +Sorting Hat goes substantially beyond the WWDC26 demo in product surface, safety, recovery, OCR, batching, local-first operation, and now a passing predeclared shipping-path quality gate. On the bounded 12-case corpus, routing policy v1 improved exact decisions from 66.7% to 100% without crossing the safety or recorded pre-validation latency limits. That makes “I built a better product than a WWDC26 demo” a defensible product headline: it compares a persistent, recoverable Mac app with an intentionally compact teaching demo. It does **not** mean Sorting Hat's model is universally more accurate than Apple's, nor that 12 private cases prove production reliability for every Inbox. diff --git a/evaluation/README.md b/evaluation/README.md index 843cdeb..cf492ab 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -1,6 +1,6 @@ # Sorting Hat Python evaluation -This is an evaluation-only Python project. Nothing under this directory is linked into or required by the shipping Swift package. It is useful for prompt, use-case, PCC, and bounded-tool research; the root `sorting-hat evaluate --live` command is the product-quality authority because it exercises the shipping Swift extraction, routing, and validation path. In its schema-v2 artifact, `raw_decision` means the model output before deterministic policy and `decision` means the resolved, validated shipping result used for scoring. +This is an evaluation-only Python project. Nothing under this directory is linked into or required by the shipping Swift package. It is useful for prompt, use-case, PCC, and bounded-tool research; the root `sorting-hat evaluate --live` command is the product-quality authority because it exercises the shipping Swift extraction, routing, and validation path. In its schema-v2 artifact, `raw_decision` means the model output before deterministic policy and `decision` means the resolved, validated shipping result used for scoring. Its latency field covers analysis plus deterministic resolution and stops before the separate `Organizer` validation pass. The completed shipping-path routing gate and the rejected prompt-only candidates are recorded in [`ROUTING_RESULTS.md`](ROUTING_RESULTS.md). diff --git a/evaluation/ROUTING_RESULTS.md b/evaluation/ROUTING_RESULTS.md index 5f54b9f..c09d690 100644 --- a/evaluation/ROUTING_RESULTS.md +++ b/evaluation/ROUTING_RESULTS.md @@ -1,6 +1,6 @@ # Routing quality result -Issue #23 is a **PASS** against the predeclared [`quality-policy.json`](quality-policy.json). Routing policy v1 improved the corrected shipping-path baseline from 66.7% to 100% exact decisions on the private 12-case corpus while preserving every safety and latency gate. +Issue #23 is a **PASS** against the predeclared [`quality-policy.json`](quality-policy.json). Routing policy v1 improved the corrected shipping-path baseline from 66.7% to 100% exact decisions on the private 12-case corpus while preserving every safety and recorded pre-validation latency gate. This is evidence for this bounded corpus and environment. It is not a claim of universal model accuracy, and the private documents and case-level outputs are not published. @@ -10,6 +10,7 @@ This is evidence for this bounded corpus and environment. It is not a claim of u - Predeclared quality-policy commit: `dce8756e67d864a0b22229de2f95a93a96874417` - Routing implementation commit measured: `2831277e270ec74c4ab6d996364e0b7bbfd10128` - Evaluator artifact-hardening commit: `fa5f2683f75762933165b88f5d801c6500d39085` +- Prompt-comparability hardening commit: `8d32b26419220e5537fc9c883d48e0431a21947b` - macOS: 27.0 beta, build `26A5378j` - Hardware: MacBook Pro (`MacBookPro17,1`), Apple M1, 16 GB memory - Model: Apple system model, `general` use case, default guardrails, PCC disabled @@ -34,11 +35,13 @@ The baseline uses three unchanged PR #22 production-path runs. The final candida | Ambiguous abstention | 0/6 (0%) | 18/18 (100%) | 100% | PASS | | Generation failures | 0/36 | 0/108 | <=5% | PASS | | Unsafe or invalid final decisions | 3/36 | 0/108 | 0 | PASS | -| Mean latency | 3,641 ms | 3,935 ms (+8.1%) | <=25% regression | PASS | -| Per-case p50 latency | 2,955 ms | 3,058 ms | reported | +3.5% | -| Per-case p95 latency | 7,990 ms | 6,329 ms | reported | -20.8% | +| Mean pre-validation decision latency | 3,641 ms | 3,935 ms (+8.1%) | <=25% regression | PASS | +| Per-case p50 pre-validation latency | 2,955 ms | 3,058 ms | reported | +3.5% | +| Per-case p95 pre-validation latency | 7,990 ms | 6,329 ms | reported | -20.8% | -Final per-run exact accuracy was 12/12 in all nine runs. Mean per-run latency was 8,772, 3,416, 3,448, 3,910, 3,727, 2,836, 3,278, 2,853, and 3,175 ms. The 8,772 ms run included one 65.6-second model response and remains in the aggregate. +Final per-run exact accuracy was 12/12 in all nine runs. Mean per-run pre-validation decision latency was 8,772, 3,416, 3,448, 3,910, 3,727, 2,836, 3,278, 2,853, and 3,175 ms. The 8,772 ms run included one 65.6-second model response and remains in the aggregate. + +The artifact clock stops after model analysis and deterministic route resolution, before the separate `Organizer` validation pass. The baseline stopped after analysis because it had no resolver, so the candidate's recorded comparison conservatively includes the new resolver work; neither side measures filesystem mutation. ## What changed @@ -56,7 +59,7 @@ The live evaluator now records both views explicitly: `raw_decision` is the mode ## Negative and excluded findings -- Prompt-only candidates were rejected. v3 scored 1/12 exact with four invalid decisions and 9,842 ms mean latency; v4 scored 4/12 with five invalid decisions and 22,029 ms; v5 scored 6/12 with two invalid decisions, no abstentions, and 10,807 ms. +- Prompt-only candidates were rejected. v3 scored 1/12 exact with four invalid decisions and 9,842 ms mean pre-validation latency; v4 scored 4/12 with five invalid decisions and 22,029 ms; v5 scored 6/12 with two invalid decisions, no abstentions, and 10,807 ms. - One routing-policy run encountered three local Vision OCR failures before inference. It is retained as an infrastructure record but excluded from quality scoring under the predeclared policy. - PR #22's Python matrix scored 0/6 exact, but diagnosis showed that binary documents had silently fallen back to filename-only text. It remains useful prompt research, not the authority for shipping product quality. The corrected baseline above uses the Swift extractor, validator, and manual-review path users actually run. - Before candidate measurements, the scan case's required filename term changed from `form` to `registration`: the document is a volunteer registration form, and `registration` is the more content-grounded descriptive term. This ground-truth correction is not counted as product improvement. @@ -65,7 +68,7 @@ The live evaluator now records both views explicitly: `raw_decision` is the mode ## Limitations - Twelve private cases are enough for a regression gate, not a population-level claim. More varied receipts, scans, screenshots, office documents, dates, languages, and overlapping rules remain useful future coverage. -- Controlled route compilation deliberately ignores arbitrary prose that does not begin with `Put ... in ...`; those rules retain the legacy model-and-safety path. +- Once any controlled `Put ... in ...` route compiles, its destinations become the authoritative allow-list for the whole ruleset. Other prose may guide naming, tagging, and classification, but it cannot add a model-selected destination; a ruleset that needs arbitrary destinations should not mix the two modes. - Strong source matching is intentionally narrow. Multiword or overlapping subjects are left to the model rather than guessed deterministically. - Static tag recovery is designed for the rule builder's simple tag syntax; route objects should remain structured longer in a future config revision. - On-device Foundation Models latency and OCR availability vary with system state. The retained outlier demonstrates why repeated runs and a manual-review path still matter.