diff --git a/crates/aisix-guardrails/src/local_model.rs b/crates/aisix-guardrails/src/local_model.rs index 88d26785..6407232e 100644 --- a/crates/aisix-guardrails/src/local_model.rs +++ b/crates/aisix-guardrails/src/local_model.rs @@ -4,8 +4,10 @@ //! Implements the design issue's three-layer pipeline for one hardcoded //! category (no prototype library resource, no standard risk categories //! yet). Pipeline per text segment: -//! 1. regex finds candidate spans with exact byte offsets -//! (dotted number runs, the EDA-version candidate shape); +//! 1. regex finds candidate spans with exact byte offsets — two +//! shapes: dotted number runs (ASCII or fullwidth) and fused +//! version tokens (letters+digits in one token, `IC618`); the +//! layer is deliberately broad, precision lives in ② and ③; //! 2. rule scoring ([`rules`]): hotword proximity co-occurrence raises //! a candidate's score, negative patterns lower it, and a double //! threshold resolves decisive candidates right here — high scores @@ -14,10 +16,11 @@ //! 3. a context window around each remaining candidate //! (±[`WINDOW_CONTEXT_CHARS`] chars — the keyword-proximity window //! magnitude mainstream DLP engines use, typically 50–300 chars) is -//! embedded by the local model and compared, by cosine similarity, -//! against the category's prototype vector set (encoded at load -//! time; see [`PrototypeStrategy`]); above-threshold candidates are -//! rewritten in place to [`MASK_REPLACEMENT`]. +//! embedded by the local model and scored RELATIVELY against the +//! category's positive and negative prototype sets +//! (`max_pos − max_neg`; encoded at load time, see +//! [`PrototypeStrategy`] / [`PrototypeSet`]); above-threshold +//! candidates are rewritten in place to [`MASK_REPLACEMENT`]. //! //! Everything not rewritten is returned byte-identical. //! @@ -92,6 +95,8 @@ //! ONNX Runtime download) with `ORT_LIB_PATH` pointing at a pre-fetched //! library (see `ort-sys` `build/vars.rs`). +#[cfg(test)] +mod adversarial_corpus; mod rules; use std::collections::BTreeMap; @@ -116,8 +121,11 @@ use rules::{RuleDecision, RuleScorer}; /// `tokenizer.json`). Set → the server bootstrap loads and injects the /// guardrail; unset → the feature is completely inert. pub const MODEL_DIR_ENV: &str = "GUARDRAIL_LOCAL_MODEL_DIR"; -/// Optional cosine-similarity gate override (default: the configured -/// strategy's calibrated `default_threshold`). +/// Optional score-gate override (default: the configured strategy's +/// calibrated `default_threshold`). NOTE the scale depends on the +/// strategy: `description` scores absolute cosine in [-1, 1]; the +/// sample strategies score the relative margin `max_pos − max_neg` +/// in [-2, 2]. pub const THRESHOLD_ENV: &str = "GUARDRAIL_LOCAL_MODEL_THRESHOLD"; /// Optional inference-lane count (default 1, clamped to /// [`MAX_LANES`]). Each lane is one ONNX session — one more core the @@ -148,15 +156,26 @@ const MAX_LANES: usize = 32; /// compile-time constant. const PROTOTYPE_DESCRIPTION_ZH: &str = "EDA 软件的版本号"; -/// Sample sentences for the sample-based prototype strategies — the v2 -/// "customer supplies example sentences" path from the design issue, -/// collapsed to a compile-time constant set (synthesized; real customer -/// corpus not yet available). Coverage is by SHAPE, not by string: the -/// upgrade/rollback phrasing and the tool-name+version phrasing that the -/// single description prototype measurably missed, in Chinese and -/// English. Tool names and numbers are deliberately DIFFERENT from the -/// probe corpus (Spectre/Xcelium here, Virtuoso in the probes) so the -/// calibration probes measure shape generalization, not string overlap. +/// Positive sample sentences for the sample-based prototype strategies — +/// the v2 "customer supplies example sentences" path from the design +/// issue, collapsed to a compile-time constant set (synthesized; real +/// customer corpus not yet available). Coverage is by SHAPE, not by +/// string: upgrade/rollback phrasing, tool-name+version phrasing, +/// anchor-free "we run X" phrasing, and FUSED version tokens, in Chinese +/// and English. Tool names and numbers are deliberately DIFFERENT from +/// the probe/adversarial corpora (Spectre/Genus and invented fused +/// tokens here; Virtuoso/Xcelium-family tokens in the corpora) so the +/// calibration probes measure shape generalization, not string overlap — +/// which is also why the MVP's `Xcelium 23.09` sample left this list +/// when `XCELIUM2309` entered the adversarial corpus. +/// +/// Scale note: 24 positives + 90 negatives (~1:3.75). The ecosystem's +/// published floor for trainable classifiers is 50–500 positives and +/// ≥150 negatives at ~1:3 (Microsoft Purview, +/// ); +/// this set moves from 10:0 to a meaningful fraction of that floor and +/// the ratio it prescribes, and the rest is the evaluation-set work +/// (AISIX-Cloud#1332), not more synthesis. const PROTOTYPE_SAMPLES: &[&str] = &[ "布局布线工具升级到 21.15 之后跑得快多了", "仿真器回退到 19.03 才恢复正常", @@ -164,22 +183,158 @@ const PROTOTYPE_SAMPLES: &[&str] = &[ "综合工具的版本号是 2020.09,不要外传", "Spectre 23.1.0 在这个工艺角下会崩溃", "签核工具装的是 22.4 这个版本", + "时序工具从 18.1 换到 20.2 就没再出过问题", + "现在生产环境跑的是 16.3 那个版本的布线器", + "形式验证工具还停在 10.6,太老了", + "装了 31.2 之后 license 就报错", + "版图工具的补丁版本是 QSV302", + "提取工具升级到 QRC1921 以后内存翻倍", + "DRC 用的签核包是 K-2019.06-SP1", + "那台机器装的仿真器是 v14.2-p004", "We upgraded the place-and-route tool to 21.15", "The simulator crashed on release 6.2.1", - "Xcelium 23.09 fails on this testbench", "The sign-off tool version is 2020.09", + "Genus 19.13 fails on this floorplan", + "the flow needs tool build 30.4 or newer", + "we rolled back to 17.0 after the crash", + "they still run SPECTRE181 in production", + "the timing box has PT-2021.06-SP3 installed", + "our extraction flow is pinned to v19.1-s022_2", + "the older 14.7 install still passes DRC", ]; -/// How the category's prototype vector set is built at load time. +/// Negative sample sentences: numbers that LOOK like the candidate shape +/// but carry non-software semantics. This is the other half of the +/// relative scoring form — under the absolute form the model had to +/// clear a fixed bar with no notion of what "not a version" looks like, +/// and the measured margin on anchor-free windows was NEGATIVE (the MVP +/// finding, reproduced on the adversarial corpus). Fifteen semantic +/// families × 6, zh+en: math constants, exchange rates / finance, body +/// measurements, dates, quantities/statistics, spelled durations, +/// physical quantities, dimensions, process nodes, scores/ratios, +/// section numbers, clock times, and bare number sequences (data rows / +/// log dumps — the driving corpus is dense compile logs, and without +/// this family a context-free run of numbers sits EXACTLY on the +/// relative-score decision boundary, where int8 noise picks the sign), +/// plus files/hashes/tickets/standard numbers and product/model +/// identifiers (the fused-token relaxation makes everyday identifiers — +/// filenames, commit hashes, GPU and LLM model names — candidates, and +/// the audit measured them mis-masking without these two families). +/// Numbers are disjoint from the corpora and the positive set. +const NEGATIVE_PROTOTYPE_SAMPLES: &[&str] = &[ + "圆周率约是 3.1416", + "自然常数 e 约等于 2.71828", + "黄金分割比大约是 1.618", + "根号二约等于 1.41421", + "pi is roughly 3.1416", + "the golden ratio is about 1.618", + "今天美元兑人民币汇率是 7.18", + "欧元汇率涨到 7.92", + "股价收在 24.35", + "年化利率是 3.65", + "the exchange rate moved to 7.15", + "the stock closed at 132.5 today", + "早上量体温 36.6,正常", + "孩子昨晚烧到 39.2", + "空腹血糖 5.2,没问题", + "体重降到 62.5 公斤了", + "her temperature was 37.8 last night", + "resting heart rate dropped to 58.5", + "会议改到 9.28 上午十点", + "项目截止日期是 2026.10.31", + "10.1 假期值班表出来了", + "发票日期写的 2025.12.05", + "the review is scheduled for 11.20", + "the contract was signed on 2026.4.30", + "这批晶圆一共 8.5 万片", + "平均每天触发 4.5 次告警", + "样本均值 6.35,标准差 1.2", + "库存还剩 3.5 箱", + "we shipped 2.4 million units last year", + "the average queue depth is 5.5", + "排队等了 3.5 个星期", + "面试聊了 1.5 个钟头", + "整个流程走了 4.5 个月", + "assembly 那步要等 2.5 个工作日", + "it took 2.5 weeks end to end", + "the outage lasted 3.5 days", + "结温升到 88.5 度就限频", + "内核电压是 0.72 伏", + "整机功耗 5.5 瓦左右", + "环境温度 23.5 度恒温", + "the die temperature hit 95.5 degrees", + "supply voltage sagged to 0.66 volts", + "die 面积是 15.21 平方毫米", + "键合线直径 25.4 微米", + "这条走线长 3.6 毫米", + "硅片厚度 0.775 毫米", + "the package is 10.5 by 10.5 millimeters", + "the wafer is 0.725 millimeters thick", + "主力工艺切到 N2 了", + "这个块还在 N16 上", + "新项目评估 4nm 的 PDK", + "老产品线停留在 14nm", + "the pilot line runs N6", + "we are qualifying the 3nm flow", + "客户满意度打了 9.2 分", + "评审平均分 8.65", + "基准测试跑分 456.5", + "良率这周爬到 91.5", + "the benchmark scores 78.5 overall", + "approval rating sits at 62.5", + "详见第 2.4 节", + "规范的 5.3.1 条款有说明", + "图 4.2 画的是数据通路", + "表 6.1 列出了引脚定义", + "see section 3.4.2 of the spec", + "chapter 12.3 covers the protocol", + "日志停在 11:42:07.333", + "[07:03:59.001] job finished", + "晚上 20.45 的班车", + "闹钟定在 6.30", + "the cron fires at 23.55 every night", + "the shuttle leaves at 8.15", + "0.2 0.4 0.6 0.8 1.0 1.2", + "数据列是 2.2 4.4 6.6 8.8", + "表里那列全是 1.3 2.6 3.9 5.2 这种数", + "the raw dump reads 0.5 1.5 2.5 3.5 4.5", + "column two is 9.1 8.2 7.3 6.4", + "坐标序列 10.5 20.5 30.5 40.5", + "配置都在 setup2.cfg 里", + "commit 是 f00dbabe42 那个", + "工单编号是 JIRA-1024", + "the log lives in run5.txt", + "the checksum is 9f8e7d6c5b", + "先过 802.3af 认证再说", + "换成 gpt-5.2 再试一次", + "这个 bug 用 llama-3.1 也能复现", + "显卡是 RTX4090", + "主控芯片是 BCM2712", + "the endpoint serves claude-haiku-4.5", + "the box ships with an RTX4080 inside", +]; + +/// How the category's prototype vector sets are built at load time. +/// +/// All three strategies score through the same relative form +/// ([`PrototypeSet::score`]): `max_pos − max_neg`, with an empty +/// negative set contributing 0 — so `Description` (no negative +/// material) keeps its MVP absolute-cosine semantics unchanged, and the +/// sample strategies gain the contrastive term the adversarial corpus +/// showed the absolute form cannot do without (its measured margin on +/// anchor-free windows was negative under EVERY positive-only +/// construction). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrototypeStrategy { - /// One vector: the embedded category description (the MVP form). + /// One positive vector: the embedded category description (the MVP + /// form). No negative set. Description, - /// One vector per [`PROTOTYPE_SAMPLES`] entry; a window scores by its - /// MAX cosine over the set (nearest sample decides). + /// One vector per [`PROTOTYPE_SAMPLES`] / [`NEGATIVE_PROTOTYPE_SAMPLES`] + /// entry; a window scores by max-cosine per set, nearest sample on + /// each side decides. SampleMax, - /// One vector: the L2-renormalized mean of the sample embeddings; a - /// window scores against the class centroid. + /// One vector per side: the L2-renormalized mean of each sample set; + /// a window scores against the two class centroids. SampleCentroid, } @@ -203,45 +358,59 @@ impl PrototypeStrategy { } } - /// Cosine gate calibrated per strategy with this module's - /// `#[ignore]` probe matrix (cosine absolute scale shifts with the - /// prototype construction, so one shared default would be wrong for - /// two of the three). Measured bands (granite-97m int8): - /// - `Description` keeps the MVP calibration: acceptance positive - /// ~0.90, every probed negative ≤0.76, hard positives 0.75–0.79 - /// below the gate — the measured single-prototype recall gap - /// (negative hard margin in all five phrasings swept) that - /// layer ② now covers. - /// - `SampleMax`: hard positives ≥0.8316, negatives ≤0.7867 - /// (hard margin +0.0449); 0.82 sits precision-leaning in that - /// band — 0.033 above the negative ceiling. - /// - `SampleCentroid`: hard positives ≥0.8616, negatives ≤0.8370 - /// (hard margin +0.0246); 0.85 likewise. + /// Score gate calibrated per strategy with this module's + /// `#[ignore]` probe matrix and the adversarial-corpus report (the + /// score SCALE shifts with the prototype construction, so one + /// shared default would be wrong for two of the three). + /// - `Description` keeps the MVP absolute-cosine calibration + /// (no negative set ⇒ score IS the positive cosine): acceptance + /// positive ~0.90, every probed negative ≤0.76. + /// - `SampleMax` / `SampleCentroid` gate the RELATIVE margin + /// `max_pos − max_neg`; the calibrated values are pinned by the + /// probe matrix so model/sample drift fails the calibration test + /// instead of silently shifting behavior. /// /// All three lean precision — a mask false-positive corrupts user /// content; layer ② carries recall for anchored shapes. fn default_threshold(self) -> f32 { match self { Self::Description => 0.80, - Self::SampleMax => 0.82, - Self::SampleCentroid => 0.85, + Self::SampleMax => 0.0, + Self::SampleCentroid => 0.0, } } } impl Default for PrototypeStrategy { /// `SampleMax`: the probe matrix (module tests) measures the widest - /// positive hard margin here (+0.0449 vs +0.0246 for the centroid — - /// averaging ten shape-diverse samples into one vector costs + /// relative hard margin here (+0.0355 vs +0.0341 for the centroid — + /// averaging shape-diverse samples into one vector costs /// nearest-shape resolution). fn default() -> Self { Self::SampleMax } } -/// Candidate shape: a dotted number run (`12.1`, `2022.4`, `6.1.8`). -/// Plain integers are out of MVP scope. -const CANDIDATE_PATTERN: &str = r"\d+(?:\.\d+)+"; +/// Candidate shape A: a dotted number run (`12.1`, `2022.4`, `6.1.8`), +/// ASCII or fullwidth (`１２．１` — Chinese-IME phrasing is accidental, +/// not adversarial, so it is in scope). Plain integers stay out of +/// scope. +const CANDIDATE_PATTERN: &str = r"[0-9０-９]+(?:[.．][0-9０-９]+)+"; + +/// Candidate shape B: a fused version token — a maximal +/// `[A-Za-z0-9._-]` run mixing ASCII letters and digits. Real EDA +/// corpora fuse the version into one token the dotted shape cannot see +/// (`IC618`, `ICADV12.3`, `XCELIUM2309`, `MMSIM151`, `E-2010.12-ICC-SP2`, +/// `v16.12-s051_1`, `T-2022.03`, `20.09-s003`) — the adversarial-corpus +/// finding this widens layer ① for. The shape is deliberately BROAD +/// (`7nm`, `N5`, `sha256`-ish identifiers all qualify): a garbage +/// candidate costs a rule score in microseconds and at worst one model +/// call, while an invisible candidate is an unconditional leak — layers +/// ②③ exist precisely so ① does not have to be precise. Leading and +/// trailing `[._-]` are trimmed (sentence punctuation), and a token +/// must mix letters AND digits — pure words and pure numbers fall back +/// to shape A or drop out. +const FUSED_TOKEN_PATTERN: &str = r"[A-Za-z0-9._-]+"; /// Context chars kept on each side of a candidate when cutting the /// window the model judges. @@ -315,17 +484,32 @@ impl LocalModelConfig { /// default rather than failing boot — the gate is a tuning knob, not /// a correctness one. The range check matters: `"NaN"` parses as a /// valid f32 and would make `score >= threshold` always false — a - /// configured-looking guardrail that silently never masks. Lanes and - /// the rule window follow the same lenient rule (malformed → - /// default). + /// configured-looking guardrail that silently never masks. The + /// accepted range is the RELATIVE-score span [-2, 2] (see + /// [`THRESHOLD_ENV`]); `description`'s meaningful values are its + /// [0, 1] subset. Lanes and the rule window follow the same lenient + /// rule (malformed → default). pub fn from_env() -> Option { let model_dir = PathBuf::from(std::env::var_os(MODEL_DIR_ENV)?); let prototypes = PrototypeStrategy::parse(std::env::var(PROTOTYPES_ENV).ok().as_deref()); let threshold = std::env::var(THRESHOLD_ENV) .ok() .and_then(|s| s.parse::().ok()) - .filter(|t| t.is_finite() && (0.0..=1.0).contains(t)) + .filter(|t| t.is_finite() && (-2.0..=2.0).contains(t)) .unwrap_or_else(|| prototypes.default_threshold()); + // The sample strategies moved from absolute cosine to the + // relative margin scale: a pre-migration override (e.g. the old + // 0.82) is far above any reachable margin and would silently + // turn every model-band judgement into a release. + if prototypes != PrototypeStrategy::Description && threshold > 0.5 { + tracing::warn!( + threshold, + strategy = ?prototypes, + "{THRESHOLD_ENV} looks like an absolute-cosine value, but this \ + strategy gates the relative margin (max_pos − max_neg, ~±0.2): \ + the model band will likely never mask" + ); + } let lanes = parse_lanes(std::env::var(LANES_ENV).ok().as_deref()); let rule_window = std::env::var(RULE_WINDOW_ENV) .ok() @@ -500,16 +684,39 @@ fn cosine(a: &[f32], b: &[f32]) -> f32 { a.iter().zip(b).map(|(x, y)| x * y).sum() } -/// A window's score against the prototype set: max cosine over the set -/// (with one vector this IS plain cosine, so all three strategies score -/// through here). -fn prototype_score(prototypes: &[Vec], v: &[f32]) -> f32 { +/// Max cosine over a prototype set (nearest prototype decides). +fn max_cosine(prototypes: &[Vec], v: &[f32]) -> f32 { prototypes .iter() .map(|p| cosine(p, v)) .fold(f32::NEG_INFINITY, f32::max) } +/// The category's prototype material: a positive set and a negative set +/// (either may be a single centroid; see [`PrototypeStrategy`]). +struct PrototypeSet { + positive: Vec>, + negative: Vec>, +} + +impl PrototypeSet { + /// The relative scoring form: `max_pos − max_neg`. Nearest-prototype + /// max-over-set on each side; an empty negative set contributes 0, + /// which collapses to the MVP's absolute form for `Description`. + /// This is the standard contrastive nearest-prototype shape (the + /// commercial precedent for sample-set semantic matching, Azure AI + /// Content Safety custom categories, likewise scores candidate + /// classes against each other rather than against a fixed bar). + fn score(&self, v: &[f32]) -> f32 { + let pos = max_cosine(&self.positive, v); + if self.negative.is_empty() { + pos + } else { + pos - max_cosine(&self.negative, v) + } + } +} + /// L2-renormalized mean of a set of L2-normalized vectors — the /// [`PrototypeStrategy::SampleCentroid`] construction. fn centroid(vectors: &[Vec]) -> Vec { @@ -538,13 +745,86 @@ fn centroid(vectors: &[Vec]) -> Vec { /// per-pass budget so they cannot starve legitimate candidates either. const MAX_CANDIDATE_SPAN_BYTES: usize = 64; -/// Candidate spans (byte ranges) in `text`, in order. Spans longer than -/// [`MAX_CANDIDATE_SPAN_BYTES`] are not candidates (see the constant). -fn candidate_spans(re: &Regex, text: &str) -> Vec> { - re.find_iter(text) - .map(|m| m.range()) - .filter(|s| s.len() <= MAX_CANDIDATE_SPAN_BYTES) - .collect() +/// Layer ①: the compiled candidate generator — the two shapes above, +/// merged and de-overlapped. +struct CandidateFinder { + dotted: Regex, + fused: Regex, +} + +impl CandidateFinder { + fn new() -> Self { + let compile = |p: &str| Regex::new(p).expect("candidate pattern must compile"); + Self { + dotted: compile(CANDIDATE_PATTERN), + fused: compile(FUSED_TOKEN_PATTERN), + } + } + + /// Candidate spans (byte ranges) in `text`, ascending and + /// non-overlapping. Fused tokens win overlaps with dotted runs (the + /// dotted digits of `ICADV12.3` are PART of the version — masking + /// only them leaks the `ICADV` identity, the pre-fix behavior). + /// Spans longer than [`MAX_CANDIDATE_SPAN_BYTES`] are not candidates + /// (see the constant). + fn spans(&self, text: &str) -> Vec> { + let mut spans: Vec> = self + .fused + .find_iter(text) + .map(|m| trim_token(text, m.range())) + .filter(|s| { + let token = text[s.clone()].as_bytes(); + s.len() <= MAX_CANDIDATE_SPAN_BYTES + && token.iter().any(u8::is_ascii_digit) + && token.iter().any(u8::is_ascii_alphabetic) + }) + .collect(); + // Maximal same-class runs never overlap each other; a dotted run + // either sits inside a fused token (drop it — the fused span + // masks more) or stands alone. The overlap (not containment) + // check also covers mixed-width pathologies (`12．3-s1`), where + // trimming could otherwise leave two intersecting spans. + // + // Both lists arrive ascending and internally disjoint + // (`find_iter` order), so the overlap check is a linear + // two-pointer walk over the FUSED prefix only — a growing-vector + // `iter().any()` here is O(n²) and turns a `"1.1 "`-flood + // megabyte into ~17 s of synchronous work on the async worker + // BEFORE the per-segment cap can meter anything (audit finding + // on this PR; measured quadratic: 1.37 s at 256 KiB, 17.6 s at + // 1 MiB — linear after the fix). + let fused_len = spans.len(); + let mut fi = 0; + for m in self.dotted.find_iter(text) { + let r = m.range(); + if r.len() > MAX_CANDIDATE_SPAN_BYTES { + continue; + } + while fi < fused_len && spans[fi].end <= r.start { + fi += 1; + } + if fi >= fused_len || spans[fi].start >= r.end { + spans.push(r); + } + } + spans.sort_by_key(|s| s.start); + spans + } +} + +/// Strip leading/trailing `[._-]` from a fused-token match: the char +/// class must include them mid-token (`E-2010.12-ICC-SP2`), which makes +/// sentence punctuation stick to a token at the rim (`v16.12-s051_1.`). +/// ASCII-only, so byte trimming is char-safe. +fn trim_token(text: &str, mut span: Range) -> Range { + let bytes = text.as_bytes(); + while span.start < span.end && matches!(bytes[span.start], b'.' | b'_' | b'-') { + span.start += 1; + } + while span.start < span.end && matches!(bytes[span.end - 1], b'.' | b'_' | b'-') { + span.end -= 1; + } + span } /// The context window around `span`: `ctx` chars on each side, snapped @@ -582,10 +862,10 @@ fn apply_masks(text: &str, spans: &[Range]) -> String { /// The runtime guardrail. Always-`Allow`; masks via the segment hooks. pub struct LocalModelGuardrail { embedder: Arc, - /// L2-normalized prototype vector set (see [`PrototypeStrategy`]). - prototypes: Vec>, + /// L2-normalized prototype vector sets (see [`PrototypeStrategy`]). + prototypes: PrototypeSet, threshold: f32, - candidate_re: Regex, + finder: CandidateFinder, /// Layer-② scorer (hotword proximity + negative patterns). rules: RuleScorer, /// Bounds in-flight `spawn_blocking` inference tasks. Sized to the @@ -602,16 +882,25 @@ impl LocalModelGuardrail { pub fn load(config: &LocalModelConfig) -> Result { let started = Instant::now(); let embedder = Embedder::load(&config.model_dir, config.lanes)?; - let embed_samples = || { - PROTOTYPE_SAMPLES + let embed_all = |samples: &[&str]| { + samples .iter() .map(|s| embedder.embed(s)) - .collect::, _>>() + .collect::, LocalModelError>>() }; let prototypes = match config.prototypes { - PrototypeStrategy::Description => vec![embedder.embed(PROTOTYPE_DESCRIPTION_ZH)?], - PrototypeStrategy::SampleMax => embed_samples()?, - PrototypeStrategy::SampleCentroid => vec![centroid(&embed_samples()?)], + PrototypeStrategy::Description => PrototypeSet { + positive: vec![embedder.embed(PROTOTYPE_DESCRIPTION_ZH)?], + negative: Vec::new(), + }, + PrototypeStrategy::SampleMax => PrototypeSet { + positive: embed_all(PROTOTYPE_SAMPLES)?, + negative: embed_all(NEGATIVE_PROTOTYPE_SAMPLES)?, + }, + PrototypeStrategy::SampleCentroid => PrototypeSet { + positive: vec![centroid(&embed_all(PROTOTYPE_SAMPLES)?)], + negative: vec![centroid(&embed_all(NEGATIVE_PROTOTYPE_SAMPLES)?)], + }, }; tracing::info!( model_dir = %config.model_dir.display(), @@ -619,7 +908,8 @@ impl LocalModelGuardrail { lanes = config.lanes, rule_window = config.rule_window, strategy = ?config.prototypes, - prototypes = prototypes.len(), + positive_prototypes = prototypes.positive.len(), + negative_prototypes = prototypes.negative.len(), load_ms = started.elapsed().as_millis() as u64, "local-model guardrail loaded (category: EDA software version)" ); @@ -627,7 +917,7 @@ impl LocalModelGuardrail { embedder: Arc::new(embedder), prototypes, threshold: config.threshold, - candidate_re: Regex::new(CANDIDATE_PATTERN).expect("candidate pattern must compile"), + finder: CandidateFinder::new(), rules: RuleScorer::new(config.rule_window), permits: Arc::new(tokio::sync::Semaphore::new(config.lanes)), }) @@ -670,7 +960,7 @@ impl LocalModelGuardrail { let mut hits: Vec> = Vec::new(); let (mut rule_masked, mut rule_passed, mut model_judged) = (0u32, 0u32, 0u32); let mut over_budget = false; - let spans = candidate_spans(&self.candidate_re, text); + let spans = self.finder.spans(text); if spans.len() > MAX_RULE_SCORED_SPANS_PER_SEGMENT { tracing::warn!( candidates = spans.len(), @@ -701,7 +991,7 @@ impl LocalModelGuardrail { let window = text[window_bounds(text, &span, WINDOW_CONTEXT_CHARS)].to_owned(); match self.embed_window(window).await { Ok(vector) => { - let score = prototype_score(&self.prototypes, &vector); + let score = self.prototypes.score(&vector); tracing::debug!( score, threshold = self.threshold, @@ -799,16 +1089,87 @@ impl Guardrail for LocalModelGuardrail { mod tests { use super::*; - fn re() -> Regex { - Regex::new(CANDIDATE_PATTERN).unwrap() + fn spans_of(text: &str) -> Vec> { + CandidateFinder::new().spans(text) + } + + fn values_of(text: &str) -> Vec<&str> { + spans_of(text).into_iter().map(|s| &text[s]).collect() } #[test] fn candidate_spans_find_dotted_runs_only() { let text = "版本是 12.1,构建号 2022.4.1,端口 8080"; - let spans = candidate_spans(&re(), text); - let values: Vec<&str> = spans.iter().map(|s| &text[s.clone()]).collect(); - assert_eq!(values, vec!["12.1", "2022.4.1"]); + // Plain integers are still not candidates. + assert_eq!(values_of(text), vec!["12.1", "2022.4.1"]); + } + + #[test] + fn candidate_spans_find_fused_tokens_whole() { + // The adversarial-corpus shapes: version fused with the tool + // name / letter affixes into ONE token — the candidate is the + // whole token, not its dotted substring. + for (text, want) in [ + ("Virtuoso IC618 又崩了", "IC618"), + ("版图工具用的是 ICADV12.3", "ICADV12.3"), + ("XCELIUM2309 的仿真结果对不上", "XCELIUM2309"), + ("MMSIM151 装在新机器上了", "MMSIM151"), + ("综合用的 T-2022.03 有已知问题", "T-2022.03"), + ("回退到 E-2010.12-ICC-SP2 就不崩了", "E-2010.12-ICC-SP2"), + ("装的是 v16.12-s051_1 这个版本", "v16.12-s051_1"), + ("hotfix 20.09-s003 已经推送了", "20.09-s003"), + ("7nm 工艺下功耗有点高", "7nm"), + ("这个块是 N5 工艺的", "N5"), + ] { + assert_eq!(values_of(text), vec![want], "text: {text}"); + } + } + + #[test] + fn candidate_spans_find_fullwidth_dotted_runs() { + assert_eq!(values_of("版本是 １２．１,不要外传"), vec!["１２．１"]); + // Mixed-width digits with a fullwidth dot still form one run. + assert_eq!(values_of("旧版是 ２０２２．4"), vec!["２０２２．4"]); + } + + #[test] + fn fused_tokens_trim_rim_punctuation() { + // Sentence punctuation from the token char class must not stick. + assert_eq!(values_of("pinned to v16.12-s051_1."), vec!["v16.12-s051_1"]); + // A pure word and a pure dash-number never become fused tokens. + assert_eq!(values_of("high-performance run -3.5 offset"), vec!["3.5"]); + } + + #[test] + fn candidate_dedupe_keeps_standalone_dotted_runs_between_fused_tokens() { + // Interleaved fused and dotted candidates: the two-pointer + // dedupe must drop exactly the dotted runs inside fused tokens + // and keep the standalone ones (regression for the O(n²) + // rewrite — audit finding). + let text = "v1.2-a 3.4 IC5.6 7.8 soc9.9x 10.11"; + assert_eq!( + values_of(text), + vec!["v1.2-a", "3.4", "IC5.6", "7.8", "soc9.9x", "10.11"] + ); + } + + #[test] + fn candidate_generation_stays_linear_on_floods() { + // The audit measured the pre-fix quadratic dedupe at 17.6 s of + // synchronous CPU for a 1 MiB `"1.1 "` flood (1.37 s at + // 256 KiB) — BEFORE the per-segment cap could meter anything. + // Linear generation does this in tens of milliseconds; the + // bound leaves two orders of magnitude of CI headroom while + // sitting far below the quadratic's floor. + let flood = "1.1 ".repeat(256 * 1024); // 1 MiB + let started = Instant::now(); + let spans = CandidateFinder::new().spans(&flood); + assert_eq!(spans.len(), 256 * 1024); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "candidate generation took {:?} on a 1 MiB flood", + started.elapsed() + ); } #[test] @@ -818,15 +1179,13 @@ mod tests { // so it can neither stall the lane nor starve real candidates. let bomb = "1.1".repeat(60); // 180 bytes, single match let text = format!("前缀 {bomb} 中缀 12.1 后缀"); - let spans = candidate_spans(&re(), &text); - let values: Vec<&str> = spans.iter().map(|s| &text[s.clone()]).collect(); - assert_eq!(values, vec!["12.1"]); + assert_eq!(values_of(&text), vec!["12.1"]); } #[test] fn window_bounds_snap_to_char_boundaries() { let text = "这个 EDA 软件的版本是 12.1,请勿外传"; - let span = candidate_spans(&re(), text).remove(0); + let span = spans_of(text).remove(0); // A tiny context still lands on char boundaries around CJK. let w = window_bounds(text, &span, 3); let window = &text[w]; @@ -837,7 +1196,7 @@ mod tests { #[test] fn window_bounds_clamp_to_text_edges() { let text = "12.1 只有后文"; - let span = candidate_spans(&re(), text).remove(0); + let span = spans_of(text).remove(0); let w = window_bounds(text, &span, 50); assert_eq!(&text[w], text); } @@ -845,7 +1204,7 @@ mod tests { #[test] fn apply_masks_rewrites_right_to_left() { let text = "从 12.1 升到 13.0 了"; - let spans = candidate_spans(&re(), text); + let spans = spans_of(text); assert_eq!(apply_masks(text, &spans), "从 *** 升到 *** 了"); } @@ -900,7 +1259,25 @@ mod tests { assert!((c[0] - inv_sqrt2).abs() < 1e-6 && (c[1] - inv_sqrt2).abs() < 1e-6); // Max-over-set picks the nearest prototype. let set = vec![vec![1.0, 0.0], vec![0.0, 1.0]]; - assert!((prototype_score(&set, &[0.0, 1.0]) - 1.0).abs() < 1e-6); + assert!((max_cosine(&set, &[0.0, 1.0]) - 1.0).abs() < 1e-6); + } + + #[test] + fn prototype_set_scores_relatively() { + let set = PrototypeSet { + positive: vec![vec![1.0, 0.0]], + negative: vec![vec![0.0, 1.0]], + }; + // Aligned with the positive prototype: margin +1 − 0. + assert!((set.score(&[1.0, 0.0]) - 1.0).abs() < 1e-6); + // Aligned with the negative prototype: margin 0 − 1. + assert!((set.score(&[0.0, 1.0]) + 1.0).abs() < 1e-6); + // No negative material collapses to the absolute form. + let desc = PrototypeSet { + positive: vec![vec![1.0, 0.0]], + negative: Vec::new(), + }; + assert!((desc.score(&[1.0, 0.0]) - 1.0).abs() < 1e-6); } #[test] @@ -927,17 +1304,16 @@ mod tests { Some(LocalModelGuardrail::load(&cfg).expect("model files present but load failed")) } - /// The MVP probe matrix, re-run for the prototype-set experiment: - /// the same 7 probe windows (1 acceptance-style positive, 2 hard - /// positives, 4 hard negatives) scored against 5 single-description - /// prototype phrasings (the MVP sweep that measured NEGATIVE margin - /// in every column) plus the two sample-based strategies. Prints the - /// full matrix and each column's hard margin - /// (min over hard positives − max over negatives). - /// - /// The MVP's original 5-phrasing sweep was scratch work; these - /// phrasings reconstruct it (the shipped description first) and are - /// committed so the experiment stays repeatable. + /// The probe matrix, re-run for the relative-scoring form: the same + /// 7 probe windows as the MVP sweep (1 acceptance-style positive, + /// 2 hard positives, 4 hard negatives) scored against 5 + /// single-description prototype phrasings (the MVP sweep that + /// measured NEGATIVE margin in every column) plus the two + /// sample-based strategies, each in BOTH scoring forms — `abs` is + /// the old positive-only max cosine, `rel` the shipped + /// `max_pos − max_neg`. Prints the full matrix and each column's + /// hard margin (min over hard positives − max over negatives) per + /// form; the calibration assertion pins the shipped (relative) form. #[tokio::test] #[ignore = "needs GUARDRAIL_LOCAL_MODEL_DIR with model.onnx + tokenizer.json"] async fn probe_similarity_matrix() { @@ -959,41 +1335,61 @@ mod tests { ("NEG", "工艺节点是 0.13um,良率还行"), ]; - // Columns: each phrasing as a single-vector prototype set, then - // the sample set (max) and its centroid. - let mut columns: Vec<(String, Vec>)> = Vec::new(); + let mut columns: Vec<(String, PrototypeSet)> = Vec::new(); for p in phrasings { let v = g.embed_window(p.to_owned()).await.unwrap(); - columns.push((format!("desc:{p}"), vec![v])); + columns.push(( + format!("desc:{p}"), + PrototypeSet { + positive: vec![v], + negative: Vec::new(), + }, + )); } - let mut samples = Vec::new(); + let mut pos = Vec::new(); for s in PROTOTYPE_SAMPLES { - samples.push(g.embed_window((*s).to_owned()).await.unwrap()); + pos.push(g.embed_window((*s).to_owned()).await.unwrap()); } - columns.push(("samples-max".to_owned(), samples.clone())); - columns.push(("samples-centroid".to_owned(), vec![centroid(&samples)])); - - for (name, prototypes) in &columns { - let mut hard_pos_min = f32::INFINITY; - let mut neg_max = f32::NEG_INFINITY; + let mut neg = Vec::new(); + for s in NEGATIVE_PROTOTYPE_SAMPLES { + neg.push(g.embed_window((*s).to_owned()).await.unwrap()); + } + columns.push(( + "samples-max".to_owned(), + PrototypeSet { + positive: pos.clone(), + negative: neg.clone(), + }, + )); + columns.push(( + "samples-centroid".to_owned(), + PrototypeSet { + positive: vec![centroid(&pos)], + negative: vec![centroid(&neg)], + }, + )); + + for (name, set) in &columns { + let mut margins = [(f32::INFINITY, f32::NEG_INFINITY); 2]; // abs, rel println!("── column: {name}"); for (kind, text) in windows { let v = g.embed_window(text.to_owned()).await.unwrap(); - let s = prototype_score(prototypes, &v); - println!(" {kind} {s:.4} {text}"); - match kind { - "POS" => hard_pos_min = hard_pos_min.min(s), - "NEG" => neg_max = neg_max.max(s), - _ => {} + let abs = max_cosine(&set.positive, &v); + let rel = set.score(&v); + println!(" {kind} abs {abs:.4} rel {rel:+.4} {text}"); + for (m, s) in margins.iter_mut().zip([abs, rel]) { + match kind { + "POS" => m.0 = m.0.min(s), + "NEG" => m.1 = m.1.max(s), + _ => {} + } } } - println!( - " hard margin (min POS − max NEG): {:+.4}", - hard_pos_min - neg_max - ); + let [abs_m, rel_m] = margins.map(|(p, n)| p - n); + println!(" hard margin (min POS − max NEG): abs {abs_m:+.4} rel {rel_m:+.4}"); // Pin the calibration contract for the sample strategies: - // the margin the experiment claims stays open, and the + // the relative margin the fix claims stays open, and the // shipped default gate sits strictly inside it. The // description columns stay unasserted — their negative // margin is the documented MVP finding, not a contract. @@ -1003,9 +1399,10 @@ mod tests { _ => None, }; if let Some(gate) = gate { + let (rel_pos_min, rel_neg_max) = margins[1]; assert!( - neg_max < gate && gate <= hard_pos_min, - "{name}: default gate {gate} outside the measured band ({neg_max:.4}, {hard_pos_min:.4}]" + rel_neg_max < gate && gate <= rel_pos_min, + "{name}: default gate {gate} outside the measured relative band ({rel_neg_max:.4}, {rel_pos_min:.4}]" ); } } @@ -1020,7 +1417,13 @@ mod tests { async fn candidate_flood_releases_the_tail() { let Some(g) = load_from_env() else { return }; let flood = "1.1 ".repeat(MAX_RULE_SCORED_SPANS_PER_SEGMENT); - let text = format!("{flood}这个 EDA 软件的版本是 12.1"); + // Padding wider than the ±50-char context window between the + // flood and the bait sentence: the model judges WINDOWS, so a + // pre-cap flood span whose window overlaps the bait would be + // (semantically correctly!) masked, and the test would measure + // window contamination instead of the cap. The padding word is + // letters-only — not a candidate. + let text = format!("{flood}{} 这个 EDA 软件的版本是 12.1", "x".repeat(60)); let outcome = g.moderate_input_segments(&[text]).await; assert_eq!(outcome.verdict, GuardrailVerdict::Allow); assert!( @@ -1043,7 +1446,12 @@ mod tests { "我们把仿真工具升级到 2022.4 之后速度快了很多", "我们把仿真工具升级到 *** 之后速度快了很多", ), - ("Virtuoso IC6.1.8 出现了崩溃", "Virtuoso IC*** 出现了崩溃"), + // Whole-token rewrite: the fused `IC6.1.8` is ONE candidate + // now, so the tool-fused prefix no longer survives (the MVP + // masked only the dotted digits: `Virtuoso IC***`). + ("Virtuoso IC6.1.8 出现了崩溃", "Virtuoso *** 出现了崩溃"), + ("Virtuoso IC618 又崩了", "Virtuoso *** 又崩了"), + ("版本是 １２．１,不要外传", "版本是 ***,不要外传"), ]; for (input, want) in masked_cases { let outcome = g.moderate_input_segments(&[input.to_owned()]).await; @@ -1058,6 +1466,9 @@ mod tests { "服务器的 IP 地址是 10.2.255.1", "圆周率约等于 3.14159", "工艺节点是 0.13um,良率还行", + // The Chinese-unit / timestamp defect classes this PR fixes. + "整个 build 花了 45.5 秒", + "[10:23:45.123] build started", ]; for input in passthrough_cases { let outcome = g.moderate_input_segments(&[input.to_owned()]).await; diff --git a/crates/aisix-guardrails/src/local_model/adversarial_corpus.rs b/crates/aisix-guardrails/src/local_model/adversarial_corpus.rs new file mode 100644 index 00000000..ec3c46e3 --- /dev/null +++ b/crates/aisix-guardrails/src/local_model/adversarial_corpus.rs @@ -0,0 +1,359 @@ +//! Report instrument for the guardrail-defect fixes: an 88-line labeled +//! adversarial corpus covering the three measured defect classes +//! (Chinese-unit negatives, prototype-scoring form, fused-token layer-① +//! recall) plus the shipped acceptance shapes as regressions. +//! +//! This module is test-only data + one `#[ignore]` model-backed test that +//! prints the PR's three required reports: +//! 1. candidate-level rule-layer stats (model-band share, rule-mask +//! precision) and line-level end-to-end accuracy; +//! 2. the prototype-margin comparison on the candidates that reach the +//! model band (old absolute form vs relative form when negative +//! prototypes are present); +//! 3. candidate counts (how much layer-① relaxation widened the funnel +//! and where the widened candidates were resolved). +//! +//! Labels are BY LINE: `sensitive` lists the exact substrings the pipeline +//! must rewrite; everything else must return byte-identical. A candidate +//! is a true positive when it overlaps an occurrence of a sensitive +//! substring, so a partial mask (the pre-fix `IC***` shape) counts as a +//! line miss but still credits the overlapping candidate. + +use std::ops::Range; + +use super::rules::RuleDecision; +use super::*; + +struct Case { + cat: &'static str, + text: &'static str, + sensitive: &'static [&'static str], +} + +/// The corpus (79 lines from the defect brief + 9 from the independent +/// audit and verification rounds: everyday identifiers, measure-word +/// durations, electrical units, the 度过 compound). Tool names and numbers are deliberately disjoint +/// from [`PROTOTYPE_SAMPLES`] (and the negative sample set once it +/// exists) so model-band scores measure shape generalization, not string +/// overlap — the same discipline as the probe matrix. +/// +/// One line per case, kept single-line on purpose (grep-friendly data +/// table — the `ebml.rs` tag-table precedent). +#[rustfmt::skip] +const CORPUS: &[Case] = &[ + // ── Chinese measurement units (defect 1): must all release ────────── + Case { cat: "zh-unit", text: "时钟周期是 0.8 纳秒", sensitive: &[] }, + Case { cat: "zh-unit", text: "这条路径的建立裕量只剩 0.5 纳秒", sensitive: &[] }, + Case { cat: "zh-unit", text: "版本升级后这条路径 slack 变成 0.5 纳秒", sensitive: &[] }, + Case { cat: "zh-unit", text: "中断响应时间 12.5 微秒", sensitive: &[] }, + Case { cat: "zh-unit", text: "升级到新内核后延迟降到 3.5 毫秒", sensitive: &[] }, + Case { cat: "zh-unit", text: "整个 build 花了 45.5 秒", sensitive: &[] }, + Case { cat: "zh-unit", text: "全量回归跑了 90.5 分钟", sensitive: &[] }, + Case { cat: "zh-unit", text: "full chip 综合要 3.5 小时", sensitive: &[] }, + Case { cat: "zh-unit", text: "数据准备还要 2.5 天", sensitive: &[] }, + Case { cat: "zh-unit", text: "日志文件有 128.5 兆字节", sensitive: &[] }, + Case { cat: "zh-unit", text: "内存峰值到了 4.2 吉字节", sensitive: &[] }, + Case { cat: "zh-unit", text: "波形数据一共 1.5 太字节", sensitive: &[] }, + Case { cat: "zh-unit", text: "主频跑到 3.2 吉赫兹", sensitive: &[] }, + Case { cat: "zh-unit", text: "时钟是 800.5 兆赫兹", sensitive: &[] }, + Case { cat: "zh-unit", text: "采样率 44.1 千赫兹", sensitive: &[] }, + Case { cat: "zh-unit", text: "覆盖率提高了 2.5 个百分点", sensitive: &[] }, + Case { cat: "zh-unit", text: "性能损失了百分之 3.5", sensitive: &[] }, + Case { cat: "zh-unit", text: "线宽是 0.15 微米", sensitive: &[] }, + Case { cat: "zh-unit", text: "芯片边长 8.5 毫米", sensitive: &[] }, + Case { cat: "zh-unit", text: "功耗降了 12.5%,别的没变", sensitive: &[] }, + // Audit round: measure-word duration and electrical units, each with + // an ADJACENT trigger — the shapes the first fix round still masked. + Case { cat: "zh-unit", text: "版本升级花了 3.5 个小时", sensitive: &[] }, + Case { cat: "zh-unit", text: "升级到新驱动后功耗 5.5 瓦", sensitive: &[] }, + // ── log timestamps (defect 1): must all release ────────────────────── + Case { cat: "timestamp", text: "[10:23:45.123] build started", sensitive: &[] }, + Case { cat: "timestamp", text: "[09:01:07.500] version check passed", sensitive: &[] }, + Case { cat: "timestamp", text: "日志停在 23:59:59.999 之后就没了", sensitive: &[] }, + Case { cat: "timestamp", text: "10:15:30.250 upgraded to the new license server", sensitive: &[] }, + Case { cat: "timestamp", text: "构建时间戳 [23:07:01.250] 已经记录", sensitive: &[] }, + // ── ASCII units (regression): must keep releasing ──────────────────── + Case { cat: "en-unit", text: "Elapsed: 12.345s, Memory: 4.2 GB", sensitive: &[] }, + Case { cat: "en-unit", text: "PrimeTime slack 0.5ns 违例", sensitive: &[] }, + Case { cat: "en-unit", text: "跑到 3.2GHz 依然稳定", sensitive: &[] }, + Case { cat: "en-unit", text: "工艺节点是 0.13um,良率还行", sensitive: &[] }, + Case { cat: "en-unit", text: "the build took 12.5 minutes", sensitive: &[] }, + Case { cat: "en-unit", text: "内存占用 1.5 GiB 左右", sensitive: &[] }, + // ── IPv4 / source locations (regression): must keep releasing ──────── + Case { cat: "locator", text: "服务器的 IP 地址是 10.2.255.1", sensitive: &[] }, + Case { cat: "locator", text: "see top.v:12.1 for the assignment", sensitive: &[] }, + Case { cat: "locator", text: "Virtuoso 主机 10.2.255.1 上跑的", sensitive: &[] }, + // ── fused version tokens (defect 3): must mask the WHOLE token ─────── + Case { cat: "fused-pos", text: "Virtuoso IC618 又崩了", sensitive: &["IC618"] }, + Case { cat: "fused-pos", text: "IC618 在新工艺角下不稳定", sensitive: &["IC618"] }, + Case { cat: "fused-pos", text: "版图工具用的是 ICADV12.3", sensitive: &["ICADV12.3"] }, + Case { cat: "fused-pos", text: "XCELIUM2309 的仿真结果对不上", sensitive: &["XCELIUM2309"] }, + Case { cat: "fused-pos", text: "仿真器升级到 XCELIUM2309 之后就好了", sensitive: &["XCELIUM2309"] }, + Case { cat: "fused-pos", text: "MMSIM151 装在新机器上了", sensitive: &["MMSIM151"] }, + Case { cat: "fused-pos", text: "综合用的 T-2022.03 有已知问题", sensitive: &["T-2022.03"] }, + Case { cat: "fused-pos", text: "回退到 E-2010.12-ICC-SP2 就不崩了", sensitive: &["E-2010.12-ICC-SP2"] }, + Case { cat: "fused-pos", text: "装的是 v16.12-s051_1 这个版本", sensitive: &["v16.12-s051_1"] }, + Case { cat: "fused-pos", text: "hotfix 20.09-s003 已经推送了", sensitive: &["20.09-s003"] }, + Case { cat: "fused-pos", text: "INNOVUS211 的时序报告在附件里", sensitive: &["INNOVUS211"] }, + // ── process nodes (defect 3 visibility): candidates, but released ──── + Case { cat: "node-neg", text: "7nm 工艺下功耗有点高", sensitive: &[] }, + Case { cat: "node-neg", text: "这个块是 N5 工艺的", sensitive: &[] }, + Case { cat: "node-neg", text: "N7 和 N3 都评估过了", sensitive: &[] }, + Case { cat: "node-neg", text: "先在 28nm 上验证流程", sensitive: &[] }, + // ── fullwidth digits (defect 3): must mask ──────────────────────────── + Case { cat: "fullwidth-pos", text: "版本是 １２．１,不要外传", sensitive: &["１２．１"] }, + Case { cat: "fullwidth-pos", text: "工具版本号 ２０２２．４ 见内部 wiki", sensitive: &["２０２２．４"] }, + // ── anchored positives (regression): must keep masking ─────────────── + Case { cat: "anchor-pos", text: "这个 EDA 软件的版本是 12.1", sensitive: &["12.1"] }, + Case { cat: "anchor-pos", text: "我们把仿真工具升级到 2022.4 之后速度快了很多", sensitive: &["2022.4"] }, + Case { cat: "anchor-pos", text: "Virtuoso IC6.1.8 出现了崩溃", sensitive: &["IC6.1.8"] }, + Case { cat: "anchor-pos", text: "we upgraded to 21.15 yesterday", sensitive: &["21.15"] }, + Case { cat: "anchor-pos", text: "PrimeTime 2022.03 跑不过时序", sensitive: &["2022.03"] }, + Case { cat: "anchor-pos", text: "Innovus 21.13 的这个 bug 已经确认了", sensitive: &["21.13"] }, + Case { cat: "anchor-pos", text: "版本回退到 20.11 才恢复正常", sensitive: &["20.11"] }, + // Verification-audit regression: 度 inside 度过 released this. + Case { cat: "anchor-pos", text: "版本 12.1 度过了回归测试", sensitive: &["12.1"] }, + Case { cat: "anchor-pos", text: "build 33.1 is broken on centos", sensitive: &["33.1"] }, + // ── model-band negatives (defect 2): decimals with non-software + // semantics — must release, and only the model can say so ────────── + Case { cat: "model-neg", text: "圆周率约等于 3.14159", sensitive: &[] }, + Case { cat: "model-neg", text: "今天美元汇率 7.23", sensitive: &[] }, + Case { cat: "model-neg", text: "孩子发烧到 38.5 了", sensitive: &[] }, + Case { cat: "model-neg", text: "合同日期是 2026.3.15", sensitive: &[] }, + Case { cat: "model-neg", text: "这批晶圆一共 12.5 万片", sensitive: &[] }, + Case { cat: "model-neg", text: "等了 2.5 个星期才排上机时", sensitive: &[] }, + Case { cat: "model-neg", text: "结温到了 85.5 度就降频", sensitive: &[] }, + Case { cat: "model-neg", text: "核心电压是 0.75 伏", sensitive: &[] }, + Case { cat: "model-neg", text: "客户满意度评分 9.5", sensitive: &[] }, + Case { cat: "model-neg", text: "第 3.2 节有详细说明", sensitive: &[] }, + Case { cat: "model-neg", text: "see section 4.1.2 for details", sensitive: &[] }, + Case { cat: "model-neg", text: "今天集群负载均值是 3.5", sensitive: &[] }, + // ── everyday identifiers (audit round): the fused-token relaxation + // makes these candidates for the FIRST time; they carry no unit + // and no anchor, so only the model can release them — the + // negative families files/hashes/tickets/standards and + // product/model names exist because these mis-masked without + // them ──────────────────────────────────────────────────────────── + Case { cat: "ident-neg", text: "把 report3.txt 发我一下", sensitive: &[] }, + Case { cat: "ident-neg", text: "模型是 gpt-4o 那个", sensitive: &[] }, + Case { cat: "ident-neg", text: "commit deadbeef123 部署上去了", sensitive: &[] }, + Case { cat: "ident-neg", text: "构建号 a1b2c3d4e5f6", sensitive: &[] }, + Case { cat: "ident-neg", text: "对应 issue 编号 GH-2048", sensitive: &[] }, + Case { cat: "ident-neg", text: "这块板子过了 802.11ac 认证", sensitive: &[] }, + // ── model-band positives: version mentions with NO lexical anchor — + // only the model can mask these ───────────────────────────────────── + Case { cat: "model-pos", text: "工具从 21.10 换到 21.12 就不崩了", sensitive: &["21.10", "21.12"] }, + Case { cat: "model-pos", text: "生产机上装的是 6.1.8", sensitive: &["6.1.8"] }, + Case { cat: "model-pos", text: "他们还在用 17.4,太老了", sensitive: &["17.4"] }, + Case { cat: "model-pos", text: "换成 2018.09 之后问题就消失了", sensitive: &["2018.09"] }, + Case { cat: "model-pos", text: "新装的 2022.4 跑不了旧工程", sensitive: &["2022.4"] }, + Case { cat: "model-pos", text: "12.1 和 13.0 都测过,后者稳定些", sensitive: &["12.1", "13.0"] }, + Case { cat: "model-pos", text: "装了 34.0 那个包就好了", sensitive: &["34.0"] }, + Case { cat: "model-pos", text: "那台机器上是 15.2", sensitive: &["15.2"] }, +]; + +/// Byte ranges of every occurrence of every sensitive substring. +fn sensitive_ranges(case: &Case) -> Vec> { + case.sensitive + .iter() + .flat_map(|s| case.text.match_indices(s).map(|(i, m)| i..i + m.len())) + .collect() +} + +fn overlaps(a: &Range, b: &Range) -> bool { + a.start < b.end && b.start < a.end +} + +/// The line's expected end-to-end output: every sensitive occurrence +/// rewritten, everything else byte-identical. +fn expected_output(case: &Case) -> String { + let mut ranges = sensitive_ranges(case); + ranges.sort_by_key(|r| r.start); + apply_masks(case.text, &ranges) +} + +/// Best 0/1 accuracy over all thresholds for `(score, is_positive)` +/// pairs, deciding `mask ⇔ score >= t`. Returns (best accuracy, one +/// threshold achieving it). +fn best_threshold_accuracy(items: &[(f32, bool)]) -> (f64, f32) { + let mut cuts: Vec = items.iter().map(|(s, _)| *s).collect(); + // The two degenerate classifiers bound the sweep: -inf = mask + // everything, +inf = release everything (bot-review finding: without + // the latter, an all-negative item set understates best accuracy). + cuts.push(f32::NEG_INFINITY); + cuts.push(f32::INFINITY); + let mut best = (0.0f64, f32::NEG_INFINITY); + for cut in cuts { + let correct = items.iter().filter(|(s, pos)| (*s >= cut) == *pos).count(); + let acc = correct as f64 / items.len() as f64; + if acc > best.0 { + best = (acc, cut); + } + } + best +} + +fn pct(part: usize, whole: usize) -> f64 { + if whole == 0 { + return 0.0; + } + 100.0 * part as f64 / whole as f64 +} + +/// The report instrument. Prints per-category and total stats; asserts +/// only the instrument's own consistency (replicated decisions must +/// reproduce the real pipeline output) so the SAME test measures the +/// before and after states without encoding either as a contract. +#[tokio::test] +#[ignore = "needs GUARDRAIL_LOCAL_MODEL_DIR with model.onnx + tokenizer.json"] +async fn adversarial_corpus_report() { + let Some(cfg) = LocalModelConfig::from_env() else { + return; + }; + let g = LocalModelGuardrail::load(&cfg).expect("model files present but load failed"); + + let mut n_candidates = 0usize; + let (mut rule_masked, mut rule_passed, mut model_band) = (0usize, 0usize, 0usize); + let (mut rule_masked_correct, mut rule_passed_correct) = (0usize, 0usize); + // Model-band scores in BOTH forms: `abs` — the old positive-only max + // cosine; `rel` — the shipped `max_pos − max_neg`. Same embeddings, + // so the comparison isolates the scoring FORM. + let mut model_items_abs: Vec<(f32, bool)> = Vec::new(); + let mut model_items_rel: Vec<(f32, bool)> = Vec::new(); + let (mut lines_correct, mut cand_correct) = (0usize, 0usize); + let mut wrong_lines: Vec<(&str, String, String)> = Vec::new(); + + for case in CORPUS { + let labels = sensitive_ranges(case); + let spans = g.finder.spans(case.text); + let mut hits: Vec> = Vec::new(); + for span in spans { + n_candidates += 1; + let positive = labels.iter().any(|l| overlaps(l, &span)); + let masked = match g.rules.decide(case.text, &span) { + RuleDecision::Mask => { + rule_masked += 1; + rule_masked_correct += usize::from(positive); + true + } + RuleDecision::Pass => { + rule_passed += 1; + rule_passed_correct += usize::from(!positive); + false + } + RuleDecision::Model => { + model_band += 1; + let window = + case.text[window_bounds(case.text, &span, WINDOW_CONTEXT_CHARS)].to_owned(); + let v = g.embed_window(window).await.expect("embed failed"); + model_items_abs.push((max_cosine(&g.prototypes.positive, &v), positive)); + let score = g.prototypes.score(&v); + model_items_rel.push((score, positive)); + score >= g.threshold + } + }; + cand_correct += usize::from(masked == positive); + if masked { + hits.push(span); + } + } + // Instrument consistency: the replication above must reproduce + // the real pipeline byte for byte. + hits.sort_by_key(|r| r.start); + let replicated = apply_masks(case.text, &hits); + let outcome = g.moderate_input_segments(&[case.text.to_owned()]).await; + let actual = outcome + .masked + .map_or_else(|| case.text.to_owned(), |m| m[0].clone()); + assert_eq!( + replicated, actual, + "instrument diverged from the pipeline on: {}", + case.text + ); + + let want = expected_output(case); + if actual == want { + lines_correct += 1; + } else { + wrong_lines.push((case.cat, case.text.to_owned(), actual)); + } + } + + println!( + "── corpus: {} lines, {} candidates", + CORPUS.len(), + n_candidates + ); + println!( + "rule-masked {rule_masked} (precision {:.1}%), rule-passed {rule_passed} ({} misreleased), model band {model_band} ({:.1}% of candidates)", + pct(rule_masked_correct, rule_masked), + rule_passed - rule_passed_correct, + pct(model_band, n_candidates), + ); + println!( + "candidate-level accuracy {:.1}% line-level accuracy {:.1}% ({}/{})", + pct(cand_correct, n_candidates), + pct(lines_correct, CORPUS.len()), + lines_correct, + CORPUS.len(), + ); + for (cat, text, actual) in &wrong_lines { + println!(" WRONG [{cat}] {text:?} -> {actual:?}"); + } + + let n_pos = model_items_rel.iter().filter(|(_, p)| *p).count(); + println!( + "── model band: {} items ({} pos / {} neg) shipped threshold {:.4}", + model_items_rel.len(), + n_pos, + model_items_rel.len() - n_pos, + g.threshold, + ); + for (name, items) in [("abs", &model_items_abs), ("rel", &model_items_rel)] { + let (acc, cut) = best_threshold_accuracy(items); + let min_pos = items + .iter() + .filter(|(_, p)| *p) + .map(|(s, _)| *s) + .fold(f32::INFINITY, f32::min); + let max_neg = items + .iter() + .filter(|(_, p)| !*p) + .map(|(s, _)| *s) + .fold(f32::NEG_INFINITY, f32::max); + println!( + " form {name}: margin (min pos − max neg) {:+.4} best-threshold accuracy {:.1}% at {:.4}", + min_pos - max_neg, + 100.0 * acc, + cut, + ); + } + for ((abs, p), (rel, _)) in model_items_abs.iter().zip(&model_items_rel) { + println!( + " {} abs {abs:.4} rel {rel:+.4}", + if *p { "POS" } else { "NEG" } + ); + } + + // Quality floor, asserted LAST so a regression still prints the full + // report above (audit finding: a report that only prints lets sample + // or model drift collapse the numbers silently while CI stays + // green). The rule layer must be EXACT on this corpus — a rule + // decision never consults the model, so a wrong one is an + // unconditional mis-rewrite (mask side) or a silent leak (pass + // side). The end-to-end floor allows exactly the two disclosed + // model-band misses. + assert_eq!( + rule_masked_correct, rule_masked, + "rule-mask precision must stay 100% on the corpus" + ); + assert_eq!( + rule_passed_correct, rule_passed, + "rule-pass must not release a labeled positive" + ); + assert!( + lines_correct >= CORPUS.len() - 2, + "line accuracy fell below the pinned floor: {lines_correct}/{}", + CORPUS.len() + ); +} diff --git a/crates/aisix-guardrails/src/local_model/rules.rs b/crates/aisix-guardrails/src/local_model/rules.rs index 10270bae..4a52d9f1 100644 --- a/crates/aisix-guardrails/src/local_model/rules.rs +++ b/crates/aisix-guardrails/src/local_model/rules.rs @@ -46,9 +46,11 @@ //! Threat-model boundary, inherited from the design issue's "只能防无意 //! 泄漏" line and inherent to score-subtraction DLP: a sender (or a //! hostile upstream, on the output side) can defeat the rule layer by -//! FORMATTING — appending a unit-looking suffix (`升级到 2022.4s`), a -//! `:digit` tail, or fullwidth digits that never become candidates. The -//! layer scores accidental phrasing, not adversarial encoding. +//! FORMATTING — appending a unit-looking suffix (`升级到 2022.4s`) or a +//! `:digit` tail. The layer scores accidental phrasing, not adversarial +//! encoding. (Fullwidth digits left this list: layer ① now candidates +//! them, because they occur in ACCIDENTAL Chinese-IME phrasing, which is +//! in scope.) //! //! Everything here is pure text work — no model, no I/O — so the layer is //! unit-testable standalone, which is also how the "rules alone" halves @@ -98,21 +100,59 @@ const ZH_TRIGGERS: &[&str] = &["版本号", "版本", "升级到", "回退到"]; const EN_TRIGGER_PATTERN: &str = r"(?i)\b(?:version|release|build|upgraded?\s+to)\b"; /// EDA tool names — the strongest anchors (`Virtuoso IC6.1.8` needs no -/// model). Word-bounded, case-insensitive (`vcs` / `VCS`). -const TOOL_PATTERN: &str = r"(?i)\b(?:virtuoso|calibre|vcs|innovus|icc2|primetime)\b"; +/// model). Word-bounded on the left, case-insensitive (`vcs` / `VCS`). +/// On the right, either a word boundary or a DIGIT continuation: real +/// corpora fuse the tool name straight into the version (`INNOVUS211`), +/// and `\b` never fires between two word chars, so the plain `\b` form +/// was blind to exactly the fused tokens layer ① now produces. The +/// digit alternative consumes one char, which only matters to the gap +/// computation by one char inside an already-overlapping span. +const TOOL_PATTERN: &str = r"(?i)\b(?:virtuoso|calibre|vcs|innovus|icc2|primetime)(?:[0-9]|\b)"; /// Negative: a measurement unit right after the span (`12.345s`, -/// `4.2 GB`, `0.13um`, `99.9%`, `0.5ns`). Beyond the design brief's -/// minimum list, this covers the full timing-unit family (`us/ns/ps/fs`, -/// the `Hz` family, spelled-out durations) because the driving corpus — -/// STA/timing logs — is ns/ps-dense, and a unit the list misses next to -/// a tool name would RULE-MASK a slack value (audit finding on this PR). -/// Anchored to the span end with optional whitespace; letter units must -/// not continue into a longer word (`12.1 subsystem` is NOT an `s` hit), -/// checked with an explicit ASCII-alnum guard rather than `\b` because -/// the regex crate's Unicode `\b` treats a following CJK char as a word -/// char. -const UNIT_SUFFIX_PATTERN: &str = r"^\s*(?:%|(?i:ms|us|ns|ps|fs|s|secs?|seconds?|mins?|minutes?|hours?|[kmgt]i?b|um|nm|[kmg]?hz)(?:[^0-9A-Za-z]|$))"; +/// `4.2 GB`, `0.13um`, `99.9%`, `0.5ns`, `0.5 纳秒`, `3.5 小时`). Beyond +/// the design brief's minimum list, this covers the full timing-unit +/// family (`us/ns/ps/fs`, the `Hz` family, spelled-out durations) +/// because the driving corpus — STA/timing logs — is ns/ps-dense, and a +/// unit the list misses next to a tool name would RULE-MASK a slack +/// value (audit finding on PR #1005). The CHINESE unit vocabulary is the +/// adversarial-corpus finding this PR fixes: the customer's dominant +/// corpus is Chinese logs, and under the double threshold a rule-mask +/// never consults the model, so `0.5ns` released while `0.5 纳秒` was +/// rewritten. Covered families: durations (纳秒→天), lengths (纳米/微米/ +/// 毫米), byte sizes (兆/吉/太字节), the Hz family (千/兆/吉赫兹), and +/// percentages (`％`, `个百分点`; the `百分之` PREFIX form is +/// [`CN_PERCENT_PREFIX_PATTERN`]). +/// Anchored to the span end with optional whitespace; ASCII letter units +/// must not continue into a longer word (`12.1 subsystem` is NOT an `s` +/// hit), checked with an explicit ASCII-alnum guard rather than `\b` +/// because the regex crate's Unicode `\b` treats a following CJK char as +/// a word char. Chinese units are substring-matched (no word boundaries +/// in CJK); durations also carry the measure-word form (`3.5 个小时`, +/// `2.5 个星期` — the audit found the bare forms alone still rule-masked +/// next to a trigger), and thermal/electrical units (`度`, `伏特`, +/// `瓦特`, `安培`) cover the power/temperature lines EDA logs are full +/// of. Single-char units with heavy compound ambiguity are deliberately +/// excluded — `分` (points vs minutes) and bare `安` (`12.1 安装之后` +/// would systematically RELEASE real versions next to the extremely +/// common 安装), and `度` must not continue into `度过` (verification +/// audit: `版本 12.1 度过了回归测试` released a real version; the +/// negated-char form costs one lookahead-free char, harmless for +/// `is_match`) — and the residual compound risk of the kept ones +/// (`12.1 天线`) is accepted: a wrong release is fail-open, a wrong +/// rewrite corrupts content. +const UNIT_SUFFIX_PATTERN: &str = r"^\s*(?:%|％|(?i:ms|us|ns|ps|fs|s|secs?|seconds?|mins?|minutes?|hours?|[kmgt]i?b|um|nm|[kmg]?hz)(?:[^0-9A-Za-z]|$)|纳秒|微秒|毫秒|秒|分钟|个?(?:小时|钟头|星期|月)|天|纳米|微米|毫米|[兆吉太]字节|[千兆吉]?赫兹|[千兆吉]赫|个?百分点|摄氏度|度(?:[^过]|$)|伏特?|瓦特?|安培|毫安)"; + +/// Negative: the span itself ENDS in an ASCII measurement unit. Fused +/// candidate tokens (`12.345s`, `3.2GHz`, `7nm`, `0.13um` as ONE span) +/// carry the unit evidence INSIDE the span rather than after it, so the +/// suffix check above never sees it. Same class as +/// [`UNIT_SUFFIX_PATTERN`] — either placement counts once. +const FUSED_UNIT_SUFFIX_PATTERN: &str = r"(?i)[0-9](?:%|ms|us|ns|ps|fs|s|secs?|seconds?|mins?|minutes?|hours?|[kmgt]i?b|um|nm|[kmg]?hz)$"; + +/// Negative: the Chinese percentage PREFIX form (`百分之 3.5`) — the +/// percent evidence precedes the number. Same class as the unit suffix. +const CN_PERCENT_PREFIX_PATTERN: &str = r"百分之\s*$"; /// Negative: the span itself is IPv4-shaped (`10.2.255.1`). Shape only — /// no octet range check, matching how DLP engines treat dotted quads. @@ -131,6 +171,33 @@ const IPV4_PATTERN: &str = r"^\d{1,3}(?:\.\d{1,3}){3}$"; const FILE_COLON_PREFIX_PATTERN: &str = r"[\w.-]+\.[A-Za-z0-9]+:$"; const COLON_DIGIT_SUFFIX_PATTERN: &str = r"^:\d"; +/// Negative: time-of-day context, same class as the source location — +/// the span completes an `HH:MM:SS.mmm` clock reading. Log lines open +/// with these (`[10:23:45.123] build started`), and the trigger word +/// right after the bracket (`build`, `version`) is ADJACENT by distance, +/// so without this class the whole timestamp family rule-masks — the +/// adversarial-corpus finding. ASCII and fullwidth colons both count. +const TIME_OF_DAY_PREFIX_PATTERN: &str = r"[0-9]{1,2}[:：][0-9]{1,2}[:：]$"; + +/// Negative: the span is FILENAME-shaped — a fused token ending in a +/// dot plus a 2–4 letter extension (`report3.txt`, `setup2.cfg`). +/// Audit finding: everyday filenames are fused-token candidates now, +/// and `GH-2048`-class identifiers sit too close to the weakest +/// anchor-free version positives for the embedding to separate — but a +/// trailing alphabetic extension is decisive LEXICAL evidence, which is +/// this layer's job, not the model's. Version tokens never end in a +/// dot-plus-letters segment (`802.11ac`'s last segment starts with +/// digits; `…-SP2` has no dot before the letters), so the shape is +/// precise. Same class as the source-location patterns. +const FILE_EXTENSION_SUFFIX_PATTERN: &str = r"(?i)\.[a-z]{2,4}$"; + +/// Negative: an identifier tag right before the span — `编号 GH-2048`, +/// `编号: JIRA-1024`, optionally through `是/为`. `版本编号` is carved +/// out (the char before `编号` must not be `本`): that compound means +/// "version number" and must keep masking. Same class as the source +/// location — a tagged identifier is a locator, not a version. +const ID_TAG_PREFIX_PATTERN: &str = r"(?:^|[^本])编号[:：]?(?:是|为)?\s*$"; + /// What layer ② decided for one candidate span. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum RuleDecision { @@ -150,9 +217,14 @@ pub(super) struct RuleScorer { en_trigger: Regex, tool: Regex, unit_suffix: Regex, + fused_unit_suffix: Regex, + cn_percent_prefix: Regex, ipv4: Regex, file_colon_prefix: Regex, colon_digit_suffix: Regex, + time_of_day_prefix: Regex, + file_extension_suffix: Regex, + id_tag_prefix: Regex, } impl RuleScorer { @@ -167,9 +239,14 @@ impl RuleScorer { en_trigger: compile(EN_TRIGGER_PATTERN), tool: compile(TOOL_PATTERN), unit_suffix: compile(UNIT_SUFFIX_PATTERN), + fused_unit_suffix: compile(FUSED_UNIT_SUFFIX_PATTERN), + cn_percent_prefix: compile(CN_PERCENT_PREFIX_PATTERN), ipv4: compile(IPV4_PATTERN), file_colon_prefix: compile(FILE_COLON_PREFIX_PATTERN), colon_digit_suffix: compile(COLON_DIGIT_SUFFIX_PATTERN), + time_of_day_prefix: compile(TIME_OF_DAY_PREFIX_PATTERN), + file_extension_suffix: compile(FILE_EXTENSION_SUFFIX_PATTERN), + id_tag_prefix: compile(ID_TAG_PREFIX_PATTERN), } } @@ -228,8 +305,13 @@ impl RuleScorer { } // Negative classes: span-local shape checks, independent of the - // proximity window. - if self.unit_suffix.is_match(&text[span.end..]) { + // proximity window. Unit evidence counts once no matter where it + // sits (after the span, fused into its tail, or the Chinese + // percent prefix before it). + if self.unit_suffix.is_match(&text[span.end..]) + || self.fused_unit_suffix.is_match(&text[span.clone()]) + || self.cn_percent_prefix.is_match(&text[..span.start]) + { score += NEGATIVE_CLASS_SCORE; } if self.ipv4.is_match(&text[span.clone()]) { @@ -237,6 +319,9 @@ impl RuleScorer { } if self.file_colon_prefix.is_match(&text[..span.start]) || self.colon_digit_suffix.is_match(&text[span.end..]) + || self.time_of_day_prefix.is_match(&text[..span.start]) + || self.file_extension_suffix.is_match(&text[span.clone()]) + || self.id_tag_prefix.is_match(&text[..span.start]) { score += NEGATIVE_CLASS_SCORE; } @@ -270,13 +355,13 @@ fn gap_chars(text: &str, span: &Range, hotword: &Range) -> usize { #[cfg(test)] mod tests { - use super::super::{candidate_spans, CANDIDATE_PATTERN}; + use super::super::CandidateFinder; use super::*; fn decisions(text: &str) -> Vec<(String, RuleDecision)> { let scorer = RuleScorer::new(DEFAULT_PROXIMITY_CHARS); - let re = Regex::new(CANDIDATE_PATTERN).unwrap(); - candidate_spans(&re, text) + CandidateFinder::new() + .spans(text) .into_iter() .map(|s| (text[s.clone()].to_owned(), scorer.decide(text, &s))) .collect() @@ -309,10 +394,12 @@ mod tests { #[test] fn hard_negatives_pass_by_rules_alone() { + // `12.345s` is now ONE fused candidate (unit evidence inside the + // span); the decision is unchanged. assert_eq!( decisions("Elapsed: 12.345s, Memory: 4.2 GB"), vec![ - ("12.345".to_owned(), RuleDecision::Pass), + ("12.345s".to_owned(), RuleDecision::Pass), ("4.2".to_owned(), RuleDecision::Pass), ] ); @@ -369,8 +456,7 @@ mod tests { // clipped-edge guard drops the match and the candidate stays in // the model band. let text = "big conversion 3.5 result"; - let re = Regex::new(CANDIDATE_PATTERN).unwrap(); - let span = &candidate_spans(&re, text)[0]; + let span = &CandidateFinder::new().spans(text)[0]; let tight = RuleScorer::new(8); assert_eq!(tight.decide(text, span), RuleDecision::Model); } @@ -410,6 +496,162 @@ mod tests { assert_eq!(only("版本是 12.1 seconds 之外的话题"), RuleDecision::Pass); } + // ── the Chinese-unit defect class (adversarial-corpus finding): + // `0.5ns` released while `0.5 纳秒` rewrote. One assertion per + // unit family the brief names; the duration rows carry a trigger + // hotword nearby, so the negative must OUTWEIGH it. ───────────── + + #[test] + fn chinese_duration_units_are_negative_evidence() { + assert_eq!( + only("版本升级后这条路径 slack 变成 0.5 纳秒"), + RuleDecision::Pass + ); + assert_eq!(only("中断响应时间 12.5 微秒"), RuleDecision::Pass); + assert_eq!(only("升级到新内核后延迟降到 3.5 毫秒"), RuleDecision::Pass); + assert_eq!(only("整个 build 花了 45.5 秒"), RuleDecision::Pass); + assert_eq!(only("全量回归跑了 90.5 分钟"), RuleDecision::Pass); + assert_eq!(only("full chip 综合要 3.5 小时"), RuleDecision::Pass); + assert_eq!(only("数据准备还要 2.5 天"), RuleDecision::Pass); + } + + #[test] + fn measure_word_durations_are_negative_evidence() { + // The measure-word (`个`) forms, each next to a trigger — the + // audit's finding: the bare-unit list alone still rule-masked + // these. + assert_eq!(only("版本升级花了 3.5 个小时"), RuleDecision::Pass); + assert_eq!(only("升级到新机器后等了 1.5 个钟头"), RuleDecision::Pass); + assert_eq!(only("等了 2.5 个星期才排上机时"), RuleDecision::Pass); + assert_eq!(only("整个项目走了 4.5 个月"), RuleDecision::Pass); + } + + #[test] + fn thermal_and_electrical_units_are_negative_evidence() { + assert_eq!(only("升级到新驱动后功耗 5.5 瓦"), RuleDecision::Pass); + assert_eq!(only("结温到了 85.5 度就降频"), RuleDecision::Pass); + assert_eq!(only("外壳温度 42.5 摄氏度"), RuleDecision::Pass); + assert_eq!(only("核心电压是 0.75 伏"), RuleDecision::Pass); + assert_eq!(only("满载电流 1.8 安培"), RuleDecision::Pass); + // Bare 安 is deliberately NOT a unit: 安装 right after a version + // must not release it (安装 is everywhere in the driving corpus). + assert_eq!(only("版本是 12.1 安装之后报错"), RuleDecision::Mask); + // 度 must not fire inside 度过 (verification-audit regression: + // this real version released as a "degrees" reading). + assert_eq!(only("版本 12.1 度过了回归测试"), RuleDecision::Mask); + } + + #[test] + fn chinese_size_and_frequency_units_are_negative_evidence() { + assert_eq!(only("日志文件有 128.5 兆字节"), RuleDecision::Pass); + assert_eq!(only("内存峰值到了 4.2 吉字节"), RuleDecision::Pass); + assert_eq!(only("波形数据一共 1.5 太字节"), RuleDecision::Pass); + assert_eq!(only("主频跑到 3.2 吉赫兹"), RuleDecision::Pass); + assert_eq!(only("时钟是 800.5 兆赫兹"), RuleDecision::Pass); + assert_eq!(only("采样率 44.1 千赫兹"), RuleDecision::Pass); + assert_eq!(only("线宽是 0.15 微米"), RuleDecision::Pass); + assert_eq!(only("芯片边长 8.5 毫米"), RuleDecision::Pass); + } + + #[test] + fn percent_forms_are_negative_evidence() { + assert_eq!(only("覆盖率提高了 2.5 个百分点"), RuleDecision::Pass); + assert_eq!(only("性能损失了百分之 3.5"), RuleDecision::Pass); + assert_eq!(only("良率到了 98.5％"), RuleDecision::Pass); + assert_eq!(only("功耗降了 12.5%,别的没变"), RuleDecision::Pass); + } + + #[test] + fn log_timestamps_release_even_next_to_triggers() { + // The trigger right after the bracket is ADJACENT by distance — + // the clock context must win (adversarial-corpus finding: the + // whole `[HH:MM:SS.mmm]` family rule-masked). + assert_eq!(only("[10:23:45.123] build started"), RuleDecision::Pass); + assert_eq!( + only("[09:01:07.500] version check passed"), + RuleDecision::Pass + ); + assert_eq!( + only("10:15:30.250 upgraded to the new license server"), + RuleDecision::Pass + ); + assert_eq!( + only("构建时间戳 [23:07:01.250] 已经记录"), + RuleDecision::Pass + ); + } + + // ── fused-token candidates (layer-① relaxation) through the rules ── + + #[test] + fn fused_tokens_ending_in_units_release() { + // One fused span each; the unit evidence is INSIDE the span. + assert_eq!(only("7nm 工艺下功耗有点高"), RuleDecision::Pass); + assert_eq!(only("先在 28nm 上验证流程"), RuleDecision::Pass); + assert_eq!(only("跑到 3.2GHz 依然稳定"), RuleDecision::Pass); + } + + #[test] + fn fused_tool_prefix_is_decisive() { + // The tool name fused straight into the version: the digit + // continuation in TOOL_PATTERN makes the anchor visible. + assert_eq!(only("INNOVUS211 的时序报告在附件里"), RuleDecision::Mask); + } + + #[test] + fn fused_tokens_with_adjacent_triggers_mask() { + assert_eq!( + only("仿真器升级到 XCELIUM2309 之后就好了"), + RuleDecision::Mask + ); + assert_eq!( + only("回退到 E-2010.12-ICC-SP2 就不崩了"), + RuleDecision::Mask + ); + assert_eq!(only("装的是 v16.12-s051_1 这个版本"), RuleDecision::Mask); + assert_eq!(only("Virtuoso IC618 又崩了"), RuleDecision::Mask); + } + + #[test] + fn filename_shaped_tokens_release() { + // Trailing dot-plus-letters extension is decisive lexical + // evidence (audit finding: `report3.txt` mis-masked in the + // model band — but it never needed the model). + assert_eq!(only("把 report3.txt 发我一下"), RuleDecision::Pass); + assert_eq!(only("参数都写在 sim7.cfg 里面"), RuleDecision::Pass); + // Decisive even against an ADJACENT trigger: the extension must + // outweigh 版本 (bot-review example). + assert_eq!(only("版本是 build_2023.log 里说的那个"), RuleDecision::Pass); + } + + #[test] + fn id_tag_prefix_releases_tagged_identifiers() { + assert_eq!(only("对应 issue 编号 GH-2048"), RuleDecision::Pass); + assert_eq!(only("工单编号: AB-3072 已建好"), RuleDecision::Pass); + // Fullwidth colon after the tag (bot-review finding: the class + // had two ASCII colons and no fullwidth one). + assert_eq!(only("工单编号：AB-3072 已建好"), RuleDecision::Pass); + // 版本编号 is carved out — it means "version number" and must + // keep masking (the char before 编号 is 本). + assert_eq!(only("版本编号 12.1 别外发"), RuleDecision::Mask); + } + + #[test] + fn bare_fused_tokens_stay_in_the_model_band() { + // No lexical anchor either way: exactly what layer ③ exists for. + assert_eq!(only("XCELIUM2309 的仿真结果对不上"), RuleDecision::Model); + assert_eq!(only("这个块是 N5 工艺的"), RuleDecision::Model); + } + + #[test] + fn fullwidth_version_with_adjacent_trigger_masks() { + assert_eq!(only("版本是 １２．１,不要外传"), RuleDecision::Mask); + assert_eq!( + only("工具版本号 ２０２２．４ 见内部 wiki"), + RuleDecision::Mask + ); + } + #[test] fn proximity_window_bounds_the_hotword_search() { // Same sentence, tool name pushed outside a tiny window: the @@ -417,8 +659,7 @@ mod tests { let text = "Innovus 的运行日志我贴在下面了,请帮忙看看统计值 21.12"; let tight = RuleScorer::new(4); let wide = RuleScorer::new(DEFAULT_PROXIMITY_CHARS); - let re = Regex::new(CANDIDATE_PATTERN).unwrap(); - let span = &candidate_spans(&re, text)[0]; + let span = &CandidateFinder::new().spans(text)[0]; assert_eq!(tight.decide(text, span), RuleDecision::Model); assert_eq!(wide.decide(text, span), RuleDecision::Model); // At window distance the tool is weak (+1) evidence, not a mask. diff --git a/tests/e2e/src/cases/guardrail-local-model-e2e.test.ts b/tests/e2e/src/cases/guardrail-local-model-e2e.test.ts index 3e526b40..f0facda2 100644 --- a/tests/e2e/src/cases/guardrail-local-model-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-local-model-e2e.test.ts @@ -52,16 +52,22 @@ const SENSITIVE = "这个 EDA 软件的版本是 12.1"; const MASKED = "这个 EDA 软件的版本是 ***"; // The layer-② acceptance matrix in one message: both hard positives, the -// hard negatives, and a bare number with no lexical evidence at all (the -// model band — this candidate pays a real in-process inference). +// hard negatives (including the Chinese-unit and timestamp classes the +// defect-fix round added), and a bare number with no lexical evidence at +// all (the model band — this candidate pays a real in-process inference). +// `Virtuoso IC6.1.8` masks as the WHOLE fused token now: layer ① sees +// fused version tokens since the defect-fix round, so the `IC` identity +// prefix no longer survives (the MVP-era output was `Virtuoso IC***`). const MIXED = "我们把仿真工具升级到 2022.4 之后,Virtuoso IC6.1.8 反而开始频繁崩溃," + "完整的运行日志我贴在下面了,麻烦帮忙看看到底是哪一步出了问题: " + + "[10:23:45.123] build started, 整个 build 花了 45.5 秒, " + "Elapsed: 12.345s, Memory: 4.2 GB, 服务器 IP 是 10.2.255.1, " + "另外圆周率约等于 3.14159"; const MIXED_MASKED = - "我们把仿真工具升级到 *** 之后,Virtuoso IC*** 反而开始频繁崩溃," + + "我们把仿真工具升级到 *** 之后,Virtuoso *** 反而开始频繁崩溃," + "完整的运行日志我贴在下面了,麻烦帮忙看看到底是哪一步出了问题: " + + "[10:23:45.123] build started, 整个 build 花了 45.5 秒, " + "Elapsed: 12.345s, Memory: 4.2 GB, 服务器 IP 是 10.2.255.1, " + "另外圆周率约等于 3.14159";