From 3f5991555755ee0430302f75bab328fcae061ffa Mon Sep 17 00:00:00 2001 From: dd3ok <15044917+dd3ok@users.noreply.github.com> Date: Sun, 5 Jul 2026 23:41:07 +0900 Subject: [PATCH 1/2] Make briefprint explicit-use --- CHANGELOG.md | 9 +++ README.ko.md | 29 ++++++-- README.md | 29 ++++++-- VALIDATION.md | 6 +- agents/openai.yaml | 14 ++-- docs/agent-skill-installation.md | 2 + evals/model_invocation_benchmark_cases.json | 22 ++++-- evals/trigger_eval_cases.json | 40 +++++++---- references/best-practices.md | 6 +- scripts/validate_skill.py | 68 ++++++++++++------ skills/briefprint/SKILL.md | 5 +- skills/briefprint/agents/openai.yaml | 14 ++-- .../briefprint/references/best-practices.md | 4 ++ tests/test_docs.py | 39 ++++++++++ tests/test_installable_skill_bundle.py | 3 +- tests/test_skill_metadata.py | 18 ++++- tests/test_trigger_evals.py | 71 ++++++++++++++++++- 17 files changed, 294 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d64372..7a3f09e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Changed + +- Made Briefprint explicit-use in agent metadata so ordinary one-off summaries do not automatically invoke the cached briefing skill. +- Replaced broad natural-language trigger examples with `$briefprint` invocation examples. +- Added `disable-model-invocation: true` to the installable skill metadata for hosts that support manual-only skill invocation. +- Documented the boundary between the static agent skill bundle and Briefprint runtime caches under `--cache-dir`. + ## 0.4.0 - 2026-07-05 ### Added diff --git a/README.ko.md b/README.ko.md index 628668a..c8e480a 100644 --- a/README.ko.md +++ b/README.ko.md @@ -110,7 +110,9 @@ skills/briefprint/ repo 루트를 에이전트 스킬로 설치하지 마세요. 루트 복사 방식은 테스트, 문서, 예제, eval, 소스 코드, 검증 스크립트까지 같이 복사할 수 있습니다. 자세한 내용은 [docs/agent-skill-installation.md](docs/agent-skill-installation.md)를 보세요. -Claude.ai description variant: Cache structured briefings for supplied documents, notes, logs, tickets, reports, JSON/XML, or transcripts. Use for repeated summaries, rerendering, digests, actions, risks, or metrics. +Briefprint는 명시 호출형 스킬입니다. Codex에서는 `$briefprint`, Claude Code에서는 `/briefprint`를 사용하고, 그 밖의 호스트에서는 해당 호스트의 명시적 스킬 호출 또는 CLI를 사용하세요. 일반적인 일회성 요약이 자동으로 이 스킬을 트리거해서는 안 됩니다. + +Claude.ai description variant: Explicit-use cached briefings for supplied documents, notes, logs, tickets, reports, JSON/XML, or transcripts. Use for repeated summaries, rerendering, digests, actions, risks, or metrics. ## 벤치마크 결과 @@ -191,19 +193,32 @@ JSON, XML, HTML, `DocumentInput.source` 안에 들어있는 URL 메타데이터 권장 기본값: -- 반복 문서에는 `document_summaries`를 TTL 캐시로 유지 -- 템플릿 렌더링은 싸기 때문에 `rendered_outputs`는 짧게 유지 +- repo-local 캐시는 `.cache/briefprint` 아래에 두고 Git에는 올리지 않음 +- 반복 문서에는 `document_summaries`를 짧은 TTL 캐시로 유지 +- 템플릿 렌더링은 싸기 때문에 `rendered_outputs`는 더 짧게 유지 +- 평소 실행 때 `--prune-on-start`로 정리 - 민감한 일회성 문서는 `ephemeral` 사용 ```bash python -m document_briefing_cache.cli run \ --input examples/mixed_documents.json \ + --cache-dir .cache/briefprint \ --cache-policy ttl \ - --document-ttl 30d \ + --document-ttl 7d \ --output-ttl 24h \ --prune-on-start ``` +Briefprint는 백그라운드 cleanup daemon을 돌리지 않습니다. TTL은 캐시 항목을 expired로 표시하는 값이고, 실제 파일 삭제는 `cache prune`, `--prune-on-start`, `--prune-on-exit`, delete-on-exit 정책 중 하나가 실행될 때 일어납니다. 이는 로컬 개발 도구에서 흔한 방식입니다. 생성 캐시는 disposable로 보고 Git에서 제외하며, 명시 명령이나 도구 실행 시점에 opportunistic하게 정리합니다. CI 캐시는 예외에 가깝고, GitHub Actions 같은 플랫폼은 자체 last-access/size eviction 정책을 적용합니다. + +보존 기간을 길게 잡는 경우는 의도적으로 persistent 캐시를 쓸 때로 제한하세요. + +- project-local 기본: `.cache/briefprint`, `--document-ttl 7d`, `--output-ttl 24h`, `--prune-on-start` +- 민감한 일회성 작업: `--sensitive` +- 장기 공유 캐시: 명시적 `--cache-policy persistent` 또는 더 긴 `--document-ttl` + +스킬 번들과 runtime cache는 별개입니다. 스킬 번들은 설치 시점의 정적 가이드이고, runtime cache는 `--cache-dir` 아래에 생기며 Briefprint CLI/runtime이 관리합니다. 에이전트 스킬을 설치, 업데이트, 삭제해도 runtime cache가 migrate, prune, delete 되지는 않습니다. 생성된 문서 상태에 대해 모든 agent-skill host가 공통으로 제공하는 portable automatic eviction 계약은 없으므로, 설치된 스킬 디렉터리 안에 문서 캐시를 쓰지 마세요. + 민감 문서: ```bash @@ -227,9 +242,9 @@ python -m document_briefing_cache.cli run \ 캐시 관리: ```bash -python -m document_briefing_cache.cli cache stats --cache-dir .cache --json -python -m document_briefing_cache.cli cache prune --cache-dir .cache --older-than 30d --dry-run --json -python -m document_briefing_cache.cli cache clear --cache-dir .cache --layer rendered_outputs --yes +python -m document_briefing_cache.cli cache stats --cache-dir .cache/briefprint --json +python -m document_briefing_cache.cli cache prune --cache-dir .cache/briefprint --older-than 7d --dry-run --json +python -m document_briefing_cache.cli cache clear --cache-dir .cache/briefprint --layer rendered_outputs --yes ``` ## LLM 요약기 diff --git a/README.md b/README.md index 30d08dd..f4ffd08 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,9 @@ skills/briefprint/ Do not install the repository root as an agent skill. Root-copy installers can include tests, docs, examples, evals, source code, and validation scripts. See [docs/agent-skill-installation.md](docs/agent-skill-installation.md). -Claude.ai description variant: Cache structured briefings for supplied documents, notes, logs, tickets, reports, JSON/XML, or transcripts. Use for repeated summaries, rerendering, digests, actions, risks, or metrics. +Briefprint is explicit-use by design. For Codex, use `$briefprint`; for Claude Code, use `/briefprint`; for other hosts, use the host's explicit skill invocation or the CLI when you want reusable cached document briefings. Ordinary one-off summaries should not trigger the skill automatically. + +Claude.ai description variant: Explicit-use cached briefings for supplied documents, notes, logs, tickets, reports, JSON/XML, or transcripts. Use for repeated summaries, rerendering, digests, actions, risks, or metrics. ## Benchmark Receipts @@ -191,19 +193,32 @@ URL-bearing metadata inside JSON, XML, HTML, or `DocumentInput.source` is preser Recommended defaults: -- keep `document_summaries` as a TTL cache for repeated documents, -- keep `rendered_outputs` short-lived because template rendering is cheap, +- keep repo-local cache under `.cache/briefprint` and keep it out of version control, +- keep `document_summaries` as a short TTL cache for repeated documents, +- keep `rendered_outputs` shorter-lived because template rendering is cheap, +- run pruning during normal use with `--prune-on-start`, - use `ephemeral` for sensitive one-off work. ```bash python -m document_briefing_cache.cli run \ --input examples/mixed_documents.json \ + --cache-dir .cache/briefprint \ --cache-policy ttl \ - --document-ttl 30d \ + --document-ttl 7d \ --output-ttl 24h \ --prune-on-start ``` +Briefprint does not run a background cleanup daemon. TTL values mark cache entries as expired; physical deletion happens when you run `cache prune`, enable `--prune-on-start` or `--prune-on-exit`, or use a delete-on-exit policy. This follows the common local-tooling pattern: generated caches are disposable, ignored by Git, and cleaned explicitly or opportunistically during tool runs. CI caches are the main exception; platforms such as GitHub Actions apply their own last-access and size eviction policies. + +Use longer retention only when the cache is intentionally persistent: + +- project-local default: `.cache/briefprint`, `--document-ttl 7d`, `--output-ttl 24h`, `--prune-on-start`, +- sensitive one-off work: `--sensitive`, +- long-lived shared cache: explicit `--cache-policy persistent` or a longer `--document-ttl`. + +Skill bundle and runtime cache are separate. The skill bundle is static install-time guidance; the runtime cache lives under `--cache-dir` and is owned by the Briefprint CLI/runtime. Installing, updating, or removing the agent skill does not migrate, prune, or delete runtime caches. No portable agent-skill host contract currently provides automatic eviction for generated document state, so do not write document caches into the installed skill directory. + For sensitive documents: ```bash @@ -227,9 +242,9 @@ For sensitive documents, the safe default is no persistent cache. `--sensitive` Cache maintenance: ```bash -python -m document_briefing_cache.cli cache stats --cache-dir .cache --json -python -m document_briefing_cache.cli cache prune --cache-dir .cache --older-than 30d --dry-run --json -python -m document_briefing_cache.cli cache clear --cache-dir .cache --layer rendered_outputs --yes +python -m document_briefing_cache.cli cache stats --cache-dir .cache/briefprint --json +python -m document_briefing_cache.cli cache prune --cache-dir .cache/briefprint --older-than 7d --dry-run --json +python -m document_briefing_cache.cli cache clear --cache-dir .cache/briefprint --layer rendered_outputs --yes ``` ## LLM Summarizer diff --git a/VALIDATION.md b/VALIDATION.md index e9bd15f..4265a47 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -1,6 +1,6 @@ # Validation -Last verified: 2026-07-04 +Last verified: 2026-07-05 ## Environment @@ -20,8 +20,8 @@ ruff check . ## Observed Results -- `python -m pytest -q -p no:cacheprovider`: `164 passed, 2 skipped` -- `python scripts\validate_skill.py --run-evals`: `OK: briefprint skill repository validated (24 test files, 9 eval cases, 9 trigger cases, 4 model benchmark cases)` +- `python -m pytest -q -p no:cacheprovider`: `168 passed, 2 skipped` +- `python scripts\validate_skill.py --run-evals`: `OK: briefprint skill repository validated (24 test files, 9 eval cases, 10 trigger cases, 5 model benchmark cases)` - `ruff check .`: `All checks passed!` `tests/test_distribution_smoke.py` remains opt-in and skips unless `DBC_RUN_INSTALLED_SMOKE=1` is set. CI or release validation should still build wheel and sdist artifacts, install each into a fresh virtual environment, and run the installed smoke test outside the repository root. diff --git a/agents/openai.yaml b/agents/openai.yaml index 662e5d1..e1333d6 100644 --- a/agents/openai.yaml +++ b/agents/openai.yaml @@ -6,7 +6,7 @@ interface: default_prompt: "Use $briefprint to summarize the supplied documents into a cached briefing and include cache stats." policy: - allow_implicit_invocation: true + allow_implicit_invocation: false metadata: name: "briefprint" @@ -18,14 +18,10 @@ metadata: - "action_items" - "digest" - "debug" - triggers: - - "summarize these supplied documents" - - "brief this report from cache" - - "recap these meeting notes and reuse the summary" - - "summarize this JSON or XML payload" - - "create an executive digest for this document set" - - "rerender this briefing as digest or action_items" - - "summarize again without reprocessing the same documents" + invocation_examples: + - "$briefprint summarize supplied documents into a cached briefing" + - "$briefprint rerender cached briefing as digest" + - "$briefprint reuse cached document summaries" boundaries: - "Do not use for live research or current-fact lookup." - "Do not use for source-code review or debugging." diff --git a/docs/agent-skill-installation.md b/docs/agent-skill-installation.md index 5a70453..98b373b 100644 --- a/docs/agent-skill-installation.md +++ b/docs/agent-skill-installation.md @@ -32,6 +32,8 @@ This repository has two different install surfaces: Do not install the repository root as an agent skill. Root-copy installers can copy tests, docs, examples, eval fixtures, source code, and validation scripts into the host skill directory. Use the lightweight skill subdirectory instead. +The skill bundle is static install-time guidance. Briefprint's runtime cache lives under `--cache-dir`; it is not part of the agent skill bundle. Installing, updating, or removing the agent skill does not migrate, prune, or delete runtime caches. No portable agent-skill host contract currently provides automatic eviction for generated document state. Do not write document caches into the installed skill directory. + ## Bundle Contents The installable skill bundle is: diff --git a/evals/model_invocation_benchmark_cases.json b/evals/model_invocation_benchmark_cases.json index 5672517..c203e96 100644 --- a/evals/model_invocation_benchmark_cases.json +++ b/evals/model_invocation_benchmark_cases.json @@ -5,27 +5,37 @@ "cases": [ { "id": "positive-meeting-rerender", - "prompt": "이 회의록을 요약하고 같은 내용을 action_items 형식으로 다시 렌더링해줘.", + "prompt": "$briefprint summarize these meeting notes into a cached briefing, then rerender the same content as action_items.", "expected_invocation": true, "observed_invocation": null, "host": null, "model": null, "date": null, - "notes": "Expected to invoke because the user supplies document-like content and asks for reusable briefing/rerendering." + "notes": "Expected to invoke because the user explicitly invokes $briefprint and asks for reusable briefing/rerendering." }, { "id": "positive-json-digest", - "prompt": "이 JSON payload를 임원용 digest로 만들고, 다음 요청에서 재사용할 수 있게 캐시해줘.", + "prompt": "$briefprint summarize this JSON payload into a cached digest for reuse in later requests.", "expected_invocation": true, "observed_invocation": null, "host": null, "model": null, "date": null, - "notes": "Expected to invoke because the user asks for cached structured briefing from supplied JSON." + "notes": "Expected to invoke because the user explicitly invokes $briefprint for cached structured briefing from supplied JSON." + }, + { + "id": "negative-ordinary-summary", + "prompt": "Summarize these meeting notes and extract action items.", + "expected_invocation": false, + "observed_invocation": null, + "host": null, + "model": null, + "date": null, + "notes": "Expected not to invoke because this is an ordinary one-off summary without explicit briefprint invocation." }, { "id": "negative-live-news", - "prompt": "오늘 최신 금융 뉴스를 찾아서 분석해줘.", + "prompt": "Find and analyze today's latest financial news.", "expected_invocation": false, "observed_invocation": null, "host": null, @@ -35,7 +45,7 @@ }, { "id": "negative-source-debugging", - "prompt": "이 Python stack trace를 보고 버그 원인을 찾아 고쳐줘.", + "prompt": "Read this Python stack trace and find the root cause.", "expected_invocation": false, "observed_invocation": null, "host": null, diff --git a/evals/trigger_eval_cases.json b/evals/trigger_eval_cases.json index 1fa6fb4..e5ab9e5 100644 --- a/evals/trigger_eval_cases.json +++ b/evals/trigger_eval_cases.json @@ -1,10 +1,10 @@ { "schema_version": "1.0.0", - "description": "Skill trigger and boundary fixtures only; no pipeline execution.", + "description": "Skill trigger and boundary fixtures only; no pipeline execution. Briefprint is explicit-use: positive cases require explicit briefprint invocation.", "cases": [ { - "id": "pos-meeting-brief-inline", - "prompt": "Summarize these meeting notes and extract action items.", + "id": "pos-explicit-meeting-brief-inline", + "prompt": "$briefprint summarize these meeting notes into a cached briefing and extract action items.", "input": { "kind": "inline_document", "doc_type": "meeting_notes", @@ -17,8 +17,8 @@ } }, { - "id": "pos-json-digest", - "prompt": "이 JSON payload를 임원용 digest로 바꿔줘.", + "id": "pos-explicit-json-digest", + "prompt": "$briefprint summarize this JSON payload into a reusable digest.", "input": { "kind": "inline_document", "doc_type": "api_payload", @@ -31,8 +31,8 @@ } }, { - "id": "pos-template-rerender", - "prompt": "같은 문서를 action_items 형식으로 재렌더링해줘.", + "id": "pos-cached-state-rerender", + "prompt": "$briefprint rerender this cached briefing as action_items.", "input": { "kind": "cached_state" }, @@ -42,8 +42,8 @@ } }, { - "id": "pos-review-comments-document", - "prompt": "이 PR 리뷰 코멘트 묶음을 요약하고 남은 액션만 정리해줘.", + "id": "pos-explicit-review-comments-document", + "prompt": "Use $briefprint on these exported PR review comments and summarize actions, risks, and decisions.", "input": { "kind": "inline_document", "doc_type": "review_comments", @@ -55,9 +55,23 @@ "intent": "summarize_review_comments" } }, + { + "id": "neg-ordinary-one-off-summary", + "prompt": "Summarize these meeting notes and extract action items.", + "input": { + "kind": "inline_document", + "doc_type": "meeting_notes", + "content_format": "markdown", + "sample": "Decision: ship. Action: Backend should patch PAY-482 by 2026-05-10." + }, + "expect": { + "invoke": false, + "boundary": "ordinary_one_off_summary" + } + }, { "id": "neg-live-research", - "prompt": "오늘 최신 금융 뉴스를 찾아서 분석해줘.", + "prompt": "Find and analyze today's latest financial news.", "input": { "kind": "no_document" }, @@ -79,7 +93,7 @@ }, { "id": "neg-debugging", - "prompt": "이 stack trace 원인을 디버깅하고 수정 방법을 알려줘.", + "prompt": "Debug this stack trace and tell me how to fix it.", "input": { "kind": "stack_trace" }, @@ -90,7 +104,7 @@ }, { "id": "neg-translation-only", - "prompt": "이 문장을 영어로 번역만 해줘.", + "prompt": "Translate this paragraph to English only.", "input": { "kind": "plain_text" }, @@ -101,7 +115,7 @@ }, { "id": "neg-simple-qa", - "prompt": "캐시는 보통 어디에 저장돼?", + "prompt": "Where is the cache usually stored?", "input": { "kind": "no_document" }, diff --git a/references/best-practices.md b/references/best-practices.md index d67ec17..c9c8177 100644 --- a/references/best-practices.md +++ b/references/best-practices.md @@ -33,11 +33,13 @@ Provider prompt caching can reduce repeated prefix cost, but it is not the same A good skill should be easy to inspect: - short `SKILL.md`, -- clear trigger phrases, +- clear invocation examples and boundaries, - minimal scripts, - concrete schemas, - tests that prove behavior. +The skill bundle is static install-time guidance. Briefprint's runtime cache lives under `--cache-dir`; it is created by the CLI/runtime, not by the installed skill bundle. Installing, updating, or removing the agent skill does not migrate, prune, or delete runtime caches. No portable agent-skill host contract currently provides automatic eviction for generated document state. Do not write document caches into the installed skill directory. + ## 8. Validate with real samples Create a small evaluation set from actual documents and track: @@ -85,4 +87,4 @@ Document titles, sources, summaries, actions, and risk text may contain raw HTML ## 11. Prepare short deployment descriptions -Codex/OpenAI skill metadata can use a detailed `SKILL.md` description for precise triggering. Claude.ai upload flows may require a shorter description, so keep a 200-character-safe variant ready for packaging. +Codex/OpenAI skill metadata can use a detailed `SKILL.md` description for discoverability and manual invocation guidance. Claude.ai upload flows may require a shorter description, so keep a 200-character-safe variant ready for packaging. diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 6aed74a..2466b71 100644 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -200,8 +200,24 @@ def validate_trigger_eval_cases(path: Path) -> tuple[dict | None, list[str]]: has_positive = False has_negative = False allowed_input_kinds = {"inline_document", "cached_state", "no_document", "source_code", "stack_trace", "plain_text"} - allowed_negative_boundaries = {"live_research", "source_code_review", "debugging", "general_writing", "translation_only", "simple_qa", "no_document_input"} - required_negative_boundaries = {"live_research", "source_code_review", "debugging", "translation_only", "simple_qa"} + allowed_negative_boundaries = { + "live_research", + "source_code_review", + "debugging", + "general_writing", + "translation_only", + "simple_qa", + "no_document_input", + "ordinary_one_off_summary", + } + required_negative_boundaries = { + "live_research", + "source_code_review", + "debugging", + "translation_only", + "simple_qa", + "ordinary_one_off_summary", + } seen_negative_boundaries: set[str] = set() for idx, case in enumerate(cases): prefix = f"Trigger eval case {idx}" @@ -229,6 +245,9 @@ def validate_trigger_eval_cases(path: Path) -> tuple[dict | None, list[str]]: has_positive = True if input_payload.get("kind") not in {"inline_document", "cached_state"}: errors.append(f"{prefix} positive trigger should use document-like input or cached state.") + lowered_prompt = str(case.get("prompt", "")).lower() + if not _has_explicit_briefprint_invocation(lowered_prompt): + errors.append(f"{prefix} positive trigger should explicitly invoke briefprint.") if not expect.get("intent"): errors.append(f"{prefix} positive trigger missing expect.intent.") if expect.get("invoke") is False: @@ -259,12 +278,22 @@ def validate_openai_yaml(path: Path) -> list[str]: 'short_description: "Rerender cached document briefings without re-summarizing unchanged supplied documents."', "$briefprint", "policy:", - "allow_implicit_invocation: true", + "allow_implicit_invocation: false", + "invocation_examples:", 'name: "briefprint"', ] for fragment in required_fragments: if fragment not in text: errors.append(f"agents/openai.yaml missing required metadata fragment: {fragment}") + forbidden_fragments = [ + "triggers:", + "summarize these supplied documents", + "summarize this JSON or XML payload", + "create an executive digest for this document set", + ] + for fragment in forbidden_fragments: + if fragment in text: + errors.append(f"agents/openai.yaml should not include broad implicit trigger fragment: {fragment}") return errors @@ -295,6 +324,8 @@ def validate_installable_skill_bundle(bundle: Path) -> list[str]: errors.append("Installable skill SKILL.md metadata must include description.") elif len(description_match.group(1).strip().strip('"')) > 1024: errors.append("Installable skill description must be at most 1024 characters.") + if not re.search(r"^disable-model-invocation:\s*true\s*$", frontmatter, flags=re.MULTILINE): + errors.append("Installable skill SKILL.md must disable model invocation for manual-use hosts.") root_only_fragments = ["src/", "tests/", "evals/", "examples/", "scripts/", "VALIDATION.md", "pyproject.toml", "pip install"] for fragment in root_only_fragments: @@ -344,6 +375,10 @@ def validate_model_invocation_benchmark_cases_from_payload(payload: dict, path: continue has_positive = has_positive or case["expected_invocation"] has_negative = has_negative or not case["expected_invocation"] + if case["expected_invocation"]: + lowered_prompt = str(case.get("prompt", "")).lower() + if not _has_explicit_briefprint_invocation(lowered_prompt): + errors.append(f"{prefix} positive expected invocation should explicitly invoke briefprint.") if "observed_invocation" not in case: errors.append(f"{prefix} missing observed_invocation.") elif case.get("observed_invocation") is not None and ( @@ -477,28 +512,15 @@ def infer_skill_trigger_for_eval(case: dict) -> bool: "무엇", "what is", ) - positive_terms = ( - "summarize", - "brief", - "회의록", - "문서", - "json", - "xml", - "payload", - "리포트", - "로그", - "티켓", - "transcript", - "재렌더", - "rerender", - "digest", - "액션", - "리뷰 코멘트", - "pr 리뷰", - ) if any(term in lowered for term in negative_terms): return False - return kind in {"inline_document", "cached_state"} and any(term in lowered for term in positive_terms) + if kind not in {"inline_document", "cached_state"}: + return False + return _has_explicit_briefprint_invocation(lowered) + + +def _has_explicit_briefprint_invocation(prompt: str) -> bool: + return re.search(r"(?= 4 assert len(negatives) >= 5 + assert all("briefprint" in case["prompt"].lower() for case in positives) assert any(case["expect"].get("boundary") == "source_code_review" for case in negatives) assert any(case["expect"].get("boundary") == "translation_only" for case in negatives) + assert any(case["expect"].get("boundary") == "ordinary_one_off_summary" for case in negatives) assert any(case["expect"].get("intent") == "summarize_review_comments" for case in positives) - assert any(case["prompt"] == "오늘 최신 금융 뉴스를 찾아서 분석해줘." for case in cases) + assert any(case["prompt"] == "Summarize these meeting notes and extract action items." for case in cases) def test_trigger_eval_runner_lints_static_boundary_fixtures(): @@ -28,3 +30,70 @@ def test_trigger_eval_runner_lints_static_boundary_fixtures(): payload = json.loads(path.read_text(encoding="utf-8")) assert run_trigger_eval_cases(payload) == [] + + +def test_trigger_eval_validator_rejects_non_explicit_positive_invocation(tmp_path): + path = tmp_path / "trigger_eval_cases.json" + path.write_text( + json.dumps( + { + "schema_version": "1.0.0", + "cases": [ + { + "id": "bad-positive-cached-rerender", + "prompt": "Rerender this cached briefing as action_items.", + "input": {"kind": "cached_state"}, + "expect": {"invoke": True, "intent": "rerender"}, + }, + { + "id": "neg-ordinary-one-off-summary", + "prompt": "Summarize these meeting notes and extract action items.", + "input": {"kind": "inline_document"}, + "expect": {"invoke": False, "boundary": "ordinary_one_off_summary"}, + }, + { + "id": "neg-live-research", + "prompt": "Find and analyze today's latest financial news.", + "input": {"kind": "no_document"}, + "expect": {"invoke": False, "boundary": "live_research"}, + }, + { + "id": "neg-source-code-review", + "prompt": "Review this diff and find bugs in the implementation.", + "input": {"kind": "source_code"}, + "expect": {"invoke": False, "boundary": "source_code_review"}, + }, + { + "id": "neg-debugging", + "prompt": "Debug this stack trace and tell me how to fix it.", + "input": {"kind": "stack_trace"}, + "expect": {"invoke": False, "boundary": "debugging"}, + }, + { + "id": "neg-translation-only", + "prompt": "Translate this paragraph to English only.", + "input": {"kind": "plain_text"}, + "expect": {"invoke": False, "boundary": "translation_only"}, + }, + { + "id": "neg-simple-qa", + "prompt": "Where is the cache usually stored?", + "input": {"kind": "no_document"}, + "expect": {"invoke": False, "boundary": "simple_qa"}, + }, + { + "id": "neg-general-writing", + "prompt": "Write a polished announcement about this product launch.", + "input": {"kind": "plain_text"}, + "expect": {"invoke": False, "boundary": "general_writing"}, + }, + ], + }, + indent=2, + ), + encoding="utf-8", + ) + + _, errors = validate_trigger_eval_cases(path) + + assert any("positive trigger should explicitly invoke briefprint" in error for error in errors) From a9f53ccb49b6f511fcb5ed9d28d0bbf0d5d58e14 Mon Sep 17 00:00:00 2001 From: dd3ok <15044917+dd3ok@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:06:21 +0900 Subject: [PATCH 2/2] Tighten explicit briefprint invocation detection --- VALIDATION.md | 2 +- scripts/validate_skill.py | 2 +- tests/test_trigger_evals.py | 67 +++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/VALIDATION.md b/VALIDATION.md index 4265a47..ef84068 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -20,7 +20,7 @@ ruff check . ## Observed Results -- `python -m pytest -q -p no:cacheprovider`: `168 passed, 2 skipped` +- `python -m pytest -q -p no:cacheprovider`: `169 passed, 2 skipped` - `python scripts\validate_skill.py --run-evals`: `OK: briefprint skill repository validated (24 test files, 9 eval cases, 10 trigger cases, 5 model benchmark cases)` - `ruff check .`: `All checks passed!` diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 2466b71..564bbc4 100644 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -520,7 +520,7 @@ def infer_skill_trigger_for_eval(case: dict) -> bool: def _has_explicit_briefprint_invocation(prompt: str) -> bool: - return re.search(r"(?