From 35b3f6f91d44256886f679e22779a2decf779f25 Mon Sep 17 00:00:00 2001 From: tak2-08 Date: Wed, 26 Aug 2026 11:25:17 +0000 Subject: [PATCH] =?UTF-8?q?fix(integration):=20Claude=C2=B7Nemotron=20?= =?UTF-8?q?=EC=99=B8=EB=B6=80=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EB=A6=AC?= =?UTF-8?q?=ED=8F=AC=ED=8A=B8=20=EB=B0=98=EC=98=81=20=E2=80=94=20=EB=8F=84?= =?UTF-8?q?=EA=B5=AC=20=EA=B0=84=20=EC=A0=95=ED=95=A9=EC=84=B1=20=EA=B2=B0?= =?UTF-8?q?=ED=95=A8=20=ED=8C=A8=EC=B9=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [Claude E2E 리포트 — 실버그] 1. init이 심는 schema.json에 고정 9-type enum → ac.mjs issue(type:issue)가 자체 validate에서 즉시 실패 (쓰기/검증 도구 철학 불일치) - 수정: index --init이 typesFluid(기본 true)를 존중해 pattern 기반 type 생성, typesFluid===false일 때만 enum 강제 - init 기본 config에 typesFluid:true 추가 2. typesFluid가 어느 코드에서도 읽히지 않는 장식 설정이었음 - 이제 index --init(schema 생성)과 validate(방어링) 양쪽에서 실제 소비 - validator는 typesFluid=true면 stale enum도 통과시켜 구버전 scaffold 호환 3. 권고 반영: tools/e2e-workflow.mjs 신설 — init→write→validate→search→handoff 종단간 테스트를 임시 프로젝트에서 실수행, 6 step 전부 통과. CI에 스텝 추가로 '개별 도구는 되지만 조합에서 깨짐' 재발 방지. [Nemotron stress 리포트] 4. 동의어 확장: benchmark searchLite도 config synonyms 사용 (recall 개선, 0 LLM) 5. cache-miss 확장 루프: 시작 레벨 miss 시 큰 레벨로 최대 2회 확장을 benchmark searchLite에 실제 구현 (문서 예고분) 6. 'saving 100.0%' 착시: >=99.95%는 '99.9%+' 표기, miss는 'n/a (miss)' - agent-search-lite에도 동일 적용 + hit 필드 누락 버그 수정 (e2e가 이 버그까지 잡아냄 — 테스트의 가치 실증) 7. --queries 플래그: 사이클링 동작 명시 검증: e2e-workflow 6/6 pass, 시드 42 재현성 diff 일치, validate ok, node --check 전체 통과 --- .github/workflows/ci.yml | 2 + BENCHMARK.md | 61 +++++++++---------- ...2026-08-26-login-500-on-refresh--system.md | 21 +++++++ agent-context/graph.json | 2 +- agent-context/index.json | 31 ++++++++-- tools/agent-context-index.mjs | 10 +-- tools/agent-context-init.mjs | 3 +- tools/agent-context-validate.mjs | 6 +- tools/agent-search-lite.mjs | 11 +++- tools/benchmark.mjs | 51 +++++++++++++--- tools/e2e-workflow.mjs | 59 ++++++++++++++++++ 11 files changed, 203 insertions(+), 54 deletions(-) create mode 100644 agent-context/bugs/2026-08-26-login-500-on-refresh--system.md create mode 100644 tools/e2e-workflow.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf55984..600504a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,8 @@ jobs: node --check tools/agent-radio.mjs node tools/agent-radio.mjs list-threads node tools/agent-radio.mjs protocol + - name: E2E workflow (init→write→validate→search→handoff) + run: node tools/e2e-workflow.mjs - name: Skill check run: | test -f .claude/skills/agent-shared-context/SKILL.md diff --git a/BENCHMARK.md b/BENCHMARK.md index 89f6b8d..b650c56 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -1,9 +1,8 @@ # Benchmark — Hierarchical Lightweight Search vs Full Read -> **Objective, public-standard-like, critical, reproducible** — synthetic 5/50/500 scale, 20 queries, **fixed seed (--seed 42)**, tokens = chars/4, hit = query tokens in title/tags/summary, latency = search vs est. full Read, no LLM. -> -> **Issue #3 반영**: (1) 시드 고정으로 동일 커맨드 재실행 시 동일 결과 보장 (2) miss 쿼리는 "saving 100%"이 아니라 **n/a (miss)**로 표기 — 실패한 검색을 절약으로 과장하지 않음 (3) avg saving은 히트 기준만 집계. +> **Objective, public-standard-like, critical, reproducible** — synthetic 5/50/500 scale, 20 queries, tokens = chars/4, hit = query tokens in title/tags/summary, latency = search vs est. full Read, no LLM. + ## Method (close to public standard) - **Dataset**: Synthetic 5 + 50 + 500 entries, distribution 40% post-it (15tok) 30% memo (50tok) 15% diary (200tok) 10% bookshelf (1000tok) 5% library (5000tok) — like cache workloads, not cherry-picked. @@ -17,9 +16,9 @@ | scale | full tokens | avg top 3 tokens | avg saving | hitRate | avg latency (search) | est. full Read latency | tokens/hit | |---|---|---|---|---|---|---| -| 5 | 1315 | 178 | 83.1% | 80.0% | 0.09ms | 0.25ms (est. Read all md) | 223 | -| 50 | 16780 | 761 | 94.7% | 85.0% | 0.37ms | 2.50ms (est. Read all md) | 895 | -| 500 | 197940 | 1883 | 98.9% | 85.0% | 1.92ms | 25.00ms (est. Read all md) | 2216 | +| 5 | 1315 | 178 | 83.1% | 80.0% | 0.16ms | 0.25ms (est. Read all md) | 223 | +| 50 | 16780 | 761 | 94.7% | 85.0% | 0.41ms | 2.50ms (est. Read all md) | 895 | +| 500 | 197940 | 1883 | 98.9% | 85.0% | 2.34ms | 25.00ms (est. Read all md) | 2216 | ### Interpretation (critical, not hype) @@ -31,11 +30,11 @@ | query | assignedLevel | top tokens | saving | hit | latency | |---|---|---|---|---| -| auth | post-it | 45 | 99.7% | ✅ | 1.31ms | -| api | post-it | 80 | 99.5% | ✅ | 0.36ms | -| jwt | post-it | 0 | n/a (miss) | ❌ | 0.31ms | -| pagination | post-it | 0 | n/a (miss) | ❌ | 0.31ms | -| cache | post-it | 0 | n/a (miss) | ❌ | 0.31ms | +| auth | post-it | 45 | 99.7% | ✅ | 0.39ms | +| api | post-it | 80 | 99.5% | ✅ | 0.33ms | +| jwt | post-it | 0 | n/a (miss) | ❌ | 0.89ms | +| pagination | post-it | 0 | n/a (miss) | ❌ | 1.56ms | +| cache | post-it | 0 | n/a (miss) | ❌ | 0.64ms | ### What we learned while benchmarking (ideas & shortcomings →补) @@ -66,7 +65,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "avgTopTokens": 178, "avgSaving": "83.1%", "hitRate": "80.0%", - "avgLatency": "0.09ms", + "avgLatency": "0.16ms", "fullLatencyEst": "0.25ms (est. Read all md)", "tokensPerHit": 223, "perQuery": [ @@ -76,7 +75,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 50, "saving": "96.2%", "hit": true, - "latency": "0.48ms" + "latency": "0.65ms" }, { "query": "api", @@ -84,7 +83,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 15, "saving": "98.9%", "hit": true, - "latency": "0.22ms" + "latency": "0.25ms" }, { "query": "jwt", @@ -92,7 +91,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "0.07ms" + "latency": "0.16ms" }, { "query": "pagination", @@ -100,7 +99,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "0.05ms" + "latency": "0.13ms" }, { "query": "cache", @@ -108,7 +107,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "0.04ms" + "latency": "0.35ms" } ] }, @@ -120,7 +119,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "avgTopTokens": 761, "avgSaving": "94.7%", "hitRate": "85.0%", - "avgLatency": "0.37ms", + "avgLatency": "0.41ms", "fullLatencyEst": "2.50ms (est. Read all md)", "tokensPerHit": 895, "perQuery": [ @@ -130,7 +129,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 45, "saving": "99.7%", "hit": true, - "latency": "1.31ms" + "latency": "0.39ms" }, { "query": "api", @@ -138,7 +137,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 80, "saving": "99.5%", "hit": true, - "latency": "0.36ms" + "latency": "0.33ms" }, { "query": "jwt", @@ -146,7 +145,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "0.31ms" + "latency": "0.89ms" }, { "query": "pagination", @@ -154,7 +153,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "0.31ms" + "latency": "1.56ms" }, { "query": "cache", @@ -162,7 +161,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "0.31ms" + "latency": "0.64ms" } ] }, @@ -174,7 +173,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "avgTopTokens": 1883, "avgSaving": "98.9%", "hitRate": "85.0%", - "avgLatency": "1.92ms", + "avgLatency": "2.34ms", "fullLatencyEst": "25.00ms (est. Read all md)", "tokensPerHit": 2216, "perQuery": [ @@ -182,17 +181,17 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "query": "auth", "assignedLevel": "post-it", "topTokens": 45, - "saving": "100.0%", + "saving": "99.9%+", "hit": true, - "latency": "4.01ms" + "latency": "3.82ms" }, { "query": "api", "assignedLevel": "post-it", "topTokens": 45, - "saving": "100.0%", + "saving": "99.9%+", "hit": true, - "latency": "2.09ms" + "latency": "1.83ms" }, { "query": "jwt", @@ -200,7 +199,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "2.05ms" + "latency": "6.05ms" }, { "query": "pagination", @@ -208,7 +207,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "2.48ms" + "latency": "4.64ms" }, { "query": "cache", @@ -216,7 +215,7 @@ No API key, no `npm install`, Node ≥18 only — like `agent-search-lite.mjs`. "topTokens": 0, "saving": "n/a (miss)", "hit": false, - "latency": "1.93ms" + "latency": "4.53ms" } ] } diff --git a/agent-context/bugs/2026-08-26-login-500-on-refresh--system.md b/agent-context/bugs/2026-08-26-login-500-on-refresh--system.md new file mode 100644 index 0000000..51fb6cb --- /dev/null +++ b/agent-context/bugs/2026-08-26-login-500-on-refresh--system.md @@ -0,0 +1,21 @@ + +--- +id: issue-20260826-b48a3b55 +type: issue +title: "login 500 on refresh" +tags: [issue] +feature: auth +level: "" +scope: global +agent: system +created: 2026-08-26T11:23:08.632Z +updated: 2026-08-26T11:23:08.632Z +status: done +priority: 4 +summary: "login 500 on refresh" +--- + +## 결과 + +(도구 호출 로그 아님 — 결론만 기록. 검증은 refs 링크로) + diff --git a/agent-context/graph.json b/agent-context/graph.json index 3b691fb..5681ef2 100644 --- a/agent-context/graph.json +++ b/agent-context/graph.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-08-26T09:51:47.670Z", + "generated_at": "2026-08-26T11:25:17.405Z", "_path": "agent-context/graph.json", "description": "기능 연관 그래프 — depends_on/affects로 영향 범위 추적. agent-context.config.json graph.edges로부터 생성됨.", "graph": { diff --git a/agent-context/index.json b/agent-context/index.json index 6af22a4..6fc4024 100644 --- a/agent-context/index.json +++ b/agent-context/index.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-08-26T09:51:47.664Z", + "generated_at": "2026-08-26T11:25:17.401Z", "generated_by": "agent-context-index.mjs", "_path": "agent-context/index.json", "description": "L1 압축 카탈로그 — 저용량 에이전트가 가장 먼저 읽는 파일. preview 60자 + summary 120자로 본문 Read 없이 관련성 판단.", @@ -8,12 +8,13 @@ "soft_limit_chars": 200000, "max_entries": 1000, "should_compress": false, - "total_chars": 1117, - "total_entries": 1 + "total_chars": 1509, + "total_entries": 2 }, "counts": { "handoff": 1, - "total": 1 + "issue": 1, + "total": 2 }, "entries": [ { @@ -38,6 +39,28 @@ "related": [], "affects": [], "chars": 1117 + }, + { + "id": "issue-20260826-b48a3b55", + "type": "issue", + "level": "diary", + "title": "login 500 on refresh", + "tags": [ + "issue" + ], + "feature": "auth", + "scope": "global", + "agent": "system", + "created": "2026-08-26T11:23:08.632Z", + "updated": "2026-08-26T11:23:08.632Z", + "status": "done", + "priority": 4, + "summary": "login 500 on refresh", + "preview": "login 500 on refresh", + "path": "bugs/2026-08-26-login-500-on-refresh--system.md", + "related": [], + "affects": [], + "chars": 392 } ] } diff --git a/tools/agent-context-index.mjs b/tools/agent-context-index.mjs index 381910c..04ca99a 100644 --- a/tools/agent-context-index.mjs +++ b/tools/agent-context-index.mjs @@ -211,8 +211,10 @@ if (ARGS.init) { required: CONFIG.schema?.required || ["id","type","title","tags","feature","agent","created","updated","status","summary"], properties: { ...(schema.properties || {}), - type: { type: "string", enum: types }, - feature: { type: "string", enum: featureEnum, description: "graph.json/features.json 키와 연결" }, + type: CONFIG.typesFluid === false + ? { type: "string", enum: types } + : { type: "string", pattern: "^[a-z0-9-]+$", description: "유동적 타입 — 자유 문자열 (typesFluid)" }, + feature: { type: "string", pattern: "^[a-z0-9-_]+$", description: "유동적 feature — graph/features 키와 연결, 자유 확장" }, agent: { type: "string", enum: agents }, }, }; @@ -225,10 +227,10 @@ if (ARGS.init) { // minimal fallback expectedSchema.properties = { id: { type: "string", pattern: CONFIG.schema?.idPattern || "^[a-z-]+-[0-9]{8}-[a-z0-9]{8}$" }, - type: { type: "string", enum: types }, + type: CONFIG.typesFluid === false ? { type: "string", enum: types } : { type: "string", pattern: "^[a-z0-9-]+$" }, title: { type: "string", minLength: 5, maxLength: 80 }, tags: { type: "array", items: { type: "string", pattern: "^[a-z0-9-]+$" }, minItems: 1, maxItems: 8 }, - feature: { type: "string", enum: featureEnum }, + feature: { type: "string", pattern: "^[a-z0-9-_]+$" }, scope: { type: "string", pattern: "^(global|page|custom:.+)$", default: "global" }, agent: { type: "string", enum: agents }, created: { type: "string", format: "date-time" }, diff --git a/tools/agent-context-init.mjs b/tools/agent-context-init.mjs index 26e2840..710fb79 100644 --- a/tools/agent-context-init.mjs +++ b/tools/agent-context-init.mjs @@ -59,7 +59,8 @@ const config = { privateMirror: null, features: Object.fromEntries(featuresList.map(f=> [f, { label: f[0].toUpperCase()+f.slice(1), files: [`src/${f}/index.ts:1`], description: `${f} feature` }])), graph: { edges: featuresList.length>=2 ? [[featuresList[1], featuresList[0]]] : [] }, - types: ["note","memo","idea","learning","bug","decision","diary","code-history","todo"], + types: ["note","memo","idea","learning","bug","decision","diary","code-history","todo","issue"], + typesFluid: true, schema: { required: ["id","type","title","tags","feature","agent","created","updated","status","summary"], featureEnum: "auto", idPattern: "^[a-z-]+-[0-9]{8}-[a-z0-9]{8}$", maxSummary: 200, maxPreview: 60 }, storage: { backend: "json", softLimits: { softLimitChars: 200000, maxEntries: 1000, archiveAfterDays: 90 } }, lint: { onIndexRegenerate: true, forbidWriteOverwrite: true, requiredKeywords: false }, diff --git a/tools/agent-context-validate.mjs b/tools/agent-context-validate.mjs index 7ce528f..7c0c4fe 100644 --- a/tools/agent-context-validate.mjs +++ b/tools/agent-context-validate.mjs @@ -85,7 +85,11 @@ for (const f of files) { if (fm.id && !new RegExp(schema.properties?.id?.pattern || "^[a-z-]+-[0-9]{8}-[a-z0-9]{8}$").test(fm.id)) { console.error(`FAIL ${f}: id pattern mismatch '${fm.id}'`); errors++; } - if (fm.type && schema.properties?.type?.enum && !schema.properties.type.enum.includes(fm.type)) { + // typesFluid=true (기본)면 유동 타입 허용 — 구버전 scaffold의 stale enum도 통과시킴 + const typeEnum = schema.properties?.type?.enum; + if (fm.type && CONFIG.typesFluid === true) { + if (!/^[a-z0-9-]+$/.test(fm.type)) { console.error(`FAIL ${f}: fluid type pattern mismatch '${fm.type}'`); errors++; } + } else if (fm.type && typeEnum && !typeEnum.includes(fm.type)) { console.error(`FAIL ${f}: type '${fm.type}' not in enum`); errors++; } if (fm.feature && schema.properties?.feature?.enum && !schema.properties.feature.enum.includes(fm.feature)) { diff --git a/tools/agent-search-lite.mjs b/tools/agent-search-lite.mjs index 7c7370f..31ca7f2 100644 --- a/tools/agent-search-lite.mjs +++ b/tools/agent-search-lite.mjs @@ -130,16 +130,23 @@ function search(query, opts={}) { const totalTokens = top.reduce((sum,s)=>sum+s.estTokens,0); const wouldBeFullRead = entries.reduce((sum,e)=>sum+(LEVELS[estimateLevel(e)]?.tokens||200),0); const saving = wouldBeFullRead ? ((wouldBeFullRead-totalTokens)/wouldBeFullRead*100).toFixed(1) : 0; + const hit = top.length > 0; + // nemotron 지적 반영: miss는 'n/a (miss)', 99.95% 이상은 '99.9%+' 표기로 착시 제거 + const hitNum = parseFloat(saving); + const savingStr = !hit ? 'n/a (miss)' : (hitNum >= 99.95 ? '99.9%+' : `${saving}%`); return { query, assignedLevel: requestedLevel, + router: { type: 'rule-based heuristic', noLLM: true, zeroTokens: true, reason: `query ${qTokens.length} words → ${requestedLevel}` }, lightweightAI: { reason: `query ${qTokens.length} words → ${requestedLevel} (hierarchical cache)`, noLLM: true, zeroTokens: true }, order: ORDER, totalEntries: entries.length, evaluated: scored.length, + hit, + expandedTo: hit ? null : null, top: top.map(s=>({ id:s.entry.id, title:s.entry.title, level:s.lev, feature:s.entry.feature, priority:s.entry.priority, score: s.score.toFixed(2), estTokens:s.estTokens, path:s.entry.path, summary:s.entry.summary })), - tokens: { top: totalTokens, full: wouldBeFullRead, saving: `${saving}%`, avgPerQuery: top.length? Math.round(totalTokens/top.length):0 }, - note: `Hierarchical: ${ORDER.slice(0, startRank+1).join('→')} first, expand to larger only if no hit — like cache→HBM→DRAM→SSD→library` + tokens: { top: totalTokens, full: wouldBeFullRead, saving: savingStr, avgPerQuery: top.length? Math.round(totalTokens/top.length):0 }, + note: `Hierarchical: ${ORDER.slice(0, startRank+1).join('→')} first, miss expands to larger levels — cache metaphor` }; } diff --git a/tools/benchmark.mjs b/tools/benchmark.mjs index eee1713..63420ee 100644 --- a/tools/benchmark.mjs +++ b/tools/benchmark.mjs @@ -3,7 +3,8 @@ // Objective benchmark — close to public standard, critical, reproducible // Measures token saving of hierarchical lightweight AI search vs full read, with synthetic scale 5/50/500 // Standard: tokens = chars/4 (Anthropic counting), hit = query tokens found in title/tags/summary, latency = ms for search vs full read -// Run: node tools/benchmark.mjs [--scale 5,50,500] [--queries 20] [--json] +// Run: node tools/benchmark.mjs [--scale 5,50,500] [--queries 20] [--seed 42] [--json] +// --queries N: 20개 고정 질의 목록을 N회 사이클링 (N>20이면 반복) // Note: No LLM calls, 0 tokens for lightweight AI itself, like cache hierarchy import { writeFileSync, readFileSync, existsSync } from 'node:fs'; @@ -66,22 +67,52 @@ function syntheticEntries(n) { return entries; } +const SYNONYMS = CONFIG.search?.synonyms || {}; +function expandTokens(tokens) { + const set = new Set(tokens); + for (const t of tokens) { const s = SYNONYMS[t]; if (Array.isArray(s)) s.forEach(x=>set.add(x)); } + for (const [k, list] of Object.entries(SYNONYMS)) if (tokens.some(t => list.includes(t))) set.add(k); + return [...set]; +} +function rankOf(level){ const m=Object.fromEntries(LEVELS.map((k,i)=>[k,i])); return m[level] ?? 0; } + function searchLite(entries, query, opts={}) { - const limit = opts.limit||3; - const qTokens = query.toLowerCase().split(/\s+/).filter(Boolean); - // Lightweight AI assign level: 1 word → post-it, 2-3 → memo, 4-8 → diary, else bookshelf (like hierarchy doc) + // nemotron 리뷰 반영: 동의어 확장(0 LLM) + cache-miss 시 큰 레벨로 최대 2회 확장 + const rawTokens = query.toLowerCase().split(/\s+/).filter(Boolean); + const qTokens = expandTokens(rawTokens); let assignedLevel = opts.level; if (!assignedLevel) { - const words = qTokens.length; + const words = rawTokens.length; const q = query.toLowerCase(); - if (q.includes('overall') || q.includes('architecture')) assignedLevel='bookshelf'; + if (q.includes('overall') || q.includes('architecture') || q.includes('flow')) assignedLevel='bookshelf'; else if (words<=1) assignedLevel='post-it'; else if (words<=3) assignedLevel='memo'; else if (words<=8) assignedLevel='diary'; else assignedLevel='bookshelf'; } + let res = collect(entries, qTokens, opts, rankOf(assignedLevel)); + res.assignedLevel = assignedLevel; + if (!res.hit && !opts.level) { + let r = LEVELS.indexOf(assignedLevel); + for (let step=0; step<2 && r+1[k,i])); - const startRank = levelRank[assignedLevel] ?? 0; + const startRank = startRankOverride !== undefined ? startRankOverride : (levelRank['post-it'] ?? 0); // Score like agent-search-lite: hit + priority + recency - levelDistance const scored = entries.map(e=>{ const levRank = levelRank[e.level] ?? 2; @@ -100,7 +131,7 @@ function searchLite(entries, query, opts={}) { const topTokens = top.reduce((s,x)=>s+x.estTokens,0); const fullTokens = entries.reduce((s,e)=>s+(TOKENS[e.level]||200),0); const hit = top.length>0; - return { assignedLevel, top, topTokens, fullTokens, saving: fullTokens? (fullTokens-topTokens)/fullTokens*100:0, hit, evaluated: scored.length }; + return { top, topTokens, fullTokens, hit, evaluated: scored.length }; } function benchmark(scales=[5,50,500], queriesPerScale=20) { @@ -127,7 +158,7 @@ function benchmark(scales=[5,50,500], queriesPerScale=20) { // Issue #3 fix: average saving over HITS only — a miss is not "infinite saving" if (res.hit) { totalSaving += res.saving; hits++; } // Issue #3 fix: miss with 0 tokens is NOT "100% saving" — it's a failed search. - const savingStr = res.hit ? res.saving.toFixed(1)+'%' : 'n/a (miss)'; + const savingStr = !res.hit ? 'n/a (miss)' : (res.saving >= 99.95 ? '99.9%+' : res.saving.toFixed(1)+'%'); perQuery.push({ query: q, assignedLevel: res.assignedLevel, topTokens: res.topTokens, saving: savingStr, hit: res.hit, latency: latency.toFixed(2)+'ms' }); } results.push({ @@ -137,7 +168,7 @@ function benchmark(scales=[5,50,500], queriesPerScale=20) { fullTokens, avgTopTokens: Math.round(totalTopTokens/queriesPerScale), // saving averaged over hits only; misses reported separately via hitRate - avgSaving: hits ? (totalSaving/hits).toFixed(1)+'%' : 'n/a', + avgSaving: hits ? ((totalSaving/hits) >= 99.95 ? '99.9%+' : (totalSaving/hits).toFixed(1)+'%') : 'n/a', hitRate: (hits/queriesPerScale*100).toFixed(1)+'%', avgLatency: (totalLatencyMs/queriesPerScale).toFixed(2)+'ms', fullLatencyEst: (entries.length*0.05).toFixed(2)+'ms (est. Read all md)', diff --git a/tools/e2e-workflow.mjs b/tools/e2e-workflow.mjs new file mode 100644 index 0000000..469c068 --- /dev/null +++ b/tools/e2e-workflow.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node +// Path: tools/e2e-workflow.mjs +// End-to-end workflow test: init → write(ac) → validate → search → handoff +// Claude 외부 리뷰 권고 반영 — "도구 간 조합" 통합 결함 방지 게이트. +// Run: node tools/e2e-workflow.mjs (temp dir에서 실제 시나리오 수행, 실패 시 exit 1) +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, existsSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { readdirSync } from 'node:fs'; + +const TOOLS = new URL('.', import.meta.url).pathname; +const dir = mkdtempSync(join(tmpdir(), 'ac-e2e-')); +let failures = 0; +function step(name, fn) { + const r = fn(); + if (r === false) { console.error(`FAIL ${name}`); failures++; } + else console.log(`ok ${name}`); +} +function ac(...a) { + return spawnSync(process.execPath, [join(TOOLS, 'ac.mjs'), ...a], { cwd: dir, encoding: 'utf8' }); +} + +step('init scaffolds into temp project', () => { + const r = spawnSync(process.execPath, [join(TOOLS, 'agent-context-init.mjs'), '--yes', '--project', 'e2e'], { cwd: dir, encoding: 'utf8' }); + return r.status === 0 && existsSync(join(dir, 'agent-context.config.json')); +}); + +step('write: ac issue creates entry with fluid type', () => { + const r = ac('issue', '--title', 'login 500 on refresh', '--feature', 'auth', '--summary', 'refresh 시 500'); + return r.status === 0 && existsSync(join(dir, 'agent-context/bugs')) && + readdirSync(join(dir, "agent-context/bugs")).some(f => f.endsWith(".md")); +}); + +step('validate passes on fluid type (typesFluid)', () => { + const r = spawnSync(process.execPath, [join(TOOLS, 'agent-context-validate.mjs')], { cwd: dir, encoding: 'utf8' }); + return r.status === 0; +}); + +step('search finds the issue (cross-agent recall)', () => { + const r = spawnSync(process.execPath, [join(TOOLS, 'agent-search-lite.mjs'), 'login 500', '--json'], { cwd: dir, encoding: 'utf8' }); + try { const j = JSON.parse(r.stdout); return j.hit === true; } catch { return false; } +}); + +step('handoff save + load roundtrip', () => { + const s = ac('export', '--session', 'agentA', '--task', 'fix login 500', '--next', 'verify'); + if (s.status !== 0) return false; + const l = spawnSync(process.execPath, [join(TOOLS, 'ac.mjs'), 'import'], { cwd: dir, encoding: 'utf8' }); + return l.status === 0 && l.stdout.includes('Session Handoff'); +}); + +step('CURRENT.md pointer exists for next session', () => existsSync(join(dir, 'agent-context/CURRENT.md'))); + +// cleanup +try { rmSync(dir, { recursive: true, force: true }); } catch {} +if (failures) { console.error(`\ne2e: ${failures} failure(s)`); process.exit(1); } +console.log('\ne2e: all steps passed'); + +