From ba4062ff37541e3e9e9524452e75af8777bee599 Mon Sep 17 00:00:00 2001 From: zhaoyinglia Date: Mon, 10 Aug 2026 20:38:49 +0800 Subject: [PATCH] docs(skills): add version-agnostic TE upgrade workflow --- skills/README.md | 84 ++++++++++++++++++ skills/te-audit-cicd/SKILL.md | 35 ++++++++ skills/te-audit-cicd/agents/openai.yaml | 4 + skills/te-audit-cicd/references/rules.md | 7 ++ skills/te-audit-cicd/scripts/audit_cicd.py | 28 ++++++ skills/te-audit-plugin-api/SKILL.md | 49 +++++++++++ skills/te-audit-plugin-api/agents/openai.yaml | 4 + .../references/decision-rules.md | 14 +++ .../scripts/audit_plugin_api.py | 38 ++++++++ skills/te-classify-fork-delta/SKILL.md | 59 +++++++++++++ .../te-classify-fork-delta/agents/openai.yaml | 4 + .../references/classification.md | 19 ++++ .../scripts/classify_fork_delta.py | 88 +++++++++++++++++++ skills/te-finalize-upstream-upgrade/SKILL.md | 38 ++++++++ .../agents/openai.yaml | 4 + .../references/fields.md | 5 ++ .../scripts/finalize_upgrade.py | 17 ++++ skills/te-integrate-build-submodules/SKILL.md | 36 ++++++++ .../agents/openai.yaml | 4 + .../references/rules.md | 12 +++ .../scripts/audit_build_submodules.py | 29 ++++++ .../te-integrate-upstream-conflicts/SKILL.md | 33 +++++++ .../agents/openai.yaml | 4 + .../references/decision-schema.md | 17 ++++ .../scripts/conflict_inventory.py | 29 ++++++ skills/te-preserve-runtime-patches/SKILL.md | 40 +++++++++ .../agents/openai.yaml | 4 + .../references/invariants.md | 5 ++ .../scripts/audit_runtime_patches.py | 29 ++++++ skills/te-run-upgrade-test-matrix/SKILL.md | 43 +++++++++ .../agents/openai.yaml | 4 + .../references/rules.md | 12 +++ .../scripts/generate_test_matrix.py | 23 +++++ skills/te-upgrade-orchestrator/SKILL.md | 52 +++++++++++ .../agents/openai.yaml | 4 + .../references/contract.md | 5 ++ .../scripts/check_orchestration.py | 13 +++ 37 files changed, 895 insertions(+) create mode 100644 skills/README.md create mode 100644 skills/te-audit-cicd/SKILL.md create mode 100644 skills/te-audit-cicd/agents/openai.yaml create mode 100644 skills/te-audit-cicd/references/rules.md create mode 100755 skills/te-audit-cicd/scripts/audit_cicd.py create mode 100644 skills/te-audit-plugin-api/SKILL.md create mode 100644 skills/te-audit-plugin-api/agents/openai.yaml create mode 100644 skills/te-audit-plugin-api/references/decision-rules.md create mode 100755 skills/te-audit-plugin-api/scripts/audit_plugin_api.py create mode 100644 skills/te-classify-fork-delta/SKILL.md create mode 100644 skills/te-classify-fork-delta/agents/openai.yaml create mode 100644 skills/te-classify-fork-delta/references/classification.md create mode 100755 skills/te-classify-fork-delta/scripts/classify_fork_delta.py create mode 100644 skills/te-finalize-upstream-upgrade/SKILL.md create mode 100644 skills/te-finalize-upstream-upgrade/agents/openai.yaml create mode 100644 skills/te-finalize-upstream-upgrade/references/fields.md create mode 100755 skills/te-finalize-upstream-upgrade/scripts/finalize_upgrade.py create mode 100644 skills/te-integrate-build-submodules/SKILL.md create mode 100644 skills/te-integrate-build-submodules/agents/openai.yaml create mode 100644 skills/te-integrate-build-submodules/references/rules.md create mode 100755 skills/te-integrate-build-submodules/scripts/audit_build_submodules.py create mode 100644 skills/te-integrate-upstream-conflicts/SKILL.md create mode 100644 skills/te-integrate-upstream-conflicts/agents/openai.yaml create mode 100644 skills/te-integrate-upstream-conflicts/references/decision-schema.md create mode 100755 skills/te-integrate-upstream-conflicts/scripts/conflict_inventory.py create mode 100644 skills/te-preserve-runtime-patches/SKILL.md create mode 100644 skills/te-preserve-runtime-patches/agents/openai.yaml create mode 100644 skills/te-preserve-runtime-patches/references/invariants.md create mode 100755 skills/te-preserve-runtime-patches/scripts/audit_runtime_patches.py create mode 100644 skills/te-run-upgrade-test-matrix/SKILL.md create mode 100644 skills/te-run-upgrade-test-matrix/agents/openai.yaml create mode 100644 skills/te-run-upgrade-test-matrix/references/rules.md create mode 100755 skills/te-run-upgrade-test-matrix/scripts/generate_test_matrix.py create mode 100644 skills/te-upgrade-orchestrator/SKILL.md create mode 100644 skills/te-upgrade-orchestrator/agents/openai.yaml create mode 100644 skills/te-upgrade-orchestrator/references/contract.md create mode 100755 skills/te-upgrade-orchestrator/scripts/check_orchestration.py diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 0000000000..cf5761f808 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,84 @@ +# TransformerEngine-FL Upgrade Skills + +这组 skills 用于维护 TransformerEngine-FL 与任意上游 TransformerEngine release 之间的升级流程。它们不绑定具体版本；每次运行通过 `base`、`fork`、`target` 三个 ref 参数确定范围。 + +## 用途 + +这套流程覆盖： + +- FL fork 设计基线和差异分类； +- plugin manager、registry、policy、discovery 和各硬件 backend； +- `transformer_engine_torch` 动态 alias、enum、callable public contract； +- PyTorch attention/device 等侵入式 runtime patch； +- C++/CUDA 构建、wheel、CMake 和 third-party submodule； +- GitHub Actions、硬件 CI、QA 和测试矩阵； +- 升级后的证据汇总、回滚和交付。 + +## 目录 + +| Skill | 作用 | +|---|---| +| `te-upgrade-orchestrator` | 按 gate 串联完整升级流程 | +| `te-classify-fork-delta` | 生成 fork 差异清单和设计基线 | +| `te-integrate-upstream-conflicts` | 分类和解决 upstream/fork 冲突 | +| `te-audit-plugin-api` | 审计 plugin/native/Python API 和动态 alias | +| `te-preserve-runtime-patches` | 保护 runtime 侵入式修改及跨模块调用链 | +| `te-integrate-build-submodules` | 审计构建、打包、CUDA 和 submodule | +| `te-audit-cicd` | 审计 CI/CD、硬件矩阵和 QA 入口 | +| `te-run-upgrade-test-matrix` | 执行 import、GPU、backend 和回归测试 | +| `te-finalize-upstream-upgrade` | 汇总证据并生成交付决策 | + +## 推荐用法 + +在仓库根目录开始，先创建工作分支，不要直接在 `main` 上操作： + +```bash +git switch -c chore/te-upgrade- +``` + +然后按以下顺序使用： + +```text +1. te-classify-fork-delta +2. te-integrate-upstream-conflicts +3. te-audit-plugin-api +4. te-preserve-runtime-patches +5. te-integrate-build-submodules +6. te-audit-cicd +7. te-run-upgrade-test-matrix +8. te-finalize-upstream-upgrade +``` + +也可以直接使用 `te-upgrade-orchestrator`，它会检查每个阶段的 artifact、ref 一致性、P0 决策和用户审批。 + +## 参数和 artifact + +典型参数： + +```text +base = 上游共同基线 ref +fork = 当前 FL 分支/ref +target = 要升级到的上游 ref +``` + +所有中间结果放在： + +```text +/share/project/zhaoyingli/flagos/temp// +``` + +不要把审计结果或生成文件放到 `/tmp`。每个阶段都应记录：输入 refs、输出 artifact、owner、验收命令和 `preserve/adapt/drop` 决策。 + +## 关键验收原则 + +- fork-owned plugin、runtime、build 和 CI 文件不能因 whole-file/tree replacement 被删除； +- 动态 `transformer_engine_torch` 必须与目标 release 的 Python/native public contract 对齐； +- enum、sentinel、import-time callable 和签名必须逐项检查； +- runtime 修改按“定义—导入—alias—调用”链验证； +- 每个 backend、CI surface、submodule 和测试入口都有 owner 或明确排除理由； +- GPU、reference、plugin manager 和 staged import 测试结果必须分开记录； +- 未经用户批准，不执行 merge-to-main、push、PR、tag 或 release。 + +## 本仓库当前实例 + +本仓库外部 artifact 中保存了本次 FL 设计基线和升级证据；仓库内的 skills 保持版本无关，只提供可复用流程和验收标准。 diff --git a/skills/te-audit-cicd/SKILL.md b/skills/te-audit-cicd/SKILL.md new file mode 100644 index 0000000000..a19334d6cf --- /dev/null +++ b/skills/te-audit-cicd/SKILL.md @@ -0,0 +1,35 @@ +--- +name: te-audit-cicd +description: Audit TransformerEngine-FL GitHub Actions, CI configs, QA entrypoints, reusable workflows, local actions, runner labels, and vendor test coverage after an upstream upgrade. Use when .github, qa, workflow matrices, hardware backends, or CI scripts may be stale or silently incomplete. +--- + +# Audit TransformerEngine-FL CI/CD + +Perform a static audit before relying on remote CI. Do not edit workflows or trigger jobs in this skill. + +## Workflow + +1. Read the fork-delta inventory and identify changed workflow, config, script, and QA paths. +2. Parse every YAML workflow/config; report syntax and duplicate-key failures. +3. Resolve local action, reusable-workflow, script, Dockerfile, config, and test-entrypoint references. Compare fork-only workflows against the target tree and require an explicit preserve/remove decision; never assume upstream absence means deletion. +4. Extract vendor/backend and test-group matrices dynamically. Compare supported plugin backends with CI coverage and mark intentional exclusions. +5. Check runner labels, images, permissions, secrets, event triggers, path filters, and failure masking. +6. Run shell syntax checks for referenced scripts and static QA entrypoint checks. +7. Write a coverage matrix and require owner/reason for every missing or blocked path. + +## Outputs + +Write cicd-audit.json, workflow-matrix.tsv, missing-references.tsv, cicd-audit.md, and raw parser logs under /share/project/zhaoyingli/flagos/temp/. + +## Acceptance Criteria + +- Every workflow/config parses or has an owned blocker. +- Every local reference resolves. Fork-only workflows and CI configs remain in the merged tree unless an explicit removal decision has evidence. +- Every discovered plugin backend has CI coverage or an explicit exclusion. +- Vendor matrices and test groups have no unexplained gaps. +- Permissions, triggers, runner labels, and secrets are recorded. +- Referenced shell scripts pass syntax checks or have an owned blocker. +- No workflow silently suppresses test failures. Do not use tree replacement or whole-file theirs resolution on fork-owned workflows. +- Audit is deterministic, main unchanged, and all artifacts are under /share/project/zhaoyingli/flagos/temp, never /tmp. + +Do not infer CI success from YAML syntax alone. diff --git a/skills/te-audit-cicd/agents/openai.yaml b/skills/te-audit-cicd/agents/openai.yaml new file mode 100644 index 0000000000..0dc8c64466 --- /dev/null +++ b/skills/te-audit-cicd/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE CI/CD Auditor" + short_description: "Audit workflows and vendor CI coverage" + default_prompt: "Use $te-audit-cicd to validate TransformerEngine-FL workflows, QA entrypoints, and vendor test coverage." diff --git a/skills/te-audit-cicd/references/rules.md b/skills/te-audit-cicd/references/rules.md new file mode 100644 index 0000000000..6241b89684 --- /dev/null +++ b/skills/te-audit-cicd/references/rules.md @@ -0,0 +1,7 @@ +# CI/CD Audit Rules + +Check uses: ./path, workflow_call workflows, shell commands, and config paths. A path existing locally but not in the target commit is stale. + +Record vendor coverage by backend identity, not workflow filename. Separate unsupported hardware from omitted tests. Inspect continue-on-error, || true, if: always(), and upload-only jobs for failure masking. + +Record permissions and secrets because pull_request and pull_request_target have different trust boundaries. diff --git a/skills/te-audit-cicd/scripts/audit_cicd.py b/skills/te-audit-cicd/scripts/audit_cicd.py new file mode 100755 index 0000000000..da74ee0378 --- /dev/null +++ b/skills/te-audit-cicd/scripts/audit_cicd.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse,json,re,subprocess +from pathlib import Path +def main(): + ap=argparse.ArgumentParser(); ap.add_argument('--repo',type=Path,required=True); ap.add_argument('--ref',required=True); ap.add_argument('--output',type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); a.output.mkdir(parents=True,exist_ok=True) + def run(*x): return subprocess.run(['git','-C',str(repo),*x],text=True,errors='replace',capture_output=True) + ref=run('rev-parse','--verify',a.ref+'^{commit}').stdout.strip() + files=run('ls-tree','-r','--name-only',ref).stdout.splitlines() + wf=[p for p in files if p.startswith('.github/workflows/') and p.endswith(('.yml','.yaml'))] + cfg=[p for p in files if p.startswith('.github/configs/') and p.endswith(('.yml','.yaml'))] + rows=[]; missing=[]; masks=[] + for path in wf+cfg: + text=run('show',ref+':'+path).stdout; rows.append({'path':path,'nonempty':bool(text.strip()),'syntax':'unverified'}) + for local in re.findall(r'uses:\s*\./([^\s#]+)',text): + local=local.rstrip('/') + if local not in files and not any(x.startswith(local+'/') for x in files): missing.append({'source':path,'reference':local}) + masks += re.findall(r'\|\|\s*true|continue-on-error:\s*true',text,re.I) + vendors=sorted({p.split('/')[5] for p in files if p.startswith('transformer_engine/plugin/core/backends/vendor/') and len(p.split('/'))>5 and p.split('/')[5]!='__init__.py'}) + alltext='\n'.join(run('show',ref+':'+p).stdout for p in wf) + coverage=[{'backend':v,'covered':v.lower() in alltext.lower(),'reason':''} for v in vendors] + data={'ref':ref,'workflows':rows,'configs':cfg,'missing_references':missing,'backend_coverage':coverage,'failure_masks':sorted(set(masks))} + (a.output/'cicd-audit.json').write_text(json.dumps(data,indent=2)+chr(10)) + (a.output/'workflow-matrix.tsv').write_text('backend\tcovered\treason\n'+''.join(x['backend']+'\t'+str(x['covered']).lower()+'\t\n' for x in coverage)) + (a.output/'missing-references.tsv').write_text('source\treference\n'+''.join(x['source']+'\t'+x['reference']+'\n' for x in missing)) + (a.output/'cicd-audit.md').write_text('# CI/CD Audit\n\n'+'- Workflows/configs: '+str(len(rows))+'\n- Missing local references: '+str(len(missing))+'\n- Failure masks: '+str(len(set(masks)))+'\n') + print(json.dumps({'output':str(a.output),'workflows':len(rows),'missing':len(missing),'failure_masks':len(set(masks))})) +if __name__=='__main__': main() \ No newline at end of file diff --git a/skills/te-audit-plugin-api/SKILL.md b/skills/te-audit-plugin-api/SKILL.md new file mode 100644 index 0000000000..b99947bb44 --- /dev/null +++ b/skills/te-audit-plugin-api/SKILL.md @@ -0,0 +1,49 @@ +--- +name: te-audit-plugin-api +description: Audit TransformerEngine-FL plugin API compatibility against an upstream release. Use when upstream pybind bindings, Python call sites, enums, dataclasses, or attention signatures change; when adding/removing plugin ops; or when verifying every implementation and registration across dynamically discovered backends. +--- + +# Audit TransformerEngine-FL Plugin API + +Use an isolated tree and exact base, fork, and target refs. This skill audits before edits; it does not invent fallbacks or silently suppress missing APIs. + +## Workflow + +1. Read the fork-delta and conflict inventories. Require their refs to match. +2. Extract target pybind exports, Python-side tex call sites, plugin base methods, registered op names, and backend methods. +3. Compare base and target signatures, including multiline definitions, constructors, enum types, dataclasses, and AttentionParams fields. +4. Build a matrix: symbol, upstream status, plugin base, CUDA/reference/FlagOS/vendor implementations, registration, signature risk, disposition, owner, test. +5. Dynamically discover backends from transformer_engine/plugin/core/backends. Never use a hard-coded vendor list. +6. Classify each gap as required, intentionally unsupported, fallback, or obsolete. Every non-required disposition needs a written reason. +7. Update code only after the matrix is approved. Re-run the audit after each API change. + +## Outputs + +Write under /share/project/zhaoyingli/flagos/temp/: + +- api-inventory.json +- api-matrix.tsv +- api-audit.md +- raw upstream/plugin symbol lists +- decisions.tsv + +## Acceptance Criteria + +- Base/fork/target SHAs match the upstream and conflict inventories. +- Binding extraction includes all binding files and macro-generated exports considered. +- Every target export and Python call site has a matrix row. +- Every matrix row records base/target signature status and enum/dataclass risk. +- Every required symbol has a base stub, implementation, registration, and focused test. +- Every backend is dynamically discovered and has native/fallback/unsupported status per required symbol. +- No unexplained missing, stale, duplicate, or signature-mismatch symbols remain. +- Allowlisted unsupported symbols have owner, reason, and test or blocked evidence. +- Re-running the audit is deterministic and produces identical symbol sets. +- All artifacts are under /share/project/zhaoyingli/flagos/temp; never /tmp. + +Stop on macro-generated exports not accounted for, mismatched refs, ambiguous capability, or unapproved API decisions. + +## Dynamic public-module contract + +When the fork registers `transformer_engine_torch` dynamically, audit that alias as a first-class public surface. Compare Python import-time consumers against both the native binding and the plugin-provided module, including enum member sets, sentinel values, signatures, and required callables. Run the audit in staged imports so the first exception does not hide later incompatibilities. + +Acceptance additionally requires: `transformer_engine.pytorch.constants`, `pytorch.cpp_extensions`, and full `transformer_engine.pytorch` imports pass; all public enum sets used at import time match; every deliberate plugin-only difference has an owner, rationale, and test. diff --git a/skills/te-audit-plugin-api/agents/openai.yaml b/skills/te-audit-plugin-api/agents/openai.yaml new file mode 100644 index 0000000000..f3226ca565 --- /dev/null +++ b/skills/te-audit-plugin-api/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Plugin API Auditor" + short_description: "Audit plugin API and backend coverage" + default_prompt: "Use $te-audit-plugin-api to compare upstream bindings with TransformerEngine-FL plugin coverage." diff --git a/skills/te-audit-plugin-api/references/decision-rules.md b/skills/te-audit-plugin-api/references/decision-rules.md new file mode 100644 index 0000000000..b355791e74 --- /dev/null +++ b/skills/te-audit-plugin-api/references/decision-rules.md @@ -0,0 +1,14 @@ +# API Audit Decision Rules + +A symbol is required when upstream Python code calls it, the fork's plugin contract exposes it, or a supported backend needs it. A pybind export alone may be intentionally unsupported only with an explicit reason. + +Check more than function names: + +- parameter order, defaults, and keyword names; +- enum domains and conversion at the tex boundary; +- dataclass/NamedTuple fields such as AttentionParams; +- return tuple shape and optional values; +- registration priority and availability predicate; +- backend capability and fallback semantics. + +Never use *args/**kwargs to conceal a signature mismatch unless the target CUDA implementation itself uses it and the exception is recorded. diff --git a/skills/te-audit-plugin-api/scripts/audit_plugin_api.py b/skills/te-audit-plugin-api/scripts/audit_plugin_api.py new file mode 100755 index 0000000000..d7505f4ae9 --- /dev/null +++ b/skills/te-audit-plugin-api/scripts/audit_plugin_api.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import argparse,json,re,subprocess +from pathlib import Path +def git(repo,*args): + p=subprocess.run(["git","-C",str(repo),*args],text=True,errors="replace",capture_output=True) + if p.returncode: raise SystemExit(p.stderr.strip() or "git failed") + return p.stdout +def names(text,pattern): + return sorted(set(re.findall(pattern,text,re.M))) +def main(): + ap=argparse.ArgumentParser() + for n in ("base","fork","target"): ap.add_argument("--"+n,required=True) + ap.add_argument("--repo",type=Path,required=True); ap.add_argument("--output",type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); a.output.mkdir(parents=True,exist_ok=True) + refs={n:git(repo,"rev-parse","--verify",getattr(a,n)+"^{commit}").strip() for n in ("base","fork","target")} + def tree(ref,glob): + return git(repo,"grep","-h","-E",glob,ref,"--","transformer_engine/pytorch/csrc",check=False) if False else "" + # Use git grep per ref and retain output even if no match. + def grep(ref,pat,paths): + p=subprocess.run(["git","-C",str(repo),"grep","-h","-E",pat,ref,"--",*paths],text=True,errors="replace",capture_output=True) + return p.stdout + bind=names(grep(refs["target"], r"\.def\(\s*\"[A-Za-z0-9_]+", ["transformer_engine/pytorch/csrc/"]), r"\.def\(\s*\"([A-Za-z0-9_]+)") + pyops=names(grep(refs["fork"],r"^\s*def\s+\w+",["transformer_engine/plugin/core/ops.py"]),r"^\s*def\s+(\w+)") + reg=names(grep(refs["fork"],r'op_name[[:space:]]*=[[:space:]]*"[^"]+',["transformer_engine/plugin/core/backends"]),r'op_name\s*=\s*"([^"]+)"') + files=git(repo,"ls-tree","-r","--name-only",refs["fork"]).splitlines() + pref="transformer_engine/plugin/core/backends/" + backend=sorted({("vendor/"+q[1]) if q[0]=="vendor" and len(q)>1 else q[0] for p in files if p.startswith(pref) for q in [p[len(pref):].split("/")] if q[0] in {"vendor","flagos","reference"} and (q[0]!="vendor" or (len(q)>1 and q[1]!="__init__.py"))}) + rows=[] + for n in sorted(set(bind)|set(pyops)|set(reg)): + rows.append({"symbol":n,"binding":n in bind,"plugin_base":n in pyops,"registered":n in reg,"backends":backend,"disposition":"proposed"}) + data={"resolved_refs":refs,"counts":{"bindings":len(bind),"plugin_base":len(pyops),"registered":len(reg),"matrix":len(rows)},"backends":backend,"symbols":rows} + (a.output/"api-inventory.json").write_text(json.dumps(data,indent=2)+chr(10)) + (a.output/"api-matrix.tsv").write_text("symbol"+chr(9)+"binding"+chr(9)+"plugin_base"+chr(9)+"registered"+chr(9)+"disposition"+chr(9)+"owner"+chr(9)+"test"+chr(10)+"".join(f"{r['symbol']}"+chr(9)+str(r['binding']).lower()+chr(9)+str(r['plugin_base']).lower()+chr(9)+str(r['registered']).lower()+chr(9)+"proposed"+chr(9)+chr(9)+chr(10) for r in rows)) + for fn,items in [("upstream-bindings.txt",bind),("plugin-base-methods.txt",pyops),("plugin-registered-ops.txt",reg)]: (a.output/fn).write_text(chr(10).join(items)+chr(10)) + (a.output/"decisions.tsv").write_text("symbol"+chr(9)+"disposition"+chr(9)+"reason"+chr(9)+"owner"+chr(9)+"test"+chr(9)+"status"+chr(9)+"evidence"+chr(10)) + (a.output/"api-audit.md").write_text("# Plugin API Audit"+chr(10)+chr(10)+f"- Matrix symbols: {len(rows)}"+chr(10)+f"- Backends: {', '.join(backend)}"+chr(10)+f"- Decisions required: {len(rows)}"+chr(10)) + print(json.dumps({"output":str(a.output),"matrix":len(rows),"bindings":len(bind),"backends":backend})) +if __name__=="__main__": main() diff --git a/skills/te-classify-fork-delta/SKILL.md b/skills/te-classify-fork-delta/SKILL.md new file mode 100644 index 0000000000..120c5faaac --- /dev/null +++ b/skills/te-classify-fork-delta/SKILL.md @@ -0,0 +1,59 @@ +--- +name: te-classify-fork-delta +description: Inventory and classify all fork-owned changes before upgrading a TransformerEngine fork. Use when establishing exact upstream base and target refs, producing a complete fork delta manifest, finding files changed by both fork and upstream, discovering plugin backends and CI surfaces, ranking merge risks, or checking that an upgrade plan covers every modified file. +--- + +# Classify TransformerEngine Fork Delta + +Create the authoritative input for later upgrade skills. Do not merge, edit source, create branches, or update submodules while using this skill. + +## Inputs + +Require repository path and exact base, fork, and target refs. Resolve them to commits. Stop if a ref is missing. Record whether base is an ancestor of target. If release histories diverge, require an explicit `--allow-divergent-upstream` decision and record their merge-base; never silently treat a divergent history as linear. + +## Workflow + +1. Record worktree status, branch, remotes, tags, and submodule status without modification. +2. Run `scripts/classify_fork_delta.py` with explicit refs and an untracked output directory. +3. Use `references/classification.md` to review category and priority assignments. +4. Resolve every `unclassified` entry with a narrow, justified rule; never hide it in `other`. +5. Review every `both_changed` path and record its invariant, observing test, and downstream owner. +6. Confirm all discovered backends, CI, QA, tests, build files, packaging files, and submodules have a downstream owner. +7. Present the manifest and risk register for approval. Do not begin integration. + +```bash +python scripts/classify_fork_delta.py --repo /path/to/repo --base --fork --target --allow-divergent-upstream --output /share/project/zhaoyingli/flagos/temp/te-upgrade-inventory +``` + +Outputs are `inventory.json`, `inventory.md`, `fork-changes.tsv`, and `both-changed.tsv`. + +## Priority + +1. P0: plugin contracts, invasive runtime/device patches, or files changed on both sides. +2. P1: build, packaging, submodules, discovery, and tests gating P0 behavior. +3. P2: CI/CD, QA, executable examples, docs, and repository metadata. +4. P3: additive material with no runtime, build, test, or release effect. + +Never lower priority merely because Git predicts a clean merge. + +## Acceptance Criteria + +- Record all refs as full SHAs, ancestry result, and merge-base. Require explicit authorization for divergent upstream tags. +- Record dirty worktree state before analysis. +- Include every `git diff --name-status base..fork` path exactly once. +- Leave zero unclassified paths. +- Include every both-side path in `both-changed.tsv`. +- Discover plugin backends dynamically; use no fixed vendor list. +- List CI, QA, tests, build, packaging, and submodule surfaces even when empty. +- Record base/fork/target gitlink SHAs or explicit absence for every submodule. +- Make JSON, Markdown, and TSV totals agree. +- Obtain user approval before merge or source modification. + +Stop on unresolved refs, uncertain base, failed Git commands, inconsistent totals, or unclassified paths. +## FL design baseline + +When the fork contains a plugin, hardware matrix, or invasive runtime layer, also produce a design baseline from `base..fork` (not only a path manifest). Group paths into plugin contract/core, reference and vendor backends, runtime patches, native build/submodules, CI/CD, QA/tests, and docs/observability. For each group record background, public entry points, call-chain owner, invariants, and an explicit `preserve`, `adapt`, or `drop` decision placeholder. + +The baseline must include dynamic module aliases (especially `sys.modules["transformer_engine_torch"]`) and public enum/callable compatibility. Compare the plugin-provided module against the upstream Python/native public contract before integration. Store the report under the approved artifact directory (never `/tmp`), for example `te-fl-design-baseline--.md`. + +Acceptance: every fork-owned path is assigned to a design group and owner; every group has a documented purpose and acceptance evidence; dynamic aliases and enum members are listed; CI/build/submodule surfaces are represented; the report is referenced by the upgrade decision log. diff --git a/skills/te-classify-fork-delta/agents/openai.yaml b/skills/te-classify-fork-delta/agents/openai.yaml new file mode 100644 index 0000000000..08ad9e975f --- /dev/null +++ b/skills/te-classify-fork-delta/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Fork Delta Classifier" + short_description: "Classify fork changes and upgrade conflict risks" + default_prompt: "Use $te-classify-fork-delta to inventory this TransformerEngine fork before an upstream upgrade." diff --git a/skills/te-classify-fork-delta/references/classification.md b/skills/te-classify-fork-delta/references/classification.md new file mode 100644 index 0000000000..3a8de1c580 --- /dev/null +++ b/skills/te-classify-fork-delta/references/classification.md @@ -0,0 +1,19 @@ +# Classification Reference + +| Category | Scope | Downstream owner | +|---|---|---| +| plugin-core | Plugin framework and registry | Plugin API audit | +| plugin-backend | Backend implementations and registrations | Capability audit | +| invasive-runtime | Modified upstream runtime files | Semantic integration | +| device-abstraction | Device constants and patches | Device audit | +| build-packaging | Setup, manifests, build tools | Build/package audit | +| submodule | Gitmodules and gitlinks | Submodule integration | +| cicd | GitHub automation | CI/CD audit | +| qa | QA entrypoints | QA audit | +| tests | Tests and test utilities | Test matrix audit | +| docs-examples-benchmarks | Support and executable examples | Compatibility audit | +| repository-metadata | Lint, license, ignore, contribution files | Finalization audit | + +P0 affects numerical behavior, dispatch, API/ABI, device placement, backend selection, or is changed on both sides. P1 affects build, install, import, tests, or dependency checkout. P2 affects automation and developer workflows. P3 has no executable or release effect. + +For every both-changed file record: fork behavior, upstream change, invariant, observing test, and downstream owner. diff --git a/skills/te-classify-fork-delta/scripts/classify_fork_delta.py b/skills/te-classify-fork-delta/scripts/classify_fork_delta.py new file mode 100755 index 0000000000..b53fa9c777 --- /dev/null +++ b/skills/te-classify-fork-delta/scripts/classify_fork_delta.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import argparse, json, subprocess +from collections import Counter +from pathlib import Path + +def git(repo,*args,check=True): + p=subprocess.run(["git","-C",str(repo),*args],text=True,capture_output=True) + if check and p.returncode: raise SystemExit(f"git {' '.join(args)} failed: {p.stderr.strip()}") + return p.stdout.rstrip("\n") + +def changes(repo,a,b): + rows=[] + for line in git(repo,"diff","--name-status","--find-renames",f"{a}..{b}").splitlines(): + f=line.split("\t"); rows.append({"status":f[0],"path":f[-1]}) + return rows + +def category(p): + rules=[("plugin-backend",lambda: p.startswith("transformer_engine/plugin/core/backends/")), + ("plugin-core",lambda: p.startswith("transformer_engine/plugin/")), + ("submodule",lambda: p==".gitmodules" or p.startswith("3rdparty/")), + ("cicd",lambda: p.startswith(".github/")),("qa",lambda: p.startswith("qa/")), + ("tests",lambda: p.startswith("tests/")), + ("build-packaging",lambda: p in {"setup.py","pyproject.toml","MANIFEST.in","CMakeLists.txt"} or p.startswith("build_tools/")), + ("device-abstraction",lambda: p=="transformer_engine/__init__.py" or "patches.py" in p), + ("invasive-runtime",lambda: p.startswith(("transformer_engine/pytorch/","transformer_engine/common/","transformer_engine/debug/"))), + ("docs-examples-benchmarks",lambda: p.startswith(("docs/","examples/","benchmarks/"))), + ("repository-metadata",lambda: p in {".gitignore",".pre-commit-config.yaml","README.rst","CONTRIBUTING.rst","CPPLINT.cfg"})] + return next((n for n,test in rules if test()),"unclassified") + +def priority(c,both): + if both or c in {"plugin-core","plugin-backend","invasive-runtime","device-abstraction"}: return "P0" + if c in {"build-packaging","submodule","tests"}: return "P1" + return "P2" if c!="unclassified" else "P3" + +def gitlink(repo,ref,path): + s=git(repo,"ls-tree",ref,"--",path,check=False) + return s.split()[2] if s.startswith("160000 commit ") else None + +def main(): + ap=argparse.ArgumentParser() + for x in ("base","fork","target"): ap.add_argument(f"--{x}",required=True) + ap.add_argument("--repo",type=Path,required=True); ap.add_argument("--output",type=Path,required=True) + ap.add_argument("--allow-divergent-upstream",action="store_true") + a=ap.parse_args(); repo=a.repo.resolve() + refs={x:git(repo,"rev-parse","--verify",f"{getattr(a,x)}^{{commit}}") for x in ("base","fork","target")} + linear=subprocess.run(["git","-C",str(repo),"merge-base","--is-ancestor",refs["base"],refs["target"]]).returncode==0 + common=git(repo,"merge-base",refs["base"],refs["target"]) + if not linear and not a.allow_divergent_upstream: + raise SystemExit(f"base is not an ancestor of target; merge-base={common}; rerun with --allow-divergent-upstream after review") + rows=changes(repo,refs["base"],refs["fork"]); target={r["path"] for r in changes(repo,refs["base"],refs["target"])} + for r in rows: + r.update(category=category(r["path"]),both_changed=r["path"] in target) + r["priority"]=priority(r["category"],r["both_changed"]) + tree=git(repo,"ls-tree","-r","--name-only",refs["fork"]).splitlines() + pref="transformer_engine/plugin/core/backends/" + def backend_id(path): + parts=path[len(pref):].split("/") + if not parts or parts[0] not in {"flagos","reference","vendor"}: return None + return f"vendor/{parts[1]}" if parts[0]=="vendor" and len(parts)>2 and parts[1]!="__init__.py" else (parts[0] if parts[0]!="vendor" else None) + backends=sorted({b for p in tree if p.startswith(pref) for b in [backend_id(p)] if b}) + target_rows=changes(repo,refs["base"],refs["target"]) + smpaths=sorted({r["path"] for r in rows+target_rows if r["path"].startswith("3rdparty/") and any(gitlink(repo,v,r["path"]) for v in refs.values())}) + surfaces={"plugin_backends":backends, + "workflows":sorted(p for p in tree if p.startswith(".github/workflows/")), + "ci_configs":sorted(p for p in tree if p.startswith(".github/configs/")), + "qa":sorted(p for p in tree if p.startswith("qa/")), + "tests":sorted(p for p in tree if p.startswith("tests/")), + "build_packaging":sorted(p for p in tree if category(p)=="build-packaging"), + "submodules":[{"path":p,**{k:gitlink(repo,v,p) for k,v in refs.items()}} for p in smpaths]} + cats=Counter(r["category"] for r in rows); pris=Counter(r["priority"] for r in rows) + a.output.mkdir(parents=True,exist_ok=True) + data={"repo":str(repo),"resolved_refs":refs,"upstream_history":{"linear":linear,"merge_base":common,"divergence_explicitly_allowed":a.allow_divergent_upstream},"dirty_worktree":git(repo,"status","--short").splitlines(), + "fork_change_count":len(rows),"both_changed_count":sum(r["both_changed"] for r in rows), + "category_counts":dict(cats),"priority_counts":dict(pris),"fork_changes":rows,"surfaces":surfaces} + (a.output/"inventory.json").write_text(json.dumps(data,indent=2,sort_keys=True)+"\n") + head="status\tpriority\tcategory\tboth_changed\tpath\n" + lines=[f"{r['status']}\t{r['priority']}\t{r['category']}\t{str(r['both_changed']).lower()}\t{r['path']}\n" for r in rows] + (a.output/"fork-changes.tsv").write_text(head+"".join(lines)) + (a.output/"both-changed.tsv").write_text(head+"".join(x for x,r in zip(lines,rows) if r["both_changed"])) + md=["# TransformerEngine Fork Delta Inventory","",*[f"- {k.title()}: `{v}`" for k,v in refs.items()], + f"- Fork-owned paths: {len(rows)}",f"- Both-changed paths: {data['both_changed_count']}","", + "## Categories","","| Category | Count |","|---|---:|",*[f"| {k} | {v} |" for k,v in sorted(cats.items())], + "","## Acceptance blockers","",f"- Unclassified paths: {cats.get('unclassified',0)}"] + (a.output/"inventory.md").write_text("\n".join(md)+"\n") + print(json.dumps({"output":str(a.output),"fork_changes":len(rows),"both_changed":data["both_changed_count"],"unclassified":cats.get("unclassified",0)})) + if not rows or cats.get("unclassified"): raise SystemExit(2) + +if __name__=="__main__": main() diff --git a/skills/te-finalize-upstream-upgrade/SKILL.md b/skills/te-finalize-upstream-upgrade/SKILL.md new file mode 100644 index 0000000000..781ecd5f2a --- /dev/null +++ b/skills/te-finalize-upstream-upgrade/SKILL.md @@ -0,0 +1,38 @@ +--- +name: te-finalize-upstream-upgrade +description: Finalize and hand off a TransformerEngine-FL upstream upgrade after conflict, plugin API, runtime patch, build, CI, and test audits. Use for evidence completeness, unresolved blocker review, commit/branch hygiene, rollback planning, release notes, or preparing a PR without pushing automatically. +--- + +# Finalize TransformerEngine-FL Upgrade + +Treat finalization as an evidence gate, not a formatting step. + +## Workflow + +1. Require all phase inventories and their exact commit refs. +2. Verify each acceptance criterion and classify it pass, fail, blocked, or not-applicable. +3. Check no unresolved P0 decisions, missing API dispositions, unowned CI gaps, or unexplained blocked tests remain. +4. Review worktree, branch ancestry, diff scope, generated artifacts, submodule gitlinks, and commit messages. +5. Produce a rollback plan naming the merge/upgrade commits and a recoverable command. +6. Produce a handoff report with changed surfaces, test evidence, known limitations, and explicit user decisions required. +7. Stop before push, PR creation, tag, or main mutation; request separate authorization. + +## Outputs + +Write final-report.md, evidence-index.json, blockers.tsv, and rollback-plan.md under /share/project/zhaoyingli/flagos/temp/. + +## Acceptance Criteria + +- All phase artifacts exist and refs agree. +- Every phase has a status and evidence path. +- No unresolved P0 item is marked pass. +- Every blocked non-NVIDIA test has owner and reason. +- Worktree and main status are recorded. +- Diff contains no unapproved generated artifact. +- Rollback command is explicit and non-destructive. +- Push/PR/tag actions are not performed without separate approval. +- All output is under /share/project/zhaoyingli/flagos/temp, never /tmp. + +## Design decision ledger + +The handoff must reference the FL design baseline and include a `preserve/adapt/drop` decision for each design group: plugin core, each backend family, runtime patch chain, native build/submodules, CI matrix, QA/tests, and observability/docs. No group may be omitted because Git merged it cleanly. diff --git a/skills/te-finalize-upstream-upgrade/agents/openai.yaml b/skills/te-finalize-upstream-upgrade/agents/openai.yaml new file mode 100644 index 0000000000..c8c2e581e7 --- /dev/null +++ b/skills/te-finalize-upstream-upgrade/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Upgrade Finalizer" + short_description: "Finalize upgrade evidence and rollback plan" + default_prompt: "Use $te-finalize-upstream-upgrade to audit upgrade evidence and prepare a safe handoff." diff --git a/skills/te-finalize-upstream-upgrade/references/fields.md b/skills/te-finalize-upstream-upgrade/references/fields.md new file mode 100644 index 0000000000..33861e0124 --- /dev/null +++ b/skills/te-finalize-upstream-upgrade/references/fields.md @@ -0,0 +1,5 @@ +# Final Handoff Fields + +Include target upstream ref/SHA, fork base/fork SHAs, changed file counts, plugin/API status, runtime patch status, build/submodule decisions, CI coverage, test pass/fail/blocked counts, known limitations, unresolved decisions, rollback commit, and next authorized action. + +A blocked hardware test remains a limitation even when all available NVIDIA tests pass. diff --git a/skills/te-finalize-upstream-upgrade/scripts/finalize_upgrade.py b/skills/te-finalize-upstream-upgrade/scripts/finalize_upgrade.py new file mode 100755 index 0000000000..223831becc --- /dev/null +++ b/skills/te-finalize-upstream-upgrade/scripts/finalize_upgrade.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import argparse,json,subprocess +from pathlib import Path +def main(): + ap=argparse.ArgumentParser(); ap.add_argument('--repo',type=Path,required=True); ap.add_argument('--output',type=Path,required=True); ap.add_argument('--artifacts',type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); out=a.output.resolve(); out.mkdir(parents=True,exist_ok=True) + def run(*x): return subprocess.run(['git','-C',str(repo),*x],text=True,errors='replace',capture_output=True) + expected=['inventory.json','conflict-inventory.json','api-inventory.json','runtime-patch-ledger.json','build-audit.json','cicd-audit.json','test-matrix.json'] + found={n:(a.artifacts/n).exists() for n in expected} + status=run('status','--short').stdout.splitlines(); branch=run('branch','--show-current').stdout.strip() + data={'repo':str(repo),'branch':branch,'dirty_worktree':status,'artifacts':found,'all_artifacts_present':all(found.values()),'push_performed':False,'pr_created':False} + (out/'evidence-index.json').write_text(json.dumps(data,indent=2)+chr(10)) + (out/'blockers.tsv').write_text('item\tstatus\towner\tevidence\n'+('missing phase artifacts\tblocked\tuser\t'+str(a.artifacts)+'\n' if not all(found.values()) else '')) + (out/'rollback-plan.md').write_text('# Rollback Plan\n\nRecord the final merge commit before applying any revert. Use git revert -m 1 after review. Do not reset or delete branches.\n') + (out/'final-report.md').write_text('# Upgrade Handoff\n\n'+'- Branch: '+branch+'\n- Dirty worktree entries: '+str(len(status))+'\n- All phase artifacts present: '+str(all(found.values()))+'\n- Push/PR performed: false\n') + print(json.dumps({'output':str(out),'all_artifacts_present':all(found.values()),'dirty_entries':len(status)})) +if __name__=='__main__': main() \ No newline at end of file diff --git a/skills/te-integrate-build-submodules/SKILL.md b/skills/te-integrate-build-submodules/SKILL.md new file mode 100644 index 0000000000..81349508cf --- /dev/null +++ b/skills/te-integrate-build-submodules/SKILL.md @@ -0,0 +1,36 @@ +--- +name: te-integrate-build-submodules +description: Audit and integrate TransformerEngine-FL build, packaging, wheel, CMake, setup, and third-party submodule changes across an upstream upgrade. Use when setup.py, pyproject.toml, build_tools, MANIFEST.in, .gitmodules, 3rdparty gitlinks, native extensions, or install/import behavior changes. +--- + +# Integrate Build and Submodules + +Audit the build graph before source integration. Do not update gitlinks or build artifacts without an explicit three-way decision. + +## Workflow + +1. Require matching fork-delta and conflict inventories. +2. Compare base, fork, and target versions of setup.py, pyproject.toml, MANIFEST.in, build_tools, CMake files, package data, and extension source lists. +3. Extract plugin compilation targets, include paths, libraries, device guards, version metadata, and wheel package inclusion. Mark fork-only build and workflow files as preserved content; do not replace them with upstream trees. +4. Compare .gitmodules and every 3rdparty gitlink at base, fork, and target. Record absent, added, removed, and changed submodules. +5. Build a decision matrix: path/gitlink, fork requirement, upstream change, selected value, compatibility risk, owner, and validation command. +6. In an isolated environment run static packaging checks, editable build/import, and wheel content inspection. Use the approved GPU machine/environment only when a CUDA extension build is required. +7. Record blocked dependency, compiler, CUDA, or network conditions explicitly. + +## Outputs + +Write build-audit.json, build-matrix.tsv, submodule-matrix.tsv, build-audit.md, and logs under /share/project/zhaoyingli/flagos/temp/. + +## Acceptance Criteria + +- Every fork-modified build/package file has one matrix row. +- Every submodule path in any of base/fork/target has base/fork/target gitlink or explicit absent value. +- Plugin source files, extension targets, include paths, package data, and version metadata are all accounted for. +- .gitmodules entries and checkout URLs are valid. +- No unexpected generated files enter the source diff. Fork-only build/CI files remain present after merge unless an explicit removal decision is recorded. +- Editable install/import and wheel manifest checks pass, or have an owned blocked reason. +- Selected submodule commits are reachable and reproducible. +- Build artifacts are not committed. +- Main is unchanged and all artifacts are under /share/project/zhaoyingli/flagos/temp, never /tmp. + +Never assume a clean Python import proves native extensions are correctly built. diff --git a/skills/te-integrate-build-submodules/agents/openai.yaml b/skills/te-integrate-build-submodules/agents/openai.yaml new file mode 100644 index 0000000000..291e77f475 --- /dev/null +++ b/skills/te-integrate-build-submodules/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Build and Submodule Integrator" + short_description: "Audit build packaging and submodule upgrades" + default_prompt: "Use $te-integrate-build-submodules to audit TransformerEngine-FL build, wheel, and submodule integration." diff --git a/skills/te-integrate-build-submodules/references/rules.md b/skills/te-integrate-build-submodules/references/rules.md new file mode 100644 index 0000000000..38b9ac13af --- /dev/null +++ b/skills/te-integrate-build-submodules/references/rules.md @@ -0,0 +1,12 @@ +# Build and Submodule Decision Rules + +For each build file check both fork additions and upstream changes: + +- extension source lists and compile definitions; +- include/library directories and device guards; +- plugin package discovery and package data; +- version source and wheel metadata; +- editable install and isolated wheel behavior; +- optional dependencies and import-time loading. + +For each gitlink choose upstream, fork, or a reviewed third value. Record why. A missing target gitlink can mean removal, not an error; distinguish it from a checkout failure. diff --git a/skills/te-integrate-build-submodules/scripts/audit_build_submodules.py b/skills/te-integrate-build-submodules/scripts/audit_build_submodules.py new file mode 100755 index 0000000000..99870d4ebe --- /dev/null +++ b/skills/te-integrate-build-submodules/scripts/audit_build_submodules.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import argparse,json,subprocess +from pathlib import Path +def run(repo,*args,check=True): + p=subprocess.run(["git","-C",str(repo),*args],text=True,errors="replace",capture_output=True) + if check and p.returncode: raise SystemExit(p.stderr.strip() or "git failed") + return p.stdout +def link(repo,ref,path): + s=run(repo,"ls-tree",ref,"--",path,check=False) + return s.split()[2] if s.startswith("160000 commit ") else None +def main(): + ap=argparse.ArgumentParser() + for n in ("base","fork","target"): ap.add_argument("--"+n,required=True) + ap.add_argument("--repo",type=Path,required=True); ap.add_argument("--output",type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); a.output.mkdir(parents=True,exist_ok=True) + refs={n:run(repo,"rev-parse","--verify",getattr(a,n)+"^{commit}").strip() for n in ("base","fork","target")} + changed=set(run(repo,"diff","--name-only",refs["base"]+".."+refs["fork"]).splitlines()) + build=sorted(p for p in changed if p in {"setup.py","pyproject.toml","MANIFEST.in","CMakeLists.txt"} or p.startswith("build_tools/") or p.startswith("transformer_engine/pytorch/setup.py")) + tree=set(run(repo,"ls-tree","-r","--name-only",refs["fork"]).splitlines()) + smpaths=sorted(p for p in set(tree)|changed if p==".gitmodules" or p.startswith("3rdparty/") if any(link(repo,r,p) for r in refs.values()) or p==".gitmodules") + sm=[{"path":p,**{n:link(repo,r,p) for n,r in refs.items()},"status":"proposed"} for p in smpaths] + plugins=[p for p in tree if p.startswith("transformer_engine/plugin/")] + data={"resolved_refs":refs,"build_files":build,"build_file_count":len(build),"plugin_file_count":len(plugins),"submodules":sm} + (a.output/"build-audit.json").write_text(json.dumps(data,indent=2)+chr(10)) + (a.output/"build-matrix.tsv").write_text("path\tplugin_target\tinclude_paths\tpackage_data\tversion\tstatus\n"+"".join(f"{p}\tproposed\tproposed\tproposed\tproposed\tproposed\n" for p in build)) + (a.output/"submodule-matrix.tsv").write_text("path\tbase\tfork\ttarget\tselection\treason\tstatus\n"+"".join(f"{x['path']}\t{x['base']}\t{x['fork']}\t{x['target']}\t\t\tproposed\n" for x in sm)) + (a.output/"build-audit.md").write_text("# Build and Submodule Audit"+chr(10)+chr(10)+f"- Build files: {len(build)}"+chr(10)+f"- Plugin files: {len(plugins)}"+chr(10)+f"- Submodule paths: {len(sm)}"+chr(10)) + print(json.dumps({"output":str(a.output),"build_files":len(build),"plugin_files":len(plugins),"submodules":len(sm)})) +if __name__=="__main__": main() diff --git a/skills/te-integrate-upstream-conflicts/SKILL.md b/skills/te-integrate-upstream-conflicts/SKILL.md new file mode 100644 index 0000000000..7706e0a43f --- /dev/null +++ b/skills/te-integrate-upstream-conflicts/SKILL.md @@ -0,0 +1,33 @@ +--- +name: te-integrate-upstream-conflicts +description: Analyze and safely resolve files changed by both TransformerEngine-FL and NVIDIA upstream during an upgrade. Use for three-way merge planning, conflict classification, semantic preservation of plugin dispatch and invasive runtime patches, dry-run merge validation, or reviewing whether every conflict has an owner and acceptance test. +--- + +# Integrate Upstream Conflicts + +Operate on an isolated worktree or branch. Never modify main, push, or resolve conflicts by blindly choosing ours/theirs. + +## Workflow + +1. Require the classifier inventory and exact base, fork, and target SHAs. +2. Run scripts/conflict_inventory.py and write outputs under /share/project/zhaoyingli/flagos/temp. +3. For every both-changed path, inspect base, fork, and target versions. Record fork invariant, upstream change, resolution strategy, owner, and acceptance test. +4. Classify each path P0 (plugin/runtime/device), P1 (build/package/submodule/tests), or P2 (CI/QA/docs/metadata). Treat fork-only additions as owned content: they are not conflicts, but they must be preserved unless an explicit decision removes them. +5. Simulate the merge before editing. Record textual conflicts separately from clean-but-semantic conflicts. +6. Require explicit approval for each P0 strategy. Use manual edits and narrow commits; never use blanket ours/theirs. +7. After resolution, run conflict-marker, diff-scope, and invariant checks. + +## Acceptance Criteria + +- Every both-changed path appears exactly once. +- Every path has priority, owner, strategy, acceptance test, and status. +- P0 paths cannot be resolved without explicit user approval. +- Textual conflicts and clean-but-semantic risks are both reported. +- No blanket ours/theirs operation is used, and no whole-file theirs/tree-replacement operation is used on fork-owned plugin, runtime, build, or CI files. +- No conflict markers remain in resolved files. +- Resolved diff contains no unexpected path outside the approved manifest. +- Each P0 invariant has a passing focused test or recorded blocked reason. Module-level plugin dispatch blocks must retain their imports, saved native fallback, original callback, replacement callback, and call sites as one unit. +- Worktree is isolated and main is unchanged. +- All artifacts are under /share/project/zhaoyingli/flagos/temp, never /tmp. + +Stop on missing inventory, ambiguous ownership, unapproved P0 decisions, unexpected files, or unavailable test evidence. diff --git a/skills/te-integrate-upstream-conflicts/agents/openai.yaml b/skills/te-integrate-upstream-conflicts/agents/openai.yaml new file mode 100644 index 0000000000..08d12320b1 --- /dev/null +++ b/skills/te-integrate-upstream-conflicts/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Upstream Conflict Integrator" + short_description: "Resolve upstream conflicts with semantic safeguards" + default_prompt: "Use $te-integrate-upstream-conflicts to classify and safely resolve TransformerEngine upgrade conflicts." diff --git a/skills/te-integrate-upstream-conflicts/references/decision-schema.md b/skills/te-integrate-upstream-conflicts/references/decision-schema.md new file mode 100644 index 0000000000..596cfcc352 --- /dev/null +++ b/skills/te-integrate-upstream-conflicts/references/decision-schema.md @@ -0,0 +1,17 @@ +# Conflict Decision Schema + +A decision is complete only when it answers: + +| Field | Requirement | +|---|---| +| path | Exact repository-relative path | +| priority | P0/P1/P2 with evidence | +| fork invariant | Behavior that must survive | +| upstream change | Behavior introduced after base | +| strategy | Manual merge, fork-preserve, upstream-preserve, or redesign | +| owner | Named person/agent or next skill | +| acceptance | Exact command or test | +| status | Proposed, approved, resolved, or blocked | +| evidence | Diff, log, or test artifact under the temp directory | + +Clean Git merges still require a semantic decision when both sides changed the path. diff --git a/skills/te-integrate-upstream-conflicts/scripts/conflict_inventory.py b/skills/te-integrate-upstream-conflicts/scripts/conflict_inventory.py new file mode 100755 index 0000000000..d2a0bbb4d0 --- /dev/null +++ b/skills/te-integrate-upstream-conflicts/scripts/conflict_inventory.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import argparse,json,subprocess +from pathlib import Path +def run(repo,*args): + p=subprocess.run(["git","-C",str(repo),*args],text=True,errors="replace",capture_output=True) + if p.returncode: raise SystemExit(p.stderr.strip() or "git command failed") + return p.stdout +def main(): + ap=argparse.ArgumentParser() + for n in ("base","fork","target"): ap.add_argument("--"+n,required=True) + ap.add_argument("--repo",type=Path,required=True); ap.add_argument("--output",type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); a.output.mkdir(parents=True,exist_ok=True) + refs={n:run(repo,"rev-parse","--verify",getattr(a,n)+"^{commit}").strip() for n in ("base","fork","target")} + def paths(right): return {x for x in run(repo,"diff","--name-only",refs["base"]+".."+right).splitlines() if x} + both=sorted(paths(refs["fork"]) & paths(refs["target"])) + merge=run(repo,"merge-tree",refs["base"],refs["fork"],refs["target"]) + (a.output/"merge-tree.txt").write_text(merge) + conflict_paths={line.rsplit(" ",1)[-1] for line in merge.splitlines() if "CONFLICT" in line and " " in line} + def priority(p): + if p.startswith(("transformer_engine/plugin/","transformer_engine/pytorch/","transformer_engine/common/","transformer_engine/debug/")) or p=="transformer_engine/__init__.py": return "P0" + if p.startswith(("setup.py","build_tools/","3rdparty/","tests/","qa/")): return "P1" + return "P2" + rows=[{"path":p,"priority":priority(p),"textual_conflict":p in conflict_paths,"semantic_risk":True,"status":"proposed"} for p in both] + data={"resolved_refs":refs,"both_changed_count":len(rows),"textual_conflict_count":len(conflict_paths),"paths":rows} + (a.output/"conflict-inventory.json").write_text(json.dumps(data,indent=2)+"\n") + (a.output/"conflict-inventory.md").write_text("# Conflict Inventory\n\n"+f"- Both-changed paths: {len(rows)}\n\n"+"\n".join(f"- [{r['priority']}] {r['path']} — proposed" for r in rows)+"\n") + (a.output/"decisions.tsv").write_text("path\tpriority\tfork_invariant\tupstream_change\tstrategy\towner\tacceptance\tstatus\tevidence\n") + print(json.dumps({"output":str(a.output),"both_changed":len(rows)})) +if __name__=="__main__": main() diff --git a/skills/te-preserve-runtime-patches/SKILL.md b/skills/te-preserve-runtime-patches/SKILL.md new file mode 100644 index 0000000000..3f08b6c404 --- /dev/null +++ b/skills/te-preserve-runtime-patches/SKILL.md @@ -0,0 +1,40 @@ +--- +name: te-preserve-runtime-patches +description: Audit and preserve invasive TransformerEngine-FL runtime changes across an upstream merge. Use when upstream modifies transformer_engine/pytorch, transformer_engine/__init__.py, common, debug, device selection, plugin dispatch, tensor/attention wrappers, or vendor patches and the fork must retain behavior across CUDA and non-CUDA backends. +--- + +# Preserve TransformerEngine-FL Runtime Patches + +Treat fork changes to upstream-owned runtime files as semantic patches, not ordinary additions. Audit before editing and require a focused acceptance test for every invariant. + +## Workflow + +1. Require matching fork-delta, conflict, and API inventories. +2. Identify fork-only and both-changed runtime files from the inventories. +3. Extract fork invariants: plugin dispatch calls, TE_DEVICE_TYPE/device substitutions, non-CUDA guards, vendor selection, tensor/attention argument forwarding, and import/initialization ordering. +4. Compare base, fork, and target call sites. Search target for new hard-coded CUDA behavior, renamed imports, changed signatures, and altered defaults. For every fork-added symbol, verify import, definition, and call/reference coupling; a call such as te_device_type() is invalid if its import was removed. +5. Create a patch ledger with path, invariant, upstream change, preservation method, test, owner, and status. +6. Apply changes only in an isolated branch. Keep upstream functional changes while reintroducing fork dispatch at the narrowest boundary. +7. Run static scans, import tests, plugin dispatch tests, and at least one representative non-CUDA path when hardware is available. + +## Outputs + +Write runtime-patch-ledger.json, runtime-patch-ledger.tsv, runtime-patch-audit.md, and raw scan logs under /share/project/zhaoyingli/flagos/temp/. + +## Acceptance Criteria + +- Every fork-modified runtime file appears exactly once in the ledger. +- Every both-changed runtime file has an explicit invariant and preservation decision. +- Every plugin dispatch and device abstraction marker is accounted for. Module-level dispatch blocks are retained atomically: native fallback alias, plugin replacement, original callback save, replacement callback, and required imports. +- No unexplained new hard-coded CUDA path remains in target-derived runtime code. +- Import order and optional vendor dependencies are tested. +- Python signatures and forwarded keyword arguments match target callers. Every te_device_type() call has a valid import and a focused assertion/device test. +- CUDA and at least one non-CUDA backend have evidence, or a blocked reason with owner. +- No conflict markers or unrelated files enter the patch. +- Main remains unchanged and all artifacts are under /share/project/zhaoyingli/flagos/temp, never /tmp. + +Never mass-replace cuda, keep fork patches only by line count, or use blanket ours/theirs resolution. + +## Cross-module patch chains + +Treat runtime changes as chains of definition, import, alias registration, and call site. Include `sys.modules` substitutions and compatibility modules in the ledger. A patch is not preserved until its downstream import-time consumers load successfully; record the chain owner and staged import evidence. diff --git a/skills/te-preserve-runtime-patches/agents/openai.yaml b/skills/te-preserve-runtime-patches/agents/openai.yaml new file mode 100644 index 0000000000..b5e817b6b5 --- /dev/null +++ b/skills/te-preserve-runtime-patches/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Runtime Patch Protector" + short_description: "Protect invasive runtime and device patches" + default_prompt: "Use $te-preserve-runtime-patches to audit TransformerEngine-FL invasive patches after an upstream merge." diff --git a/skills/te-preserve-runtime-patches/references/invariants.md b/skills/te-preserve-runtime-patches/references/invariants.md new file mode 100644 index 0000000000..cf3e4cb9de --- /dev/null +++ b/skills/te-preserve-runtime-patches/references/invariants.md @@ -0,0 +1,5 @@ +# Runtime Patch Invariants + +Look for plugin dispatch, TE_DEVICE_TYPE and device-agnostic tensor allocation, import-time patching and optional dependency guards, attention/tensor/module/optimizer argument forwarding, vendor-specific patches and fallback behavior, and runtime environment variables. + +A textual merge can be clean while deleting a dispatch call or changing a keyword name. Compare call sites and behavior, not only conflict markers. diff --git a/skills/te-preserve-runtime-patches/scripts/audit_runtime_patches.py b/skills/te-preserve-runtime-patches/scripts/audit_runtime_patches.py new file mode 100755 index 0000000000..704400ac85 --- /dev/null +++ b/skills/te-preserve-runtime-patches/scripts/audit_runtime_patches.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import argparse,json,subprocess +from pathlib import Path +def run(repo,*args,check=True): + p=subprocess.run(["git","-C",str(repo),*args],text=True,errors="replace",capture_output=True) + if check and p.returncode: raise SystemExit(p.stderr.strip() or "git failed") + return p.stdout +def main(): + ap=argparse.ArgumentParser() + for n in ("base","fork","target"): ap.add_argument("--"+n,required=True) + ap.add_argument("--repo",type=Path,required=True); ap.add_argument("--output",type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); a.output.mkdir(parents=True,exist_ok=True) + refs={n:run(repo,"rev-parse","--verify",getattr(a,n)+"^{commit}").strip() for n in ("base","fork","target")} + changed=run(repo,"diff","--name-only",refs["base"]+".."+refs["fork"]).splitlines() + runtime=[p for p in changed if p=="transformer_engine/__init__.py" or p.startswith(("transformer_engine/pytorch/","transformer_engine/common/","transformer_engine/debug/"))] + targetfiles=set(run(repo,"diff","--name-only",refs["base"]+".."+refs["target"]).splitlines()) + both=[p for p in runtime if p in targetfiles] + rows=[] + for p in sorted(runtime): + text=run(repo,"show",refs["fork"]+":"+p,check=False) + markers=[m for m in ("TE_DEVICE_TYPE","transformer_engine.plugin","plugin.ops","cuda","device_type","register_ops") if m in text] + rows.append({"path":p,"both_changed":p in both,"markers":markers,"status":"proposed","invariant":"","preservation":"","test":""}) + data={"resolved_refs":refs,"runtime_file_count":len(rows),"both_changed_count":len(both),"paths":rows} + (a.output/"runtime-patch-ledger.json").write_text(json.dumps(data,indent=2)+chr(10)) + (a.output/"runtime-patch-ledger.tsv").write_text("path"+chr(9)+"both_changed"+chr(9)+"markers"+chr(9)+"invariant"+chr(9)+"preservation"+chr(9)+"test"+chr(9)+"status"+chr(10)+"".join(f"{r['path']}"+chr(9)+str(r['both_changed']).lower()+chr(9)+",".join(r['markers'])+chr(9)+chr(9)+chr(9)+chr(9)+"proposed"+chr(10) for r in rows)) + (a.output/"runtime-patch-audit.md").write_text("# Runtime Patch Audit"+chr(10)+chr(10)+f"- Runtime files: {len(rows)}"+chr(10)+f"- Both changed: {len(both)}"+chr(10)+f"- Decisions required: {len(rows)}"+chr(10)) + (a.output/"raw-runtime-files.txt").write_text(chr(10).join(sorted(runtime))+chr(10)) + print(json.dumps({"output":str(a.output),"runtime_files":len(rows),"both_changed":len(both)})) +if __name__=="__main__": main() diff --git a/skills/te-run-upgrade-test-matrix/SKILL.md b/skills/te-run-upgrade-test-matrix/SKILL.md new file mode 100644 index 0000000000..1f7a4d7742 --- /dev/null +++ b/skills/te-run-upgrade-test-matrix/SKILL.md @@ -0,0 +1,43 @@ +--- +name: te-run-upgrade-test-matrix +description: Plan, execute, and report TransformerEngine-FL upstream-upgrade tests by capability and hardware. Use after merge/plugin/build audits for plugin tests, Python/C++ tests, NVIDIA GPU validation, vendor backend checks, CI/QA entrypoints, and FlagScale E2E; explicitly record blocked non-NVIDIA tests instead of treating them as passed. +--- + +# Run TransformerEngine-FL Upgrade Test Matrix + +Separate test selection from execution and never hide unavailable hardware. + +## Workflow + +1. Require completed conflict, API, runtime, build, and CI inventories. +2. Generate a matrix of static, CPU/import, NVIDIA CUDA, plugin, and vendor-specific tests. +3. Mark each row runnable, blocked, skipped-by-policy, or not-applicable with reason and owner. +4. Run cheap static checks first, then install/import, plugin tests, focused CUDA tests, broader CUDA tests, and finally FlagScale E2E. +5. Use the approved NVIDIA machine and conda environment only for GPU-required rows. Do not claim non-NVIDIA rows passed on that host. +6. Capture command, commit, environment, start/end time, exit code, log path, and artifact path for every row. +7. Re-run failed rows only after recording diagnosis and a scoped change. + +## Hardware Policy + +The available GPU host is NVIDIA-only. CUDA rows may be runnable there. Hygon, MUSA, NPU, Ascend, Kunlunxin, Enflame, MetaX, Iluvatar, Tsingmicro, and other non-NVIDIA native rows must be blocked or static-only unless a matching host is later authorized. + +## Outputs + +Write test-matrix.json, test-matrix.tsv, test-report.md, environment.txt, and per-row logs under /share/project/zhaoyingli/flagos/temp/. + +## Acceptance Criteria + +- Every discovered test group has one matrix row and hardware classification. +- Static and NVIDIA-runnable rows have command, result, and log evidence. +- Every blocked non-NVIDIA row has explicit reason and owner. +- No skipped or blocked row is counted as pass. +- Tests run against the recorded commit and environment. +- Failed tests retain first failure logs and diagnosis. +- Focused plugin/API tests run before broad E2E. +- Main remains unchanged and all artifacts are under /share/project/zhaoyingli/flagos/temp, never /tmp. + +Never run destructive cleanup or alter test fixtures to make a row pass. + +## Import-contract smoke tier + +Before GPU kernels, run a staged public-contract smoke tier: native/core import, dynamic `transformer_engine_torch` registration, enum/callable comparison, `pytorch.constants`, `pytorch.cpp_extensions`, and full PyTorch import. Record the exact module path and enum sets. diff --git a/skills/te-run-upgrade-test-matrix/agents/openai.yaml b/skills/te-run-upgrade-test-matrix/agents/openai.yaml new file mode 100644 index 0000000000..1bef4e5d68 --- /dev/null +++ b/skills/te-run-upgrade-test-matrix/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Upgrade Test Matrix" + short_description: "Run and report upgrade test coverage" + default_prompt: "Use $te-run-upgrade-test-matrix to execute and report TransformerEngine-FL upgrade tests by capability." diff --git a/skills/te-run-upgrade-test-matrix/references/rules.md b/skills/te-run-upgrade-test-matrix/references/rules.md new file mode 100644 index 0000000000..3a3bd9ce9c --- /dev/null +++ b/skills/te-run-upgrade-test-matrix/references/rules.md @@ -0,0 +1,12 @@ +# Test Matrix Rules + +Recommended order: + +1. Python syntax, YAML/workflow and stale-reference scans. +2. Editable install/import and package manifest. +3. Plugin lifecycle, policy, registry, and backend unit tests. +4. NVIDIA CUDA focused operator/API tests. +5. NVIDIA broader QA and integration tests. +6. FlagScale single-config smoke test, then batch E2E. + +Use statuses pass, fail, blocked, skipped, and not-applicable. A blocked hardware test is evidence of missing coverage, not success. diff --git a/skills/te-run-upgrade-test-matrix/scripts/generate_test_matrix.py b/skills/te-run-upgrade-test-matrix/scripts/generate_test_matrix.py new file mode 100755 index 0000000000..19526c2ccc --- /dev/null +++ b/skills/te-run-upgrade-test-matrix/scripts/generate_test_matrix.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import argparse,json,subprocess +from pathlib import Path +def main(): + ap=argparse.ArgumentParser(); ap.add_argument('--repo',type=Path,required=True); ap.add_argument('--ref',required=True); ap.add_argument('--output',type=Path,required=True) + a=ap.parse_args(); repo=a.repo.resolve(); a.output.mkdir(parents=True,exist_ok=True) + def run(*x): return subprocess.run(['git','-C',str(repo),*x],text=True,errors='replace',capture_output=True) + ref=run('rev-parse','--verify',a.ref+'^{commit}').stdout.strip(); files=run('ls-tree','-r','--name-only',ref).stdout.splitlines() + groups=[] + groups.append({'name':'python-static','hardware':'none','status':'runnable','command':'python -m compileall transformer_engine'}) + groups.append({'name':'plugin-tests','hardware':'NVIDIA-or-CPU','status':'runnable','command':'pytest tests/plugin -q'}) + groups.append({'name':'cuda-qa','hardware':'NVIDIA-CUDA','status':'runnable','command':'bash qa/L0_pytorch_unittest/test.sh'}) + groups.append({'name':'flagscale-smoke','hardware':'NVIDIA-CUDA','status':'runnable','command':'FlagScale single-config smoke'}) + vendors=sorted({p.split('/')[5] for p in files if p.startswith('transformer_engine/plugin/core/backends/vendor/') and len(p.split('/'))>5 and p.split('/')[5]!='__init__.py'}) + native={'cuda'} + for v in vendors: + if v not in native: groups.append({'name':'vendor-'+v,'hardware':v,'status':'blocked','reason':'Only NVIDIA GPU host authorized','owner':'user','command':''}) + data={'ref':ref,'groups':groups,'counts':{'total':len(groups),'blocked':sum(x['status']=='blocked' for x in groups)}} + (a.output/'test-matrix.json').write_text(json.dumps(data,indent=2)+chr(10)) + (a.output/'test-matrix.tsv').write_text('name\thardware\tstatus\tcommand\treason\towner\n'+''.join(x['name']+'\t'+x['hardware']+'\t'+x['status']+'\t'+x.get('command','')+'\t'+x.get('reason','')+'\t'+x.get('owner','')+'\n' for x in groups)) + (a.output/'test-report.md').write_text('# Test Matrix\n\n'+'- Total: '+str(len(groups))+'\n- Blocked: '+str(data['counts']['blocked'])+'\n') + print(json.dumps({'output':str(a.output),'total':len(groups),'blocked':data['counts']['blocked']})) +if __name__=='__main__': main() \ No newline at end of file diff --git a/skills/te-upgrade-orchestrator/SKILL.md b/skills/te-upgrade-orchestrator/SKILL.md new file mode 100644 index 0000000000..c9f52ee739 --- /dev/null +++ b/skills/te-upgrade-orchestrator/SKILL.md @@ -0,0 +1,52 @@ +--- +name: te-upgrade-orchestrator +description: Orchestrate the gated TransformerEngine-FL upstream upgrade workflow across fork inventory, conflict, plugin API, runtime patch, build/submodule, CI/CD, test matrix, and finalization skills. Use when starting or resuming a complete upstream upgrade and when every phase needs explicit inputs, evidence, approvals, and stop conditions. +--- + +# Orchestrate TransformerEngine-FL Upgrade + +Use the phase skills in dependency order. The orchestrator coordinates; specialized skills perform analysis and tests. + +## Phase Order + +1. Classify fork delta and produce the FL design baseline. +2. Integrate upstream conflicts. +3. Audit plugin API. +4. Preserve runtime patches. +5. Integrate build and submodules. +6. Audit CI/CD. +7. Run the test matrix. +8. Finalize evidence and rollback. +9. Request separate approval for merge-to-main, push, PR, tag, or release. + +## Gates + +Before each phase verify: + +- exact base, fork, and target refs; +- prior phase artifact exists and refs agree; +- prior acceptance criteria pass; +- no unresolved P0 decision; +- worktree and branch isolation; +- output directory is under /share/project/zhaoyingli/flagos/temp. + +Pause after analysis, before source edits, before GPU runs, before merge, and before any external publication. A blocked hardware test pauses only the affected capability, but must remain visible in the final report. + +## Safety + +Do not use blanket ours/theirs, reset, clean, force push, or tree replacement. Do not count blocked non-NVIDIA tests as pass. Do not mutate main. Keep each phase in a focused commit or uncommitted review state until approved. + +## Acceptance Criteria + +- All eight specialized phases have an artifact and status. +- Ref identities match across artifacts. +- Every P0 item has an approved decision or remains blocked. +- Every blocked test and missing backend has owner and reason. +- Final report includes evidence index and rollback plan. +- No push, PR, tag, merge-to-main, or release occurs without explicit user approval. +- No artifact or instruction references /tmp. + + +## Design-baseline gate + +Conflict integration cannot begin until the design baseline for the selected `base..fork` delta exists and every design group has an owner plus a preserve/adapt/drop decision placeholder. Carry those decisions into finalization. diff --git a/skills/te-upgrade-orchestrator/agents/openai.yaml b/skills/te-upgrade-orchestrator/agents/openai.yaml new file mode 100644 index 0000000000..62360d2dfe --- /dev/null +++ b/skills/te-upgrade-orchestrator/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TE Upgrade Orchestrator" + short_description: "Orchestrate gated TransformerEngine upgrades" + default_prompt: "Use $te-upgrade-orchestrator to run the gated TransformerEngine-FL upstream upgrade workflow." diff --git a/skills/te-upgrade-orchestrator/references/contract.md b/skills/te-upgrade-orchestrator/references/contract.md new file mode 100644 index 0000000000..7a84962103 --- /dev/null +++ b/skills/te-upgrade-orchestrator/references/contract.md @@ -0,0 +1,5 @@ +# Orchestration Contract + +Each phase consumes the prior phase's JSON and emits a stable artifact. The orchestrator must fail closed when a file is missing, refs differ, a status is unknown, or a blocker has no owner. A user approval is a state transition and must be recorded with date, scope, and exact authorized action. + +The only allowed hardware assumption for this upgrade is the authorized NVIDIA host and its conda environment; non-NVIDIA native tests remain blocked unless separately authorized. diff --git a/skills/te-upgrade-orchestrator/scripts/check_orchestration.py b/skills/te-upgrade-orchestrator/scripts/check_orchestration.py new file mode 100755 index 0000000000..6a1d3b87d9 --- /dev/null +++ b/skills/te-upgrade-orchestrator/scripts/check_orchestration.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import argparse,json +from pathlib import Path +def main(): + ap=argparse.ArgumentParser(); ap.add_argument("--evidence",type=Path,required=True); ap.add_argument("--output",type=Path,required=True) + a=ap.parse_args(); a.output.mkdir(parents=True,exist_ok=True) + names=["inventory.json","conflict-inventory.json","api-inventory.json","runtime-patch-ledger.json","build-audit.json","cicd-audit.json","test-matrix.json","evidence-index.json"] + present={n:(a.evidence/n).exists() for n in names} + data={"phases":present,"complete":all(present.values()),"external_actions_authorized":False} + (a.output/"orchestration-status.json").write_text(json.dumps(data,indent=2)+chr(10)) + (a.output/"orchestration-status.md").write_text("# Orchestration Status"+chr(10)+chr(10)+"- Complete: "+str(data["complete"])+chr(10)+"- External actions authorized: false"+chr(10)) + print(json.dumps({"output":str(a.output),"complete":data["complete"],"missing":[n for n,v in present.items() if not v]})) +if __name__=="__main__": main()