From c30a5bfa23a9cf274bab62ac2945990cd7165fb4 Mon Sep 17 00:00:00 2001 From: Anurag Garg Date: Fri, 21 Aug 2026 12:02:03 +0530 Subject: [PATCH] chore: sync ia skill with upstream and bump to v1.3.0 - Add six new workflows: change impact analysis (+ effort estimate workbook), object context matrix, new-developer onboarding guide, cross-version comparison, repository-wide member diff, and RPG fixed-form to free-form conversion, with their five build scripts and two templates - Tool catalog 51 -> 54: ia_object_context_matrix, ia_member_variants, ia_variant_summary - SKILL.md routing extended; REFERENCE_USAGE now documented as type-dependent (on *FILE rows I means Input, not Implicit) - Strip internal repository table names, raw SQL and execute_sql references from the published copy; keep the functional guidance - Bump all nine manifests to 1.3.0 and add the CHANGELOG entry --- .agents/plugins/marketplace.json | 2 +- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- .cursor-plugin/plugin.json | 2 +- .github/plugin/marketplace.json | 2 +- CHANGELOG.md | 19 + gemini-extension.json | 2 +- package.json | 2 +- plugin.json | 2 +- skills/ia/SKILL.md | 24 +- .../ia/references/change-impact-analysis.md | 413 ++++++++++++ skills/ia/references/index.md | 22 +- skills/ia/references/member-diff.md | 158 +++++ skills/ia/references/object-context-matrix.md | 205 ++++++ skills/ia/references/onboarding-guide.md | 178 +++++ skills/ia/references/rpg-conversion.md | 182 +++++ skills/ia/references/rpg-cycle-recipes.md | 141 ++++ skills/ia/references/rpg-opcode-map.md | 112 ++++ skills/ia/references/templates/README.md | 4 + .../template-modernization-summary.md | 80 +++ .../templates/template-onboarding-guide.md | 127 ++++ skills/ia/references/tool-catalog.md | 5 +- skills/ia/references/version-comparison.md | 285 ++++++++ skills/ia/scripts/build_change_estimate.py | 424 ++++++++++++ skills/ia/scripts/build_context_matrix.py | 417 ++++++++++++ skills/ia/scripts/build_member_diff.py | 554 ++++++++++++++++ skills/ia/scripts/build_onboarding_guide.py | 264 ++++++++ skills/ia/scripts/build_version_diff.py | 626 ++++++++++++++++++ skills/ia/scripts/convert_md_to_docx.py | 78 ++- 30 files changed, 4317 insertions(+), 19 deletions(-) create mode 100644 skills/ia/references/change-impact-analysis.md create mode 100644 skills/ia/references/member-diff.md create mode 100644 skills/ia/references/object-context-matrix.md create mode 100644 skills/ia/references/onboarding-guide.md create mode 100644 skills/ia/references/rpg-conversion.md create mode 100644 skills/ia/references/rpg-cycle-recipes.md create mode 100644 skills/ia/references/rpg-opcode-map.md create mode 100644 skills/ia/references/templates/template-modernization-summary.md create mode 100644 skills/ia/references/templates/template-onboarding-guide.md create mode 100644 skills/ia/references/version-comparison.md create mode 100644 skills/ia/scripts/build_change_estimate.py create mode 100644 skills/ia/scripts/build_context_matrix.py create mode 100644 skills/ia/scripts/build_member_diff.py create mode 100644 skills/ia/scripts/build_onboarding_guide.py create mode 100644 skills/ia/scripts/build_version_diff.py diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 7d0c4c3..32dbf2d 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "source": "./" } diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7d0c4c3..32dbf2d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "source": "./" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ec80dc2..bfda380 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "homepage": "https://github.com/programmersio-ibmi/skills", "license": "MIT", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 4d4fcc0..8e780bb 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis for coding agents.", "homepage": "https://github.com/programmersio-ibmi/skills", "license": "MIT", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index e02366b..d7492fb 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ia", "displayName": "iA — IBM i Impact Analysis", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "publisher": "programmersio", "homepage": "https://github.com/programmersio-ibmi/skills", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 7d0c4c3..32dbf2d 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "source": "./" } diff --git a/CHANGELOG.md b/CHANGELOG.md index d74524e..0587a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ All notable changes to this project are documented here. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.3.0] — 2026-08-21 + +### Added + +- **Change impact analysis** workflow (`references/change-impact-analysis.md`) — classify a *planned* change into one of 15 classes, run the class-specific checks and silent-failure traps (DS offset shifts, KLIST partial keys, join field length mismatch, truncating MOVEs, REFFLD cascade, `*SRVPGM` signatures), and emit a formula-driven effort estimate workbook via `scripts/build_change_estimate.py`. +- **Object context matrix** (`references/object-context-matrix.md`) — everything one object uses plus everything that uses it, for any object type, from a single `ia_object_context_matrix` call, with `.xlsx`/`.html` derived from the markdown by `scripts/build_context_matrix.py`. +- **New-developer onboarding guide** (`references/onboarding-guide.md`) — menu → program → file relations for someone new to an application, with `scripts/build_onboarding_guide.py` for the workbook. +- **Cross-version comparison** (`references/version-comparison.md`) — diff one member across the libraries holding it, with N-way alignment and dual-RRN diff hunks generated by `scripts/build_version_diff.py`. +- **Repository-wide member diff** (`references/member-diff.md`) — which members differ across libraries, with a three-sheet workbook from `scripts/build_member_diff.py`. +- **RPG fixed-form → free-form conversion** (`references/rpg-conversion.md`) — covers RPG II/III, RPG/400 and fixed RPG IV, always eliminating the cycle, with companions `references/rpg-opcode-map.md` and `references/rpg-cycle-recipes.md`. +- Three tools in `references/tool-catalog.md` (now 54): `ia_object_context_matrix`, `ia_member_variants`, `ia_variant_summary`. +- Templates `references/templates/template-modernization-summary.md` and `references/templates/template-onboarding-guide.md`. + +### Changed + +- `SKILL.md` routing table extended for the new document types, and the tool count updated from 51 to 54. +- `SKILL.md` now documents that `REFERENCE_USAGE` means two different things by row type — on `*FILE` rows `I` is *Input*, not *Implicit* — and that `FILE_USAGES` is empty in current repositories. +- `scripts/convert_md_to_docx.py` improvements for the new deliverables. + ## [1.2.1] — 2026-07-19 ### Added diff --git a/gemini-extension.json b/gemini-extension.json index df0b9d5..f3ccbb3 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "contextFileName": "GEMINI.md" } diff --git a/package.json b/package.json index cb4cf9c..6db2124 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ia-tools-skill", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "license": "MIT", "type": "module", diff --git a/plugin.json b/plugin.json index f3e77d7..1f8b61c 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "ia", - "version": "1.2.1", + "version": "1.3.0", "description": "IBM i Impact Analysis — dependency tracing, field impact, call hierarchy, program documentation.", "publisher": "programmersio", "homepage": "https://github.com/programmersio-ibmi/skills", diff --git a/skills/ia/SKILL.md b/skills/ia/SKILL.md index f2c96f1..09e0c81 100644 --- a/skills/ia/SKILL.md +++ b/skills/ia/SKILL.md @@ -1,13 +1,13 @@ --- name: ia -description: Guide for using iA by programmers.io MCP tools to analyze IBM i programs — dependency tracing, call hierarchies, field impact, source retrieval, and program documentation. ALWAYS use this skill for ANY IBM i analysis question. +description: Guide for using iA by programmers.io MCP tools to analyze IBM i programs — dependency tracing, call hierarchies, field impact, source retrieval, program documentation, change impact analysis with effort estimation, and RPG fixed-form → free-form conversion. ALWAYS use this skill for ANY IBM i analysis question. --- # iA Impact Analysis — Agent Guide iA by [programmers.io](https://programmers.io/ia/) pre-parses IBM i source (RPG, CL, COBOL, DDS) into a queryable repository accessed through the `ia_*` MCP tools. -**Goal:** Answer most questions in 1-2 tool calls. Consult [quick-reference.md](references/quick-reference.md) for tool selection, [tool-catalog.md](references/tool-catalog.md) for the full 51-tool list. +**Goal:** Answer most questions in 1-2 tool calls. Consult [quick-reference.md](references/quick-reference.md) for tool selection, [tool-catalog.md](references/tool-catalog.md) for the full 54-tool list. ## Rule Zero — Always Query iA, Never the Workspace @@ -41,6 +41,10 @@ If a tool returns zero rows for a name you passed, the object/file/field **does | `ia_rpg_source` returns nothing | confirm `MEMBER_TYPE` first (`ia_member_lookup`): CL/CLLE/CLP → `ia_cl_source`; COBOL isn't in the RPG tables. Empty ≠ missing | assuming the source doesn't exist | | "Obsolete / unreferenced objects" | `ia_unused_objects` — source physical files (QRPGLESRC, QCLSRC…) are already excluded; remaining `*FILE` rows show `OBJECT_ATTRIBUTE` | treating every unreferenced `*FILE` as dead — DSPF/PRTF and SQL-only tables can be false positives | | Data files vs source files in a library | `ia_object_list(object_attribute='PF-DATA')` for data files, `'PF-SRC'` for source files; plain `PF` returns both with a `pf_kind` label | assuming a source library (QRPGLESRC etc.) has data files — it usually has none | +| Full context of object X (what it uses **and** what uses it) / "object context matrix" | `ia_object_context_matrix(object_name=X)` — one call, pre-bucketed by usage mode, with each referenced object's attribute + description | `ia_object_references` + `ia_find_object_usages` — neither returns the *referenced* object's attribute or description, so you cannot split display/printer files from data files without one extra lookup per object | +| "Onboard a new developer on menu X", "menu → program → file mapping" | load [onboarding-guide.md](references/onboarding-guide.md) — menu-scoped reading document | [app-map.md](references/app-map.md) — same data, but its deliverable is a 3D graph, not something you can read or hand to someone | +| Menu **option numbers / option text** for menu X | the menu's source members — `{MENU}QQ` (`MNUCMD`) holds `NNNN CALL PGM(...)`, `{MENU}` (`MNUDDS`) holds the text | `ia_call_hierarchy` — it returns *which* programs the menu launches, but `CALL_SEQUENCE` is empty on those rows, so it can tell you nothing about option order | +| "What breaks if I change / resize / drop X?", "how long will this change take?" | load [change-impact-analysis.md](references/change-impact-analysis.md) — classify the change first, then run the class-specific traps + estimate | a bare `ia_find_object_usages` — where-used is the *start* of a change assessment, not the answer; it misses DS offsets, KLIST keys and REFFLD cascade entirely | ## Top 10 Tools (80% of Queries) @@ -80,6 +84,8 @@ If a tool returns zero rows for a name you passed, the object/file/field **does Present as four sections: **Direct (NEEDS_CHANGE)**, **Direct (NEEDS_RECOMPILE)**, **Structural Dependents**, **Programs via LF**. +> This answers *what references the field*. If the user is actually **about to change something** — resize/retype/drop a field, alter a parameter, retire a file — load [change-impact-analysis.md](references/change-impact-analysis.md) instead. It classifies the change first, adds the traps that class carries (DS offsets, KLIST partial keys, REFFLD cascade, `*SRVPGM` signatures), and produces a costed estimate workbook. + ## When to Chain **DO chain:** `*SRVPGM` in results (amplifier — check what binds to it), field impact (run all steps), a **SQL long name** before any system-name tool (resolve via `ia_sql_table_names` → use the `system_short_name`). @@ -102,8 +108,10 @@ Present as four sections: **Direct (NEEDS_CHANGE)**, **Direct (NEEDS_RECOMPILE)* | Empty results | Object/file not found under that name (Rule Two), or scheduler-invoked / external | | `REFERENCE_SOURCE = O` | Detected from compiled object | | `REFERENCE_SOURCE = S` | Detected from source code | -| `REFERENCE_USAGE = I` | Implicit (via binding directory) | -| `REFERENCE_USAGE = E` | Explicit (direct bind/call) | +| `REFERENCE_USAGE` on a **`*FILE`** row | **File access mode:** `I`=Input, `O`=Output, `U`=Update, `C`=**Combined** (a workstation/display file opened for read *and* write — not "create") — and they combine (`I/O`, `I/U`, `O/U`, `I/O/U`, `C/O`) | +| `REFERENCE_USAGE` on `*SRVPGM` / `*MODULE` / `*BNDDIR` | `I` = Implicit (via binding directory), `E` = Explicit (direct bind/call) | + +**`REFERENCE_USAGE` means two different things depending on the row's type** — on `*FILE` rows `I` is *Input*, not *Implicit*. Read the type first. The separate `FILE_USAGES` column is **empty in current repositories** — never take file access mode from it. **Empty results with library filter:** Report the negative explicitly. Don't silently retry without filter. @@ -137,10 +145,16 @@ Do not attempt to diagnose server-side issues or retry indefinitely. | Need | Load | |------|------| | Tool selection unclear | [quick-reference.md](references/quick-reference.md) | -| Full 51-tool list | [tool-catalog.md](references/tool-catalog.md) | +| Full 54-tool list | [tool-catalog.md](references/tool-catalog.md) | | Complex analysis chains | [query-flows.md](references/query-flows.md) | | Analysis playbooks | [playbook.md](references/playbook.md) | | Program documentation | [program-documentation.md](references/program-documentation.md) | +| Impact + effort of a **planned change** (resize/drop/rename a field, change a parm, retire a file) + estimate workbook | [change-impact-analysis.md](references/change-impact-analysis.md) | +| Object context matrix (everything around one object) + downloads | [object-context-matrix.md](references/object-context-matrix.md) | +| Onboard a new developer onto an application (menu → program → file) | [onboarding-guide.md](references/onboarding-guide.md) | +| Compare multiple versions of a program (across libraries) + side-by-side Excel diff | [version-comparison.md](references/version-comparison.md) | +| Which members differ across libraries, repository-wide + Excel workbook | [member-diff.md](references/member-diff.md) | | Test case document for a program (QA/UAT scripts) | [test-case-generation.md](references/test-case-generation.md) | | Visual flowchart of a program (single-page HTML) | [flowchart.md](references/flowchart.md) | | 3D app map of a library or application area (interactive HTML) | [app-map.md](references/app-map.md) | +| Convert / modernize RPG to free form (RPG II/III/fixed RPG IV, cycle-aware) | [rpg-conversion.md](references/rpg-conversion.md) | diff --git a/skills/ia/references/change-impact-analysis.md b/skills/ia/references/change-impact-analysis.md new file mode 100644 index 0000000..d209875 --- /dev/null +++ b/skills/ia/references/change-impact-analysis.md @@ -0,0 +1,413 @@ +# Change Impact Analysis + +Use this when the user is about to **change something** and wants to know what it +breaks, who has to be told, and what it costs — *"what happens if I make CUSTNO +10 long"*, *"can I drop this field"*, *"impact of adding a parameter to +ORDVAL"*, *"we're retiring ITMMAST, what depends on it"*. + +The deliverable is a **Change Impact Assessment**: a classified change, a +verified list of affected artifacts, the failures that will not announce +themselves, a risk verdict, and a costed estimate workbook. + +> Not the same as [object-context-matrix.md](object-context-matrix.md). That +> answers *what surrounds this object today*. This one answers *what happens if I +> alter it* — the same edges, read forward, plus the change-type-specific traps +> and an estimate. + +--- + +## 1. Step 0 — classify the change before querying anything + +Everything downstream depends on this. Ask if it is not stated; guessing wastes +calls and produces the wrong trap list. + +| # | Change class | Typical phrasing | +|---|--------------|------------------| +| C1 | **Field resize** | "make it bigger", "CHAR(6) → CHAR(10)", "add two digits" | +| C2 | **Field type change** | "make it numeric", "convert to a real date" | +| C3 | **Field add** | "add a column", "new field on the file" | +| C4 | **Field remove** | "drop it", "we don't use it any more" | +| C5 | **Field rename** | "rename CUSTNO to CUSTNBR" | +| C6 | **Key change** | "add to the key", "reorder the key", "make it unique" | +| C7 | **File add / new LF** | "new table", "new index over it" | +| C8 | **File retire / delete** | "can we drop this file" | +| C9 | **Program logic change** | "change how the discount is worked out" | +| C10 | **Program interface change** | "add a parameter", "change the parm length" | +| C11 | **Service program export change** | "add a procedure", "change a PR" | +| C12 | **Copybook change** | "change the shared DS" | +| C13 | **Screen / report layout change** | "add a field to the screen" | +| C14 | **Constraint / trigger change** | "add an FK", "change the check" | +| C15 | **Object move / library change** | "promote to production", "move to a new library" | + +Also capture **direction** where it applies — widening vs narrowing (C1), +optional vs mandatory (C3, C10). It changes the trap list, not just the wording. + +--- + +## 2. The spine — run this for every change class + +Six calls, most of them parallel. Everything in §3 is *added* to this. + +``` +1. ia_object_lookup / ia_member_lookup → confirm the thing exists, get type + library +2. ia_object_context_matrix(object_name=X) → all inbound + outbound edges, pre-bucketed +3. ia_find_object_usages(object_name=X) → where-used, with reference source S/O +4. ia_object_lifecycle(object_name=X) → is any of this actually alive +5. ia_code_complexity(member_name=*ALL, …) → line counts for every affected program (drives the estimate band) +6. ia_repo_config → freshness: when was the metadata last built +``` + +Step 6 is not optional. An assessment built on a stale repository is worse than +none, because it reads as authoritative. If the usage data is old or blank, say so +in the report header and treat "unused" claims as unverified. + +**If step 1 returns nothing, stop.** The object does not exist under that name +(Rule Two). Do not assess a similarly-named object. + +--- + +## 3. Per-class additions + +Each row lists only what the spine does **not** already cover. + +### C1 / C2 — field resize or retype + +The highest-risk class, because the compiler catches almost none of it. + +| Check | Why this class | Tool | +|-------|----------------|------| +| Is the field REFFLD-inherited? | Changing it on this file is the *wrong edit* — the definition lives in the field-reference file | `ia_file_fields` (reference chain) | +| Who else inherits it? | One FRF edit resizes the field in every consuming file | `ia_field_reffld_consumers` | +| LFs, indexes, views | Key-length change forces rebuild; select/omit literals may no longer match | `ia_file_dependencies` | +| Join LFs | Join field pairs must match length **and** type — resize one side and the join is invalid | `ia_join_logical_files` | +| Constraints | FK means the parent key must move in lockstep; CHECK literals may stop holding | `ia_file_constraints` | +| DS overlays | A DS over the record format shifts every subfield *after* the field. Numeric widening changes packed byte length, so offsets move even when digits look harmless | `ia_data_structures` | +| KLIST / KFLD | Key resize invalidates every key list containing it, and every CHAIN/SETLL using them | `ia_klist_usage(kfld_name=…)` | +| Truncating operations | `MOVE`/`MOVEL` into an unchanged work field, `%SUBST` at hardcoded offsets, fixed-position arrays | `ia_variable_ops`, `ia_rpg_source_search` | +| Screen / print fit | A wider field may not fit its position or may overlap its neighbour | `ia_object_context_matrix` (DSPF/PRTF buckets) | + +Split the affected-program table by **direction**. Widening and narrowing are +different remediation lists — widening risks truncation on the receiving side and +layout overflow; narrowing risks data loss and every widening risk in reverse. +Merging them forces the developer to re-derive the split by hand. + +### C3 — field add + +Cheaper than it looks, and saying so is useful. Level check still forces a +recompile of every program with an F-spec on the file, but almost nothing needs a +code change. + +- `ia_object_context_matrix` → the recompile list (this is most of the work) +- `ia_data_structures` → externally-described DS pick the field up automatically; program-defined overlays do not +- `ia_file_constraints` → a NOT NULL addition without a default fails on existing rows +- Flag any `*OUTPUT`-mode program writing the record without the new field + +### C4 / C5 — field remove or rename + +The one class where the compiler *is* on your side — every reference fails to +compile. The work is finding them all so nothing is missed at cutover. + +- `ia_file_field_impact_analysis(file_name=X, field_name=Y)` → direct references +- `ia_field_reffld_consumers` → if it is an FRF field, removal cascades +- `ia_klist_usage(kfld_name=Y)` → key lists naming it +- `ia_rpg_source_search(search_text=Y)` → catches literal/comment/SQL mentions the cross-reference misses +- `ia_file_constraints`, `ia_file_dependencies` → LF key or select/omit referencing it +- **Rename is not one change** — treat as remove + add, and check whether an SQL long name exists via `ia_sql_table_names` + +### C6 — key change + +- `ia_file_dependencies` → every LF/index over the file +- `ia_klist_usage` → partial-key reads break silently when key order changes +- `ia_file_constraints` → PK/UQ and any FK pointing at this key +- Adding a **unique** key can fail outright on existing duplicate data — flag as a data check, not a code change +- Reordering a key changes `READE`/`SETLL` semantics with **no compile error** + +### C7 — new file or new LF + +Lowest risk class. Keep the report short. + +- `ia_library_files` → name collision check +- `ia_file_dependencies` on the base PF → a new index adds maintenance cost to every write path +- Note the write-path cost in the summary; do not pad the report + +### C8 — file retire + +- `ia_object_context_matrix` → everything reaching it +- `ia_file_dependencies` → LFs must go first, and in order +- `ia_file_overrides` + `ia_override_chain` → a program may reach it only through an override, so it will not appear as a direct reference +- `ia_object_lifecycle` + `ia_obj_size` → last-used date and row count are the evidence for "safe to drop" +- `ia_unused_objects` → corroboration, not proof + +**Never call a file dead from a zero row count alone.** Find the caller and the +state it runs under first — a year-end-only program looks dead for eleven months. + +### C9 — program logic change + +- `ia_call_hierarchy(direction='CALLERS')` → who depends on this behaviour +- `ia_subroutines` → usage counts locate the blast radius inside the program +- `ia_program_files` → which files the change can touch, with PREFIX detail +- `ia_code_complexity` → effort band and review depth +- Field-level lineage (§4) if the change alters what a field *contains* rather than only control flow + +### C10 / C11 — interface or export change + +The most under-estimated class. A signature change to a `*SRVPGM` invalidates +every program bound to it, whether or not they use the changed procedure. + +- `ia_call_parameters(member_name=X)` → every call site and what it passes today +- `ia_procedure_params` → the PR/PI signature being altered +- `ia_srvpgm_exports` → the export list; **export order changes break the signature** +- `ia_procedure_xref` → procedure-level callers +- `ia_find_object_usages(object_name=SRVPGM)` → everything bound, the true recompile set +- Adding a parameter at the **end** with a default is far cheaper than inserting one — say which was chosen + +### C12 — copybook change + +- `ia_copybook_impact(copybook_name=X)` → every member with `/COPY` +- `ia_member_copybooks` → the reverse view, to catch nested copybooks +- Fan-out here is members, not objects — a single copybook edit can touch hundreds + +### C13 — screen or report change + +- `ia_object_context_matrix` → programs using the DSPF/PRTF (usage mode `C` = Combined) +- `ia_file_fields` on the DSPF → position collisions +- Indicator usage via `ia_program_variables` — display files break on indicator conflicts more often than field ones + +### C14 — constraint or trigger change + +- `ia_file_constraints` → existing PK/UQ/FK/CHK and delete/update rules +- A new FK fails to add if existing data has orphans — always a data check +- Trigger changes affect **every** write path, including ones with no source-level reference to the trigger + +### C15 — object move / library change + +- `ia_file_overrides`, `ia_override_chain` → library-qualified overrides break on move +- `ia_object_lookup` → does the name already exist in the target library +- Anything still resolving through `*LIBL` becomes ambiguous — call these out individually + +--- + +## 4. Field-level data lineage + +**Available, but best-effort — never present it as complete.** It is derived from +parsed source, so it sees what the source says, not what runs. + +### How it works + +`ia_variable_ops` returns an assignment graph: one row per operation, with +`FACTOR1_VAL` / `FACTOR2_VAL` as sources and `RESULT_VAL` as the target, plus +`BIF` for the transform and `SOURCE_RRN` for the line. + +``` +ia_variable_ops(member_name='ORDENT', variable_name='CUSTNO', opcode='*ALL') +``` + +Follow `RESULT_VAL` of one row into `FACTOR1_VAL` of the next to build a chain: + +``` +CUSTNO ──(read from CUSTMST)──► WKCUST ──%CHAR──► PRTCUST ──(written to ORDPRT) +``` + +**Bridging file fields to variables:** the repository carries no populated +variable→file binding in the repositories checked so far, so there is nothing +direct to join on. Use the naming identity instead — for an +externally-described file, the RPG field name *is* the variable name, so match +`ia_file_fields` field names against `FACTOR`/`RESULT` values directly. Resolve +`PREFIX` first with `ia_program_files`, or every prefixed field silently misses. + +**Crossing program boundaries:** `ia_call_parameters` gives the parameter at each +call site, which is the edge from caller variable to callee parameter. Chain +lineage inside program A → parameter → lineage inside program B. + +### Confidence levels — label every edge + +| Confidence | When | +|------------|------| +| **High** | Direct `=` assignment, both sides plain variable names, single row | +| **Medium** | A `BIF` transforms the value (`%CHAR`, `%EDITC`, `%SUBST`), or the expression spans continuation rows (`COMPARE_OPER='+'`) | +| **Low** | Target reached via DS overlay, array index, pointer, or embedded SQL | +| **None** | Dynamic — the field name is computed at runtime | + +### Limits to state in the report + +1. `RESULT_VAL` is `CHAR(50)` while `FACTOR1_VAL` is `CHAR(80)` — long qualified + `DS.SUB.SUB` targets can be **truncated**, so a chain may break mid-path. +2. Multi-line expressions split across rows joined by `COMPARE_OPER='+'`. Reading + a single row loses operands — gather all continuation rows for one `SOURCE_RRN`. +3. `RESULT_VAL` is not always a bare name (`EVAL-CORR UDPSDS` appears literally). + Strip the opcode prefix before matching. +4. `FACTOR1_VAL` of the form `Const("…")` is a literal, not a variable. Exclude + these from lineage edges or every constant becomes a false source. +5. Available opcodes vary by source vintage — free-form repositories carry `=` and + `SQLEXC`; fixed-form carry `MOVE`/`MOVEL`/`Z-ADD`. Do not assume a fixed set. +6. `SQLEXC` rows carry the statement in `FACTOR2_VAL` with no `FACTOR1_VAL`. + Lineage through embedded SQL needs the statement parsed — mark it **Low**. + +### When to run it + +Only for C1, C2, C4, C5, C9 — where the field's *value* moves. Skip it for +recompile-only classes; it costs calls and adds nothing to a level-check list. + +--- + +## 5. The silent-failure catalogue + +Give this its own section in every report for C1, C2, C6, C9. These compile +clean and produce wrong output — they are the reason the assessment exists. + +| Failure | Class | How it is found | +|---------|-------|-----------------| +| Truncation into an unchanged work field | C1 | `ia_variable_ops` — target shorter than new source | +| DS subfield offset shift | C1, C2, C3 | `ia_data_structures` — any DS over the record format | +| Partial-key mismatch | C1, C6 | `ia_klist_usage` | +| Select/omit literal no longer matching | C1, C2 | `ia_file_constraints` | +| Join field length mismatch | C1, C2 | `ia_join_logical_files` | +| Screen/report column overflow | C1, C3, C13 | DSPF/PRTF rows + `ia_file_fields` | +| Override pointing at a different physical | all | `ia_file_overrides`, `ia_override_chain` | +| `*SRVPGM` signature invalidation | C11 | `ia_srvpgm_exports` | +| Numeric edit/format assumptions | C1, C2 | `ia_rpg_source_search` for `%EDITC`, `%EDITW` | + +--- + +## 6. Risk verdict + +Use the counts you actually have. Reuse the rubric in +[playbook.md](playbook.md) and add the class weighting: + +| Verdict | When | +|---------|------| +| **Low** | ≤5 affected objects, no silent-failure rows, no `*SRVPGM`, no FRF cascade | +| **Medium** | 6–20 affected objects, or any DS overlay / KLIST hit | +| **High** | >20 affected objects, **or** any of: FRF cascade, `*SRVPGM` signature change, trigger change, narrowing with existing data | + +A single `*SRVPGM` or FRF hit outranks the count. Say which factor drove the +verdict — a bare "High" that the developer cannot trace is not actionable. + +--- + +## 7. The estimate workbook + +Always offered, generated from the report — never hand-authored. + +The report must contain a table under a `## Affected Artifacts` heading with +exactly these columns, because the script keys on them: + +``` +| Library | Object | Type | Attribute | Impact | Lines | Notes | +``` + +- **Impact** — one of `CHANGE`, `RECOMPILE`, `REBUILD`, `REVIEW`, `NONE` +- **Lines** — from `ia_code_complexity`; blank for non-source objects +- **Type** / **Attribute** — as returned by `ia_object_lookup` (`*PGM`, `*FILE` + `PF`/`LF`/`DSPF`/`PRTF`) + +``` +python .claude/skills/ia/scripts/build_change_estimate.py REPORT.md --xlsx +``` + +Add `--narrowing` when the change can lose data (C1 narrowing, C2, C4), which +adds the data-conversion line. + +Three sheets, in this order: **Assumptions → Summary → Estimate Detail.** The rates +come first because they are what a client argues about; the workbook opens on the +negotiable page rather than on the total. + +The workbook is **formula-driven**: every rate lives on the `Assumptions` sheet and +the Summary and Detail sheets reference it. Change a rate in Excel and the totals +recalculate — no re-run needed. Summary heads with a single count, **Total impacted +objects** (every row of the Affected Artifacts table, whatever its Impact verb); the +per-basis breakdown lives once, in the Basis table at the foot of the same sheet. +Defaults are deliberately lean: + +| Line | Default | +|------|---------| +| Impact review & change design | 2.0 h fixed | +| Cutover / implementation | 2.0 h fixed | +| Program change — S (≤ 500 lines) | 1.5 h | +| Program change — M (501–2,000) | 3.0 h | +| Program change — L (> 2,000) | 5.0 h | +| Program — recompile only | 0.15 h | +| PF / DDS-DDL change | 0.15 h | +| Logical file / index rebuild | 0.15 h each | +| Display or printer file change | 1.0 h | +| Copybook change | 0.15 h | +| Other object change | 0.15 h | +| Review-only item | 0.5 h | +| Unit test | 0.5 h per changed program | +| System / regression test | 15% of development | +| UAT support | 10% of development | +| Data conversion (`--narrowing`) | 2.0 h | +| Contingency | 12% of subtotal | + +Every non-program artifact is 0.15 h except display and printer files, which stay +at 1.0 h because a layout change is real design work rather than a rebuild. The +effort therefore sits almost entirely in the changed programs, which is where it +belongs — and it makes the S/M/L banding, not the object count, the thing worth +arguing about. + +**Quote the number, then the assumption.** A total without the rate table behind +it invites a haggle; with it, the conversation is about the rates, which is a +conversation worth having. + +--- + +## 8. Shape of the deliverable + +1. **Title** — `# Change Impact Assessment — {OBJECT} · {CHANGE CLASS}` +2. **The change** — one paragraph: what, from what, to what, direction. If the + user gave a before/after, restate it precisely; ambiguity here invalidates + everything below. +3. **Verdict** — Low/Medium/High plus the one factor that drove it, and the + headline effort from the workbook. +4. **Affected Artifacts** — the canonical table from §7. This is the spine of the + document and the input to the estimate. +5. **Silent failures** (§5) — omit the section only if genuinely empty. +6. **Field lineage** (§4) — only for the classes that warrant it. +7. **Sequence** — dependency-ordered: data → files → LFs → copybooks → + programs (callees before callers) → screens → cutover. +8. **Data checks** — what must be verified against real data before proceeding. +9. **Out of scope** (§9). +10. **Download menu.** + +Escape `|` in every cell — object descriptions contain them, and an unescaped +pipe silently truncates the cell and shifts every column right of it. + +### Where the files go + +``` +docs/change-impact/{OBJECT}/{OBJECT}_Change_Impact.md +docs/change-impact/{OBJECT}/{OBJECT}_Change_Estimate.xlsx +``` + +For a `*PGM` whose spec already exists, use +`docs/program-specs/{PGM}/` instead so the assessment sits with its spec. + +### Downloads + +``` +Download this assessment as: + 1. Markdown (.md) 2. Estimate workbook (.xlsx) 3. Word (.docx) + 4. PDF (.pdf) 5. All of the above +``` + +``` +python .claude/skills/ia/scripts/build_change_estimate.py ASSESSMENT.md --xlsx +python .claude/skills/ia/scripts/convert_md_to_docx.py ASSESSMENT.md +python .claude/skills/ia/scripts/convert_md_to_pdf.py ASSESSMENT.md +``` + +--- + +## 9. What this cannot tell you — state it explicitly + +A section the report must always carry. It is what makes the rest credible. + +- **Actual data.** Whether values exceed a narrowed size, whether duplicates + block a unique key, whether orphans block an FK. All need a query against live + data, not metadata. Name the specific check. +- **Physical layout feasibility.** Whether a wider field fits the screen. +- **External interfaces.** EDI layouts, flat-file exports, APIs, downstream ETL, + reports consumed outside the box — none are in the repository. +- **Runtime-dynamic behaviour.** Dynamic CALLs with computed names, dynamic SQL, + `*LIBL` resolution that depends on the job. +- **Business meaning.** Whether the field's semantics survive the change. +- **Anything newer than the last metadata build** (`ia_repo_config`). diff --git a/skills/ia/references/index.md b/skills/ia/references/index.md index e1c6041..2f1e96f 100644 --- a/skills/ia/references/index.md +++ b/skills/ia/references/index.md @@ -8,23 +8,37 @@ This index guides progressive loading of skill references. Load only what you ne |------------|-----------|-----------------| | **Simple lookup** ("what uses X?", "find object") | SKILL.md only | — | | **Tool selection unclear** | + [quick-reference.md](quick-reference.md) | Decision tree + intent mapping | -| **Need full tool list** | + [tool-catalog.md](tool-catalog.md) | All 51 tools by category | +| **Need full tool list** | + [tool-catalog.md](tool-catalog.md) | All 54 tools by category | | **Complex analysis** (field impact, call chains) | + [query-flows.md](query-flows.md) | Optimal tool sequences | | **Troubleshooting / edge cases** | + [playbook.md](playbook.md) | Analysis playbooks | | **Program documentation** | + [program-documentation.md](program-documentation.md) | 8-step workflow | +| **Change impact** ("what breaks if I resize/drop X", "effort to add a parm") | + [change-impact-analysis.md](change-impact-analysis.md) | classify the change (C1–C15) → spine + class traps → silent-failure catalogue → risk verdict → `build_change_estimate.py` workbook | +| **Object context matrix** ("context matrix for X", "everything around X") | + [object-context-matrix.md](object-context-matrix.md) | one `ia_object_context_matrix` call → sectioned tables → download menu (`build_context_matrix.py`) | +| **Onboarding guide** ("onboard a new developer on menu X", "menu to program to file mapping") | + [onboarding-guide.md](onboarding-guide.md) | resolve the menu → spine (call hierarchy + complexity + one context matrix per program) → option numbers from the menu source members → sectioned guide → freshness + cross-check gate → downloads (`build_onboarding_guide.py`) | +| **Version comparison** ("compare the versions of X", "diff X across libraries") | + [version-comparison.md](version-comparison.md) | discover all versions → ask which to compare + output format(s) → `build_version_diff.py` → changed-only report + optional side-by-side Excel | +| **Member diff** ("which members differ between our libraries", repository-wide) | + [member-diff.md](member-diff.md) | `ia_variant_summary` → `ia_member_variants` (LOGIC shortlist) → `build_member_diff.py` workbook | | **App map** ("app map of library/area", "3D map") | + [app-map.md](app-map.md) | JSON contract + build script | +| **RPG conversion** ("convert to free form", "modernize RPG II") | + [rpg-conversion.md](rpg-conversion.md) | Progressive OPM→ILE → externalize I/O specs → `**FREE` + opcode map, cycle recipes | ## Reference Files | File | Purpose | Load When | |------|---------|-----------| | [quick-reference.md](quick-reference.md) | Tool selection by user intent | Tool choice unclear | -| [tool-catalog.md](tool-catalog.md) | Full 51-tool inventory | Need specific tool details | +| [tool-catalog.md](tool-catalog.md) | Full 54-tool inventory | Need specific tool details | | [query-flows.md](query-flows.md) | Optimal tool chains | Complex multi-step analysis | | [playbook.md](playbook.md) | Playbooks + chaining rules | Edge cases, troubleshooting | | [program-documentation.md](program-documentation.md) | Spec generation workflow | "Document program X" | +| [change-impact-analysis.md](change-impact-analysis.md) | Planned-change impact + field lineage + costed estimate | "What breaks if I change X?", "how long will it take?" | +| [object-context-matrix.md](object-context-matrix.md) | Inbound + outbound context for any object, with downloads | "Object context matrix for X" | +| [onboarding-guide.md](onboarding-guide.md) | Menu-scoped onboarding document for a developer new to the application | "Onboard a new developer on CASEMNU" | +| [version-comparison.md](version-comparison.md) | Cross-version comparison report workflow | "Compare the versions of program X" | +| [member-diff.md](member-diff.md) | Repository-wide cross-library member comparison | "Which members differ between LIB2025 and LIB2026?" | | [app-map.md](app-map.md) | 3D application map workflow | "App map of library/area X" | -| [templates/](templates/) | 4 audience-specific templates | Spec generation | +| [rpg-conversion.md](rpg-conversion.md) | Fixed-form → free-form conversion workflow | "Convert / modernize program X" | +| [rpg-opcode-map.md](rpg-opcode-map.md) | Opcode → free-form map + review flags | During conversion (Step 4) | +| [rpg-cycle-recipes.md](rpg-cycle-recipes.md) | Eliminating the cycle (L1–L9, MR, 1P → linear/SQL) | Cycle programs | +| [templates/](templates/) | Audience + conversion templates | Spec generation / conversion deliverables | ## Quick Decision @@ -37,5 +51,7 @@ User asks about iA / IBM i analysis? │ ├─ Multi-step analysis? → Load query-flows.md for optimal chain │ +├─ About to CHANGE something? → Load change-impact-analysis.md +│ └─ Document a program? → Load program-documentation.md ``` diff --git a/skills/ia/references/member-diff.md b/skills/ia/references/member-diff.md new file mode 100644 index 0000000..8bbfd48 --- /dev/null +++ b/skills/ia/references/member-diff.md @@ -0,0 +1,158 @@ +# Cross-Library Member Comparison + +Use this when the question is about the **whole repository**, not one program: +*"which members differ between our libraries?"*, *"is this application the same in +2024, 2025 and 2026?"*, *"what diverged between dev and prod?"* + +> Not the same as [version-comparison.md](version-comparison.md). That one diffs +> **one member** line by line across the libraries holding it. This one answers +> **"which members are worth diffing at all"** across the entire repository, and +> hands you the shortlist. Normal order: this first, then version-comparison on +> what it flags. + +--- + +## 1. What it rests on + +Cross-library source fingerprints the repository build must already have +produced — two SHA-256 hashes per member per library, one over the source +verbatim and one over code lines only, normalized — plus the ranking and verdict +that `ia_member_variants` and `ia_variant_summary` read. + +**If `ia_variant_summary` returns nothing, stop and say so.** The tools will +return nothing and the workbook will be empty; that is a deployment gap in the +repository build, not an analysis result. + +### The four verdicts + +| Verdict | Meaning | What to do | +|---------|---------|------------| +| `SINGLE` | One copy only | Nothing to compare | +| `IDENTICAL` | Byte-for-byte the same in every library | Safe | +| `COSMETIC` | Differs only in comments, blank lines, trailing blanks, free-form indentation or case | Ignore | +| `LOGIC` | **The code itself differs** | The shortlist — this is what people act on | + +--- + +## 2. Workflow + +Two entry points, and they take different routes. Pick one before you call anything. + +### Asked for a **report / workbook / comparison document** — exactly three steps + +``` +1. ia_variant_summary() → non-empty? proceed. Empty → §1, stop +2. python build_member_diff.py --out … → exit 0 = no data-quality blockers +3. read Overview + Members from the .xlsx (openpyxl) and narrate from that +``` + +Nothing else. In particular, on this path do **not**: + +- call `ia_member_variants` — the script pulls those rows itself and writes them + to the Members sheet. Calling it first buys the same shortlist twice, once into + the context and once into the workbook +- go looking for the underlying fingerprint objects — step 1 returning rows + already proves they exist and are populated. That check belongs in the + *diagnostic for when step 1 comes back empty*, not in a pre-flight +- hand-check the fingerprints for empty or duplicated hashes — the script's + data-quality block does exactly that and **exits 1** on a blocker. Read its + Overview findings instead of re-deriving them (and the column is `NORM_HASH`, + not `CODE_HASH`) + +The script is the report tier because it keeps tens of thousands of rows out of +the conversation. Investigating by hand first spends the context the script exists +to save. + +### Asked a **question** about divergence — query directly, no workbook + +``` +ia_variant_summary() → the shape of the problem +ia_member_variants() → defaults to LOGIC only: the shortlist +ia_member_variants(member_name='X', + verdict='*ALL') → one member, every library +``` + +Then, for anything on the LOGIC shortlist that matters, drop into +[version-comparison.md](version-comparison.md) to see the actual line changes. + +**Never call `ia_member_variants(verdict='*ALL')` just to look around** — on a real +repository that is tens of thousands of rows into the context for no benefit. The +summary tool already tells you the totals. Use `*ALL` only for a named member. + +--- + +## 3. The workbook + +``` +python .claude/skills/ia/scripts/build_member_diff.py \ + [--url http://localhost:3010/mcp] [--out FILE] [--member-type RPGLE] +``` + +Requires `pip install openpyxl` and a running MCP server. The script pulls every +member+library row over the MCP HTTP endpoint itself — **do not** pull those rows +into the conversation and hand them over; that is the whole reason the report tier +is a script. Three sheets: + +1. **Overview** — run metadata, verdict distribution, and data-quality findings +2. **Members** — one row per member, one column per library, variant label in the + cell. Three explanatory notes sit above the header; the header row and the + member column are frozen. Autofilter on; filter Verdict = LOGIC for the shortlist +3. **Detail** — one row per member+library copy, with line counts and hash prefixes + +Member-level columns, and what they actually mean: + +| Column | Meaning | +|--------|---------| +| `Member Type Differs` | The member is a different type in different libraries (`CLP` here, `CLLE` there). Grouping is by name alone, so converted programs are flagged rather than split apart | +| `Source File Differs` | The copies sit in different source physical files. Can also mean two unrelated members share a name — check the source file in Detail before concluding | +| `Max Source Lines` | Verbatim line count of the **longest** copy, comments and blanks included. Reduce across the copies — the first row read is whichever library sorted first, not the longest | +| `Line Count Delta` | Longest copy minus shortest. Length only — **not** a measure of sameness | + +`Line Count Delta` = 0 does **not** mean the copies are identical: lines changed in +place keep the count the same. Read `Verdict` for sameness. The reverse pairing is +the useful one — `IDENTICAL` with a delta above 0 is self-contradictory and is +raised as a blocker. + +Default output `member-diff-YYYYMMDD.xlsx`; for a client deliverable write it to +`docs/reports/`. + +--- + +## 4. Read the data-quality block before you believe anything + +The script checks the fingerprints for self-contradiction and writes findings to +the top of the Overview sheet. It **exits 1** when it finds a blocker. Report +blockers to the user rather than narrating the verdicts as if they were sound: + +| Finding | What it means | +|---------|---------------| +| Every `EXACT_HASH` is `e3b0c44298fc...` | That is the SHA-256 of the **empty string**. The hash routine is opened and closed but never fed any data, so every member looks identical to every other. Every verdict is meaningless. Report it and rebuild the fingerprints | +| All members share one hash | Same conclusion — distinct source cannot share a fingerprint | +| Reported `IDENTICAL` but line counts differ | A direct contradiction, and the cheapest proof the hashes are wrong | +| Only one library has fingerprints | Nothing to compare; every member can only be `SINGLE`. At least two libraries must be fingerprinted | + +A wrong report that looks right is worse than no report, so these are surfaced +loudly and never smoothed over. + +--- + +## 5. Interpretation notes + +- **Grouping is by member name alone.** A program that is `CLP` in one library and + `CLLE` in another is still the same program and still compares. `TYPE_DRIFT` + and `PF_DRIFT` flag those cases instead of splitting them apart — so a `Y` in + `PF_DRIFT` can also mean two genuinely unrelated members happen to share a name. + Check the source file in the Detail sheet before concluding. +- **Variant labels are `Base`, `v1`, `v2`, …** — `Base` is the version most + libraries share, and each further distinct version gets the next number, so a + member reads as a version history rather than as arbitrary letters. The + underlying data still ranks them `A`, `B`, `C`; the script relabels them so a + member reads as a version history. +- **Labels are deterministic**: most-shared hash first, ties broken by earliest + change timestamp, then library name. Two runs over an unchanged repository + produce the same labels. If they move between runs, something is wrong + upstream. +- **Case-folding is deliberate.** A member differing only in the case of a literal + shows as `COSMETIC` with two exact variants — visible, but not escalated. +- There is no ceiling on the labels — a member with 40 distinct versions runs to + `v39`, where the old lettering had to fall back to `V27` past the alphabet. diff --git a/skills/ia/references/object-context-matrix.md b/skills/ia/references/object-context-matrix.md new file mode 100644 index 0000000..aee8885 --- /dev/null +++ b/skills/ia/references/object-context-matrix.md @@ -0,0 +1,205 @@ +# Object Context Matrix + +Use this when the user asks for the **context matrix** of an object — *"object +context matrix for CUSTMNTR"*, *"what's the full context of CUSTMST"*, *"show me +everything around ICS100"*. The deliverable is a sectioned report of everything +the object **uses** and everything that **uses it**, followed by a download menu. + +It is the AI counterpart to the portal's Object Context Matrix screen, but a +**superset** of it: the same sections, plus whatever else the object's shape +makes relevant (§4), plus a short opening summary the screen has no room for. + +> Not the same as [program-documentation.md](program-documentation.md). That +> produces a written spec of how a program *works*. This one is purely the +> object's **surroundings** — its inbound and outbound edges — and it works for +> any object type, not just programs. + +--- + +## 1. One call gets everything + +``` +ia_object_context_matrix(object_name='CUSTMNTR', object_type='*PGM', + object_library='CASELIB') +``` + +`object_type` and `object_library` are both optional (`*ALL`). Pass them when the +user gave them, or when the bare name is ambiguous. Row 1 is always +`section = OBJECT` — the requested object itself. + +**Do not** build this from `ia_object_references` + `ia_find_object_usages` + +per-object `ia_object_lookup`. Neither of those returns the referenced object's +**attribute** (which is what splits display and printer files out from data +files) or its **description**, so that route costs one extra call per referenced +object and still has to bucket by hand. + +**If the call returns zero rows the object does not exist under that name** — +report the negative (Rule Two). Do not fall back to a similarly-named object. + +### Usage mode — read `usage_mode`, never `FILE_USAGES` + +The tool derives `usage_mode` from `REFERENCED_OBJUSG`. On `*FILE` rows the raw +codes are `I`/`O`/`U`/`C` — **Input / Output / Update / Combined** — and they +combine (`I/U`, `I/O/U`, `C/O`); the tool has already merged duplicate rows and +rendered them as `Input Update`. On `*SRVPGM`, `*MODULE` and `*BNDDIR` rows the +same `I` means *Implicit* instead. The separate `FILE_USAGES` column is empty in +current repositories — never read usage from it. + +**`C` is Combined, not Create.** It marks a workstation file opened for both +read and write — in practice always a `DSPF`. Reading it as "create" turns every +program that merely displays a screen into the program that *created* the +display file. The tool guards the inbound side so combined references to a +`DSPF`/`PRTF` land in `READ_BY`, never `CREATED_BY`; `FILE_CREATE` and +`CREATED_BY` are reserved for a genuine create and are empty in every repository +seen so far. If you ever see a display file under "Created By", the guard has +regressed. + +--- + +## 2. Sections → headings + +Render one `##` heading per section **that has rows**, with the row count, in +this order. Omit empty sections silently — do not print "none". + +| `section` | Heading | +|-----------|---------| +| `FILE_UPDATE` | Files Used In Update Mode(s) | +| `FILE_INPUT` | Files Used In Input Mode(s) | +| `FILE_OUTPUT` | Files Used In Output Mode(s) | +| `FILE_CREATE` | Files Created | +| `DISPLAY_FILE` | Display File(s) | +| `PRINTER_FILE` | Printer File(s) | +| `CALLS_PROGRAM` | Programs Called | +| `USES_SRVPGM` | Service Programs Used | +| `BOUND_MODULE` | Modules Bound | +| `USES_DTAARA` | Data Areas Used | +| `USES_BNDDIR` | Binding Directories | +| `USES_OTHER` | Other Objects Used | +| `CALLED_FROM` | Called From Program(s) | +| `UPDATED_BY` | Updated By Program(s) | +| `WRITTEN_BY` | Written By Program(s) | +| `CREATED_BY` | Created By Program(s) | +| `READ_BY` | Read By Program(s) | +| `REFERENCED_BY` | Referenced By | + +`section_seq` already sorts them — render in the order returned. + +### Columns per section family + +| Family | Columns | +|--------|---------| +| Data files (`FILE_*`) | File Library · File Name · File Description · File Attribute · Record Format · Key Field · Select/Omit | +| `DISPLAY_FILE`, `PRINTER_FILE` | File Library · File Name · File Description · File Attribute · Record Format | +| Everything else | Object Library · Object Name · Object Type · Object Description · Object Attribute | + +Add **Usage Mode** as a column only when a section mixes modes (e.g. `UPDATED_BY` +holding both `Update` and `Input Update`). Drop columns that are empty for every +row in that section — a Select/Omit column of seven blanks is noise. + +### Escape pipes in every cell value + +Object text on IBM i may contain `|` — e.g. the real description +`CLPGM1 CALLED BY COMMAND || AICAL ||`. Written into a markdown table unescaped +it ends the cell early: the description is **silently truncated** and every +column to its right shifts left and loses its last value. Nothing errors, the +row count is unchanged, and the damage is carried into the `.xlsx`, `.html`, +`.docx` and `.pdf` alike. Replace `|` with `\|` in every cell you emit — +descriptions are the only field where it realistically occurs, but the rule is +cheapest applied to all of them. + +--- + +## 3. Shape of the answer + +1. **Title** — `# Object Context Matrix — {NAME} ({TYPE}, {LIBRARY})` +2. **Summary** — 2–4 sentences from the `OBJECT` row plus the section counts: + what the object is, what it touches, what reaches it, and one risk sentence. + This is the AI's contribution; keep it factual and derived only from rows you + actually have. +3. **The section tables** (§2). +4. **Enrichment sections** (§4), if any earned their place. +5. **The download menu** (§5). + +Do not invent a "what stayed the same" or filler section. If the object has one +inbound edge and no outbound ones, the report is short — that is the finding. + +--- + +## 4. What to add beyond the portal (the superset) + +The tool covers every reference-derived section. Add these **only when the +object's own data makes them relevant** — each costs a call, so none are +automatic: + +| Add | When | Tool | +|-----|------|------| +| Copybooks | Object is a program/module and the user asked about change impact | `ia_member_copybooks` | +| Override routing | The program has any `OVRDBF` — a declared file may not be the file actually read | `ia_file_overrides`, then `ia_override_chain` if it chains | +| LFs / views over each PF | A `FILE_UPDATE` row exists — changing that PF also hits everything built over it | `ia_file_dependencies` | +| Complexity / risk framing | The user asked "is this risky to change" | `ia_code_complexity` | +| Indirect reach | `CALLED_FROM` is non-empty and the user wants the full blast radius, not just direct callers | `ia_call_hierarchy` | + +Rule of thumb: if the extra call would not change a sentence of the summary or +add a row someone would act on, skip it. A matrix that answers the question in +one call is better than a thorough one that took eight. + +--- + +## 5. Offer the downloads + +After the tables, always offer the menu — the user picks, then you build: + +``` +Download this matrix as: + 1. Markdown (.md) 2. Excel (.xlsx) 3. Word (.docx) + 4. PDF (.pdf) 5. HTML (interactive) 6. All of the above +``` + +Markdown is the source of truth. Write the `.md` first, then derive: + +``` +python .claude/skills/ia/scripts/build_context_matrix.py REPORT.md --xlsx --html +python .claude/skills/ia/scripts/convert_md_to_docx.py REPORT.md +python .claude/skills/ia/scripts/convert_md_to_pdf.py REPORT.md +``` + +`build_context_matrix.py` needs `openpyxl` for `--xlsx` (`--html` has no +dependencies). It parses the `##` headings and pipe tables out of the `.md`, so +whatever you rendered — including sections you added in §4 — is exactly what +lands in every format. Never hand-author the workbook or re-query for it. + +### Where the files go + +| Object type | Path | +|-------------|------| +| `*PGM` | `docs/program-specs/{NAME}/{NAME}_Context_Matrix.{ext}` — alongside that program's spec and flowchart | +| everything else | `docs/context-matrix/{NAME}/{NAME}_Context_Matrix.{ext}` | + +--- + +## 6. Interpretation notes + +- **The same name can appear twice under different types.** `MNTO0101` shows up + as a `*MODULE` in `READ_BY` and as a `*PGM` in `UPDATED_BY`. Both are real — + the module and the program it is bound into are separate objects. Do not + dedupe them. +- **The same name across yearly libraries is not a duplicate.** `ICS100` in + `LIB2024`, `LIB2025` and `LIB2026` are three copies of one program. Say so in + the summary, and offer [version-comparison.md](version-comparison.md) as the + next step rather than listing them as three unrelated callers. +- **A row still showing `*LIBL`** means the program did not qualify the library + *and* the name is not unique across libraries, so it could not be resolved. + Those rows come back with no attribute, description or record formats — the + blanks are unresolvable, not missing data. Say which library you think it + resolves to at runtime only if `ia_object_lookup` confirms it. +- **`detected_from`** — `S` = found in source, `O` = found in the compiled + object, `S+O` = both. An `O`-only row on a program whose source you can read + usually means the source was changed and not recompiled, or vice versa; worth a + sentence when it shows up. +- **Empty `CALLED_FROM` does not mean dead.** The object may be launched by a + scheduler, a menu outside the repository, or a command line. Say "no callers + recorded in iA", not "unused" — `ia_unused_objects` is the tool that makes that + claim. +- **`DAYS_USED = 0` with a recent `last_changed`** is a common false alarm: + usage statistics are only collected when the repository is configured to gather + them. Check `ia_repo_config` before drawing a conclusion from it. diff --git a/skills/ia/references/onboarding-guide.md b/skills/ia/references/onboarding-guide.md new file mode 100644 index 0000000..74bb2cb --- /dev/null +++ b/skills/ia/references/onboarding-guide.md @@ -0,0 +1,178 @@ +# New-Developer Onboarding Guide (menu → program → file) + +Use this when a user asks to **onboard someone onto an application** — "onboarding guide for CASEMNU", "get a new developer up to speed on this menu", "menu to program to file mapping", "what does this menu do and what does it touch", "new developer walkthrough". + +The deliverable is one readable document that answers, for a whole menu: **what the menu launches, what each program touches, what data is shared, and where to start reading.** The reader is a developer new to this application — and often new to IBM i — but not new to programming. Explain platform idioms, never `if` statements. + +``` +docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.md ← source of truth (you author this) +docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.xlsx ← built BY SCRIPT from the .md, on request +docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.docx ← existing converter, on request +docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.pdf ← existing converter, on request +``` + +> **Markdown is the source of truth.** Every other format is derived from it, so what the user reads is exactly what they download. Never hand-author the `.xlsx`. + +**Scope in this version is a `*MENU`.** A single-program deep tour (annotated source excerpts + idiom glossary) is a planned later arm of this same DocType — if the user asks for one program, generate a program spec instead ([program-documentation.md](program-documentation.md)) and say why. + +--- + +## 1. Resolve the menu + +- User named a `*MENU` → `ia_object_lookup(object_name=MENU)` confirms type, library, description. Proceed. +- User named a **library** → `ia_object_list(library=L, object_type='*MENU')`. +- User named an **application area** → `ia_application_area(area_name=A)`, then keep its `*MENU` rows. +- User named a **program** → this DocType is menu-scoped. Offer a program spec or an object context matrix instead. + +Then: + +| Menus found | Do | +|---|---| +| Exactly 1 | Proceed, and **say so** — "CASELIB has one menu, CASEMNU; generating for that." | +| More than 1 | Show them in a table with the program count each launches, and ask which. Never auto-pick the biggest. | +| Zero | Report the negative plainly (Rule Two). Offer [app-map.md](app-map.md) or a program spec. Do **not** invent a scope. | + +## 2. The data spine — run in this order + +| # | Call | Gives you | +|---|------|-----------| +| 1 | `ia_object_lookup(object_name=MENU)` | type, library, description | +| 2 | `ia_repo_config()` | repository collection date → the freshness stamp | +| 3 | `ia_call_hierarchy(program_name=MENU, direction='CALLEES')` | the programs the menu launches | +| 4 | `ia_code_complexity(library=L, limit=5000)` | one call: exec lines, subroutine/procedure/SQL/file counts and `called_by_count` for **every** member — used for the cap ranking, the Start Here ranking and each program's stats line | +| 5 | `ia_member_lookup(source_library=L, limit=5000)` | `CHANGED_DATE` + member description + member type for every member in **one** call — the freshness gate at library scale. Do **not** call `ia_object_lifecycle` per program. | +| 6 | `ia_object_context_matrix(object_name=PGM)` — one per kept program | that program's files bucketed by usage mode, each with attribute, description, record formats and key fields, plus its own callees (`CALLS_PROGRAM`) and callers (`CALLED_FROM`) | +| 7 | `ia_cl_jobs(member_name='*ALL')` | SBMJOB batch pattern — job name and job queue | +| 8 | `ia_file_dependencies(file_name=PF)` — one per distinct data PF | logical files over each PF, for the LF annotation in §5. **Skip entirely if the scope has no LF objects** (check once with `ia_object_list(library=L, object_attribute='LF')`). | +| 9 | `ia_program_summary(program_name=PGM)` — **only for the Start Here 5** | compile detail where it matters. Every other program's role comes from its object text description plus its complexity stats. | + +**Expanding the tree.** Step 3 gives level 1. Step 6's `CALLS_PROGRAM` rows give the next level for each program — follow them until no new program appears. A CL that appears as a callee is usually the batch driver; its own matrix reveals the RPG it submits. + +**Cap:** rank every reached `*PGM` by `EXEC_LINES` descending and give a detail section to the top **25**. Programs the menu launches directly are always kept regardless of rank. Everything beyond the cap goes in the "Reached but not detailed" table with name, role and how it is reached — **state the overflow count in the document. Never truncate silently.** + +## 3. Menu option numbers — not in the call hierarchy + +`ia_call_hierarchy(MENU, 'CALLEES')` returns *which* programs a menu launches but **not the option number or the option text**: those rows come from the object cross-reference fallback, so `CALL_SEQUENCE` is empty. The mapping lives in the menu's two source members, and **no `ia_*` tool exposes DDS/menu source today** — read them directly: + +- **`{MENU}QQ`, member type `MNUCMD`** — one line per option, `NNNN CALL PGM(TARGET)`. This is the authoritative option → program mapping. +- **`{MENU}`, member type `MNUDDS`** — the screen. Option numbers appear as positioned literals (`5 7'1.'`) and the option text on the same screen row (`5 10'Customer Maintenance'`). **Join by screen row**, not by sequence number. + +Retrieve both members' source lines in RRN order and read the option numbers, target programs and screen text out of them directly. + +If the `MNUCMD` member is absent, fall back to the call-hierarchy list, order it alphabetically, and **say in the document that option numbers could not be recovered** — never number the options yourself. + +> A menu option can run a command other than `CALL PGM(...)` (e.g. `SBMJOB`, `STRSQL`, a `*CMD`). Take the option's command verbatim; only map it to a program when the command names one. + +## 4. Ranking for "Start here" + +Pick **5**, in this order, skipping any that don't exist: + +1. The **front door** — the option-1 program, or the one with the highest `called_by_count`. +2. The **workhorse** — the highest `EXEC_LINES` among reached programs. +3. The **shared master file** — the data PF touched by the most programs (from your own file tables, not a guess). +4. The **batch driver** — the CL with an SBMJOB, if any. +5. The **oddity** — whichever of these is present: a program with an unusually high SQL count, an override/QTEMP pattern, a program reached only indirectly. + +One line each on *why it matters*, not what it is. + +## 5. Building the file tables + +Per program, from its context matrix rows: + +| Column | Source | +|---|---| +| File | `OBJECT_NAME` | +| Type | `OBJECT_ATTRIBUTE` — PF / LF / DSPF / PRTF | +| Over | base PF, for LF rows only (from step 8) — blank otherwise | +| Access | `USAGE_MODE` verbatim (Input / Output / Update / Combined) | +| Purpose | `DESCRIPTION`; if blank, write "no description recorded" — **never invent one** | + +Rules: + +- **Logical files stay visible.** List the LF by its own name and annotate the base PF in the "Over" column. The reader opening the source will see the LF in the F-specs; collapsing it to the PF (which [app-map.md](app-map.md) does deliberately, because a graph node per LF is noise) makes the document disagree with the code. +- **`C` on a display file means Combined** — read and written — not "create". Never render it as Create. +- **The same file can appear twice** with different `DETECTED_FROM` (`O` = from the compiled object, `S` = from source). Merge them into one row and union the access modes; note the union, e.g. `Input Update`. +- **Files in `QTEMP` and `#`-suffixed duplicates are runtime work files**, not repository objects — they surface as unenriched rows (blank attribute/description). Don't table them as data files; explain them in the program's role sentence instead (see the override pattern in §7). + +## 6. Shared data and risk + +Build one matrix of every data PF touched by **≥2** programs: rows = files, columns = programs, cells = access mode. + +Then call out, in prose: + +- Any file **updated by more than one program** — "change its layout and every one of these needs recompiling and retesting". +- The file with the **most** touching programs — the application's spine. +- Any program that updates a file it does not own by name (cross-area write). + +Keep it factual. This is not a change-impact assessment — if the user is actually about to change something, route to [change-impact-analysis.md](change-impact-analysis.md). + +## 7. "Terms you'll see" — only what this scope contains + +Detect from the metadata already fetched; include an entry **only if the scope actually has one**. 2–3 plain sentences each, plus where it appears. + +| Include when | Term | +|---|---| +| Any `*FILE` with attribute DSPF | Display file, record format, subfile (`SFL`/`SFLCTL` in the format list) | +| Any PRTF | Printer file and spooled output | +| Any LF | Logical file over a physical file | +| An `ia_cl_jobs` row | SBMJOB — user-triggered submission to a job queue. **Never say "nightly"/"scheduled"** unless a scheduler entry proves it | +| CL source shows `OVRDBF` / `OPNQRYF` / `CRTDUPOBJ` to QTEMP | The override pattern — the program reads a name that has been redirected to a job-scoped copy at runtime | +| Members typed `SQLRPGLE` | Embedded SQL inside RPG | +| `SOURCE_FORMAT` mixes FULLY FREE and MIXED/FIXED | Fixed-form vs free-form RPG | +| Consistent name prefixes/suffixes | The shop's naming convention — state the pattern you observed, e.g. `*R` suffix = RPG, `*C` suffix = CL driver | +| Any file access mode column | The `I` / `O` / `U` / `C` legend | + +No generic RPG textbook padding. If the scope has no LFs, there is no LF entry. + +## 8. Document structure + +Follow [templates/template-onboarding-guide.md](templates/template-onboarding-guide.md). Sections, in order: + +1. Header — menu, library, repository + collection date, `Author: iA by programmers.io`, freshness verdict +2. What this menu does +3. Start here (the 5) +4. Menu options table +5. Call map — a **text ASCII tree in a fenced ```text``` block. Never Mermaid.** +6. Program details — one subsection each +7. Shared data +8. Terms you'll see +9. Reached but not detailed (omit if the cap wasn't hit) +10. Where to go next + +## 9. Verification gate — before showing the document + +**Freshness.** Compare each in-scope member's `CHANGED_DATE` (step 5) with the repository collection date (step 2). Any member changed *after* collection → flag it in the header: the document describes the metadata, which is older than the source. Convert the `CYYMMDD` form (`1260708` → 2026-07-08) before comparing. + +**Cross-check every claim against your own tool output:** + +- [ ] Every program in the document appears in the call-hierarchy or a `CALLS_PROGRAM` row +- [ ] Every file row traces to a context-matrix row — no file appears that no tool returned +- [ ] Every option number came from the `MNUCMD` member, or the document says they couldn't be recovered +- [ ] Counts in the prose match the table row counts +- [ ] Overflow count stated if the cap was hit +- [ ] No access mode rendered as "Create" +- [ ] Nothing described as scheduled/nightly without evidence + +## 10. Downloads + +After showing the document, offer: **Word**, **PDF**, **Excel**, or all three. + +``` +python scripts/build_onboarding_guide.py docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.md +python scripts/convert_md_to_docx.py docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.md +python scripts/convert_md_to_pdf.py docs/onboarding/{MENU}/{MENU}_Onboarding_Guide.md +``` + +`build_onboarding_guide.py` parses the markdown tables and writes a four-sheet workbook (Menu Options, Program → File Relations, Shared Data, Program Stats). If it errors, fix the **markdown** — never patch the workbook by hand. + +## Common mistakes + +| Symptom | Cause / fix | +|---|---| +| Option numbers invented or alphabetical | You used `ia_call_hierarchy` for ordering. Read the `MNUCMD` member (§3). | +| A display file listed as "Created by" the program | `C` is Combined. §5. | +| Same file listed twice per program | Object-detected and source-detected rows not merged. §5. | +| A `#`/QTEMP file tabled as a data file | Runtime work file from an override. §5, §7. | +| "Runs nightly" | Nothing in iA says that. An SBMJOB is user-triggered. §7. | +| Document is 40 pages | The cap wasn't applied. §2. | +| Every program says "no description recorded" | The repo's object text is blank — that's a real finding, say it once in the header rather than 25 times. | diff --git a/skills/ia/references/rpg-conversion.md b/skills/ia/references/rpg-conversion.md new file mode 100644 index 0000000..591e143 --- /dev/null +++ b/skills/ia/references/rpg-conversion.md @@ -0,0 +1,182 @@ +# RPG Fixed-Form → Free-Form Conversion Reference + +Use this guide whenever a user asks to **convert, modernize, or free-form an RPG program** — RPG II (S/36 or S/38 heritage), RPG III / RPG/400 (OPM), or fixed-format RPG IV. Typical triggers: "convert X to free form", "modernize this RPG II program", "free-format RPGLE for X", "OPM to ILE". + +**Target end state is fully free-form (`**FREE`).** Conversion is *progressive*: bring an OPM program to ILE fixed format first, externalize I-specs and O-specs while still in fixed format, then convert the whole program to `**FREE`, compiling at each stage. `/FREE` (free calculations inside an otherwise fixed member) is only a transitional stepping stone — never the delivered result. + +Companion references (load on demand): + +| Doc | Purpose | +|-----|---------| +| [rpg-opcode-map.md](rpg-opcode-map.md) | Fixed-form opcode → free-form mapping, with the sites that need a human to inspect data types/timing | +| [rpg-cycle-recipes.md](rpg-cycle-recipes.md) | Eliminating the RPG cycle — primary/secondary files, L1–L9 control breaks, match fields (MR), 1P, lookahead — into explicit linear/SQL logic | +| [templates/template-modernization-summary.md](templates/template-modernization-summary.md) | The modernization summary report delivered at the end of a conversion | + +--- + +## Ground Rules + +1. **Source of truth is the iA repository.** Retrieve every source line through `ia_rpg_source` (Rule Zero of SKILL.md). Never convert from a workspace file unless the user explicitly hands you one. +2. **Preserve the original.** Convert into new deliverables under `docs/conversions/{PGM}/`; never edit the original member. On a live box the developer should archive the source member before recompiling over it. +3. **Semantic equivalence — no functional change during conversion.** The converted program must do exactly what the original did. Renaming variables to meaningful names, turning subroutines into procedures, replacing indicators with named booleans, and RLA→SQL rewrites are **separate modernization steps** — list them as opportunities in the summary, never fold them into the conversion diff. +4. **Interface stability.** Entry parameters (`*ENTRY PLIST` → `DCL-PI`), files, and callees must match the original exactly, in the same order and with the same types. +5. **Compile and test on IBM i is the acceptance gate.** iA reads source and produces the converted source, copybooks, DDS, and a summary report. Recompiling (`CRTBNDRPG`, or `CRTSQLRPGI` for SQL/embedded-SQL members) and regression testing happen on the box — state this plainly. Column-based and byte-position errors are subtle; do not claim a converted program is correct without the developer's compile-and-test result. +6. **Convert incrementally, but migrate related programs together.** One program at a time per file, reusing shared copybooks; when several programs share files or an activation group, plan them as a unit so scoping and record layouts stay consistent. + +--- + +## Step 0 — Resolve the member and detect the generation + +1. `ia_member_lookup(member_name=X)` (uppercase). Capture MEMBER_TYPE, source file, library, line count, and last-changed date/time. + - Multiple rows → present the version table and ask which member to convert. + - Zero rows → report not found (Rule Two); suggest `ia_object_lookup` with `%` wildcards. +2. Fetch the **full** source: `ia_rpg_source(member_name=X, library_name=L)`, paginating to TOTAL_LINES (from `ia_code_complexity`). A partial source silently drops specs and constructs. +3. Classify the generation and the conversion path: + + | MEMBER_TYPE (typical) | Generation | Path | + |---|---|---| + | `RPG36`, `RPG38`, and RPG II/III members | OPM, S/36 or S/38 heritage | Step 1 (OPM→ILE) → Steps 2–4 | + | `RPG`, `RPT`, `SQLRPG` | OPM RPG III / RPG/400 | Step 1 (OPM→ILE) → Steps 2–4 | + | `RPGLE`, `SQLRPGLE` (fixed or partial `/FREE`) | ILE RPG IV | Steps 2–4 | + | `RPGLE` already `**FREE` | Nothing to convert | Say so; offer the modernization opportunities instead | + + Confirm generation from the source itself, not just MEMBER_TYPE: `ia_rpg_source_stats` gives the free-format percentage, and the F/I/O-spec column layout distinguishes RPG II/III from RPG IV. +4. **Existing-output check:** look at `docs/conversions/{PGM}/`. If a conversion already exists, surface it with its date and ask before regenerating. +5. **Scope the work up front** with `ia_program_spec_bundle(program_name=X)` (complexity, files, callees, callers, parameters, subroutines in one call), `ia_member_copybooks` + `ia_copybook_impact` (shared copybooks — converting one affects every including program), and `ia_call_hierarchy(direction='CALLERS')` (who depends on this interface). Note whether the program uses the cycle (primary/secondary files, L1–L9, match fields, 1P, total-time C-specs), program-described files, or printer O-specs — these decide how much of Steps 2–4 apply. + +--- + +## Step 1 — OPM → ILE (RPG II / RPG III members only) + +Bring the source to **ILE RPG IV, still fixed format**, before any free-form work. IBM's `CVTRPGSRC` command does the bulk of this (re-columning specs, updating opcode spellings, E-specs → D-specs) — it is not 100% accurate, so review its conversion report (`CVTRPT(*YES)`) for warnings and severe errors that need manual fixing. + +- **Activation-group strategy:** compile OPM-compatible (`DFTACTGRP(*YES)`) for a quick like-for-like migration, or bound/named (`DFTACTGRP(*NO) ACTGRP(...)`) when migrating a whole application. Scoping differs between OPM and ILE (overrides, open scope, `RCLRSC`) — note this as a test focus. +- **Constructs `CVTRPGSRC` cannot handle** become manual work carried into the later steps: RPG II device files (`CONSOLE`, `KEYBORD`, `CRT`, `BSCA`) have no ILE equivalent and need redesign; complex `GOTO` webs are best restructured during Step 4. `BITON`/`BITOF`/`TESTB` and zone-manipulation opcodes need per-site verification. + +If `CVTRPGSRC` is not available to the user, the same normalization can be done by hand (re-column H/F/I/C/O specs to RPG IV positions — F-spec designation moves col 16→18, format col 19→22; E-spec arrays/tables become D-specs with `DIM`/`CTDATA`/`PERRCD`), but the output still goes through Steps 2–4. + +Compile the fixed ILE program to establish a clean baseline before continuing. + +--- + +## Step 2 — Externalize I-specs (keep fixed format) + +Read **F-spec column 22** for each file first — this one character decides everything. (RPG II/III members: normalize columns in Step 1 first.) + +| Col 22 | File is | I-spec handling | +|---|---|---| +| `E` | Externally described | I-specs are optional add-ons. Remove them; fields resolve from the external DDS. Field renames, control-level, match, field-indicator, or binary I-specs get the specific free-form treatment below. | +| `F` + external DDS exists | Program-described in source, but a DDS `*FILE` exists | Usually re-declare as externally described (`DISK(*EXT)`); handle renames/control-level/match/field-indicator/binary per the cases below. No copybook. | +| `F` + no external DDS (flat file) | Truly program-described | Build an **OVERLAY copybook** — see below. The physical `*FILE` object is never rebuilt. | + +Determining DDS existence needs a live IBM i lookup (the physical file's `PROGRAM_DESCRIBED` flag and any DDS source member). When no connection is available, treat a file with no locatable DDS as program-described and use the OVERLAY-copybook path. + +**OVERLAY copybook (program-described flat files):** one `DCL-DS` with a `CHAR(reclen)` buffer and one `OVERLAY` subfield per I-spec field at its **From** byte position. +- Packed digits = **(bytes × 2) − 1**; zoned = 1 byte/digit; blank decimal column = `CHAR`; binary(0 dec) → `INT` (or `EXTBININT(*YES)` on externally described). +- **Byte position is the identity, not the field name.** Never reposition or retype an existing subfield once a copybook is published — compiled programs depend on the exact offsets. Different programs naming the same bytes differently get **sibling OVERLAY subfields at the same offset**; the same bytes needed as both numeric and char get two typed siblings at that offset (no `%DEC`/`%CHAR` needed). +- **No `QUALIFIED`** keyword — dependent programs reference fields by their plain names. +- **Multi-record-format files** (multiple `NS` entries with a record-ID byte test) get one shared read-target DS (buffer + discriminator) plus one DS per format; after each read, assign the raw buffer into the matching format DS, then read that format's subfields. Never a single flat DS combining all formats — the byte ranges overlap. +- Name the copybook `{FILE}CPY`; **check `ia_copybook_impact` and reuse an existing one** rather than regenerating it. + +Externally described files needing RPG-IV I-spec functions (renames, control level, match, field indicators, binary) — apply the specific free-form equivalents; the control-level and match-field cases require **eliminating the cycle** (see Step 4 and [rpg-cycle-recipes.md](rpg-cycle-recipes.md)). + +`READ`/`CHAIN` on a program-described file **must carry the DS operand** — without it the record is read but no fields are populated. + +--- + +## Step 3 — Externalize O-specs (keep fixed format) + +O-specs are not allowed in `**FREE`, so every `EXCEPT` output must become an operation code first. + +**Printer files (`PRINTER`, e.g. `QSYSPRT`):** externalize to a **DDS PRTF** member. +- Each O-spec record group (`H`/`D`/`T`/`E` line) → a DDS record format; end position → start = **end − length + 1**; edit codes → `EDTCDE`/`EDTWRD`; `SPACEB/SPACEA/SKIPB/SKIPA` carried as DDS keywords. +- Overflow: the `OF`/`OA`–`OG` indicator becomes an `OFLIND(*INxx)` keyword on the F-spec (an OF indicator is not valid for an externally described printer file). A program-described default `QSYSPRT` gets a named PRTF. +- `EXCEPT name` → `WRITE format`; indicator-conditioned lines → `IF`/`ENDIF` guards. + +**Program-described DISK files:** replace `EXCEPT` per the file's DDS status — + +| O-spec pattern | Program-described (no DDS) | Externally described (re-declared `E`) | +|---|---|---| +| `EXCEPT` all fields | `UPDATE file ds;` | `UPDATE recfmt;` / `WRITE recfmt;` | +| `EXCEPT` subset of fields | fill the copybook DS via CHAIN/READ, assign only the written subfields, then `UPDATE file ds;` | `UPDATE recfmt %FIELDS(f1:f2);` | +| `EXCEPT ADD` (new record) | `WRITE file ds;` | `WRITE recfmt;` | +| `EXCEPT DEL` (delete) | `DELETE file;` (file name) | `DELETE recfmt;` | + +`%FIELDS` is valid **only** for externally described files. `WRITE`/`UPDATE` on a program-described file need the DS operand (omitting it writes zeros). Indicator conditioning (`O N51`, `O 76`) becomes an `IF`/`ENDIF` guard on both paths. + +--- + +## Step 4 — Convert the whole program to `**FREE` + +Convert every remaining spec, preserving order: **`**FREE` (line 1) → CTL-OPT → DCL-F → DCL-DS/DCL-S/DCL-C/DCL-PR → main logic → procedures → subroutines.** + +1. **H → CTL-OPT** — merge multiple H-specs into one statement; carry `DFTACTGRP`/`ACTGRP`/`BNDDIR`/`OPTION`. RPG II H-spec options (date format, currency) become CTL-OPT keywords. +2. **F → DCL-F** — usage (`IF`→`USAGE(*INPUT)`, `UF`→`*UPDATE`, `O`→`*OUTPUT`), `KEYED`, `RENAME`, `PREFIX`, `OFLIND`, `INFDS`, `USROPN`; `DISK(*EXT)` for externally described, `DISK(reclen)` for program-described. +3. **D → DCL-S / DCL-DS / DCL-C / DCL-PR** — carry `INZ`, `DIM`, `OVERLAY`, `LIKEDS`, `QUALIFIED`, etc. Data-type mapping: `10I 0`→`INT(10)`, `9P 2`→`PACKED(9:2)`, `7S 2`→`ZONED(7:2)`, `30A`→`CHAR(30)`, `50A VARYING`→`VARCHAR(50)`, `D`/`T`/`Z`→`DATE`/`TIME`/`TIMESTAMP`. +4. **`*ENTRY PLIST` → `DCL-PI`** — same parameter order/types/lengths (interface stability). `CALL 'PGM'`+`PARM` → `DCL-PR ... EXTPGM('PGM')` + `CALLP`. +5. **C-specs → free form** using [rpg-opcode-map.md](rpg-opcode-map.md). `EVAL`/`Z-ADD`/`Z-SUB` → assignment; `SETON/SETOFF` → `*INxx = *ON/*OFF`; `GOTO`/`TAG`/`CABxx` → structured logic (backward branch = loop, forward = `IF` skip, branch-to-end = `LEAVE`/`RETURN`/`LEAVESR`); `MOVE`/`MOVEL` → assignment or the right conversion BIF **after inspecting both source and target types**; embedded SQL `C/EXEC SQL … C/END-EXEC` → `EXEC SQL …;`. `BEGSR`/`ENDSR`/`EXSR` stay valid as-is. +6. **Cycle programs — eliminate the cycle.** The RPG cycle, control-level indicators (L1–L9), match fields (M1–M9/MR), 1P, and lookahead have no `**FREE` equivalent; convert them to explicit logic per [rpg-cycle-recipes.md](rpg-cycle-recipes.md): a priming read + `DOW NOT %EOF` loop, control breaks via saved previous-value comparison, matched files via an SQL `INNER JOIN` cursor (batch/reporting — member becomes SQLRPGLE) or `CHAIN`+`%FOUND` (transactional). Convert cycle programs carefully and test extensively — this is the most error-prone case. + +**Indicators:** preserve `*INxx` behavior exactly during conversion; where a result indicator only feeds a following test, use the matching BIF (`%EOF`/`%FOUND`/`%ERROR`/`%EQUAL`). Converting indicators to named boolean variables is a *modernization* step, not part of the conversion. + +### Converted-source header (put at the top of the output) + +```rpgle +// ===================================================================== +// {PGM} — converted to **FREE from {MEMBER_TYPE} +// Source: {LIBRARY}/{SRCPF}/{PGM} changed {YYYY-MM-DD HH:MM} {N} lines +// Converted {YYYY-MM-DD} by iA by programmers.io — semantics preserved. +// Must be compiled and regression-tested on IBM i before use. +// Items needing developer attention are marked // REVIEW: ... +// ===================================================================== +``` + +Mark any line whose correctness depends on data types, ranges, or timing that only the developer can confirm with a short `// REVIEW: {what to check}` comment — e.g. every `MOVE`/`MOVEL` with a type or length change, `GOTO` webs that could not be cleanly structured, bit/zone opcodes, device files, and each eliminated-cycle block. These are the same items to list in the summary report. + +Save under `docs/conversions/{PGM}/`: converted source `{PGM}.rpgle` (or `.sqlrpgle`), copybooks `{FILE}CPY.rpgleinc`, printer DDS `{NAME}.prtf`. + +--- + +## Step 5 — Compile-and-fix, then test (developer's step on IBM i) + +Conversion is only finished when the member compiles and passes testing on the box: + +1. Compile the converted member (`CRTBNDRPG`, or `CRTSQLRPGI` for SQL/embedded-SQL). Compile any generated DDS first (`CRTPRTF` for printer files). +2. Read the compile errors, fix, recompile — repeat until clean. Ask the user to paste errors if they want iA to help resolve them; for an unfamiliar message, point them at the IBM i ILE RPG documentation for that opcode/message. +3. Regression-test against known inputs, paying special attention to OPM↔ILE behavioral differences, byte-position alignment on program-described files, and cycle→linear correctness (control-break totals, matched-record logic). + +iA cannot compile or run the program — never report compiled/tested status without the developer's own result. + +--- + +## Step 6 — Modernization summary report + +Deliver `docs/conversions/{PGM}/{PGM}_Modernization_Summary.md` from [templates/template-modernization-summary.md](templates/template-modernization-summary.md): what was converted per spec type (H→CTL-OPT, F→DCL-F, D→declarations, C→free form), files externalized (copybooks and DDS created), how the cycle was eliminated if applicable, every `// REVIEW:` item with why it needs attention, interface-stability confirmation (entry params, files, callees unchanged — checked against the Step 0 bundle and `ia_call_parameters`), compile/test status, and the follow-up modernization opportunities (procedures, named indicators, RLA→SQL, error monitors) that were deliberately left out of the conversion. + +--- + +## Batch / portfolio mode + +"Which programs should we convert first?" → an analysis-only sweep, no conversion: + +1. `ia_object_list` / `ia_member_lookup` wildcards to enumerate candidate members; `ia_rpg_source_stats(member_name=*ALL)` for free-format %. +2. `ia_code_complexity(member_name=*ALL)` — rank by GOTO count, executable lines, file count. +3. Present a ranked table: easiest wins first (no cycle, full-procedural files, already-high free-format %, no program-described files) → hardest last (cycle + match fields, program-described flat files, device files, heavy `GOTO`/bit manipulation). + +--- + +## Key rules (never violate) + +| Rule | Why | +|------|-----| +| Fetch the full source (paginate to TOTAL_LINES) before analyzing or converting | A partial source silently drops constructs | +| Preserve semantics — no renames, procedures, or SQL refactors mixed into the conversion | The diff must be reviewable for behavioral equivalence | +| Inspect source and target types before converting any `MOVE`/`MOVEL` | Packed/zoned/char reinterpretation is the top source of silent data corruption | +| Never rename or reposition fields of a program-described file | Byte positions are the identity; a change breaks every sibling program without a compile error | +| Check `ia_copybook_impact` and reuse a shared copybook; only append new subfields at the end | Compiled programs depend on exact offsets | +| The cycle and match fields must be eliminated for `**FREE`, not preserved | They have no free-form equivalent | +| Compile and regression-test on IBM i before trusting the result | iA does static conversion; the box is the acceptance gate | + +--- + +*Analysis powered by iA from [programmers.io](https://programmers.io/ia/)* diff --git a/skills/ia/references/rpg-cycle-recipes.md b/skills/ia/references/rpg-cycle-recipes.md new file mode 100644 index 0000000..39f35c4 --- /dev/null +++ b/skills/ia/references/rpg-cycle-recipes.md @@ -0,0 +1,141 @@ +# Eliminating the RPG Cycle — Rewrite Recipes + +Companion to [rpg-conversion.md](rpg-conversion.md) Step 4. The RPG program cycle, control-level indicators (L1–L9), match fields (M1–M9/MR), 1P output, and lookahead **have no equivalent in fully free-form (`**FREE`)**. To convert a cycle program you must eliminate the cycle and re-express its implicit control flow as explicit code. This is the most error-prone conversion — rewrite carefully and test extensively. + +The cycle is compiler-generated control flow: it reads the next record, sets record-identifying and control-level indicators, runs **total-time** calculations/output when a control field changed, then runs **detail-time** calculations/output for the new record, and repeats until LR. Making it explicit means writing every one of those implicit steps out — in the right order. + +## Recipe 0 — Detection cheat sheet + +| Signal | Where to look | +|---|---| +| Primary / secondary files | F-spec designation `P`/`S` — col 16 (RPG II/III), col 18 (fixed RPG IV) | +| Control levels | `L1`–`L9` on I-spec field lines (RPG II/III 59–60; RPG IV 63–64); total-time C-specs with `L0`–`L9`/`LR` in cols 7–8 | +| Match fields | `M1`–`M9` on I-spec field lines (RPG II/III 61–62; RPG IV 65–66); `MR` tested anywhere | +| Cycle output | O-spec type `H`/`D`/`T` in the record-id line, conditioned by `1P`, `L1`–`L9`, `LR`, `OF` | +| Lookahead | `**` in I-spec record-identifying-indicator positions | +| Forced read order | `FORCE` opcode | +| End-of-program control | `E` in F-spec end-of-file column; LR conditioning | + +## Recipe 1 — Primary-file read loop (simple cycle, no level breaks) + +The implicit cycle read becomes a priming read + `DOW`: + +```rpgle +// before (implicit): FTRANS IP E DISK drives the loop; detail calcs run per record +**FREE +DCL-F trans DISK(*EXT) USAGE(*INPUT); // designation P removed + +READ trans; +DOW NOT %EOF(trans); + // --- detail-time calculations, in original C-spec order --- + READ trans; +ENDDO; + +// --- LR total-time calculations here (they ran once at end of file) --- +*INLR = *ON; +``` + +Rules: detail calcs keep their original order; anything conditioned by `LR` moves **after** the loop; record-identifying indicators (`01`…) for a single-format file simply disappear — logic conditioned on them runs unconditionally inside the loop. + +## Recipe 2 — Control-level breaks (L1–L9) + +The trap: on a break, the cycle runs the totals for the **previous** group *before* processing the record that caused the break, and at end-of-file it fires **all** levels once more. A naive "compare after processing" rewrite double-counts or drops the last group. + +```rpgle +**FREE +DCL-F trans DISK(*EXT) USAGE(*INPUT); +DCL-S prevDept LIKE(dept); // one prev-var per control field +DCL-S prevRegn LIKE(regn); +DCL-S firstRec IND INZ(*ON); + +READ trans; +DOW NOT %EOF(trans); + IF NOT firstRec; + // Cascade: a higher-level change implies every lower level also broke. + // Replicate the source's total-time order exactly (see ordering rule below). + IF regn <> prevRegn OR dept <> prevDept; + deptTotals(); // L1 total-time block + ENDIF; + IF regn <> prevRegn; + regnTotals(); // L2 total-time block + ENDIF; + ENDIF; + firstRec = *OFF; + prevDept = dept; + prevRegn = regn; + + // --- detail-time calculations (accumulate into dept/regn/grand totals) --- + READ trans; +ENDDO; + +IF NOT firstRec; // end of file = final break at every level + deptTotals(); + regnTotals(); +ENDIF; +grandTotals(); // LR block +*INLR = *ON; +``` + +**Ordering rule (do not assume L1-first):** the example calls `deptTotals()` (L1) before `regnTotals()` (L2) because that is the conventional source arrangement, but the RPG cycle runs total-time calculations **in the order they appear in the source**, not by any fixed lowest-to-highest level rule. Read the original C-specs and emit the `Lx`-conditioned blocks in their **original source sequence** — if the source placed an L2 block before an L1 block, preserve that. The `firstRec`/end-of-file calls must follow the same order. Getting this wrong reorders subtotals on the report and can misplace accumulator resets. + +Checklist per instance: cascade (higher break ⇒ all lower totals fire), **total-time blocks emitted in original source order** (not assumed L1-first), first-record suppression, end-of-file firing, total blocks reset their accumulators exactly where the original `Lx`-conditioned calcs did, composite control fields compared as a group. + +## Recipe 3 — Matching records (M1–M9 + MR) + +The cycle merges a primary and secondary file on match-field order: all matching primary records process first, then matching secondaries; unmatched secondaries never reach detail time; `MR` is ON while the current record's match key equals the other file's. Three rewrite options, in order of preference: + +1. **SQL join** (batch/reporting): `EXEC SQL DECLARE c CURSOR FOR SELECT … FROM primary INNER JOIN secondary ON key… ORDER BY key;` — semantically `MR = *ON` rows only. If the program also processed unmatched primaries, use LEFT JOIN and test for null secondaries. Member becomes SQLRPGLE (compile with `CRTSQLRPGI`). +2. **CHAIN per primary record** (transactional, secondary is keyed): read the primary loop; `CHAIN key secondary; matched = %FOUND(secondary);` — replaces `MR` tests. Only valid when each primary matches ≤1 secondary, or the secondary is READE-looped. +3. **True two-file merge loop** (general case, duplicates on both sides): read both files, repeatedly compare keys, advance the file with the lower key, set `matched` while keys are equal. This is the only faithful equivalent when both files have repeating match values — write it explicitly and test it hard. + +`MR` timing trap: at total time the cycle's `MR` reflects the **previous** record. Any `MR`-conditioned total-time calc needs a `prevMatched` variable, not `matched`. + +## Recipe 4 — Total-time output (`T` lines, EXCEPT at total time) + +Move each `T`-line's field list into the corresponding `totals()` procedure from Recipe 2 as a `WRITE` to the externalized PRTF format. Blank-after (`B` in the field line) becomes an explicit `field = *ZEROS/*BLANKS;` after the write — the cycle did that reset implicitly. + +## Recipe 5 — Headings, 1P, and overflow + +```rpgle +DCL-F rpt PRINTER OFLIND(*IN70) USROPN; + +writeHeadings(); // replaces 1P output (before any read) +… +DCL-PROC printDetail; + IF *IN70; // replaces OF-conditioned H lines + writeHeadings(); + *IN70 = *OFF; + ENDIF; + WRITE detail; +END-PROC; +``` + +1P output ran once before the first read — hoist it above the priming read; it may not reference input fields. Fetch-overflow (`F` in the O-spec) means the overflow check ran at that specific line, not at the next cycle boundary — keep the check adjacent to that write. L-spec form/overflow line numbers move to the PRTF (`FORMLEN`/`OFLIND` handling). + +## Recipe 6 — Lookahead fields (`**`) + +Buffer the next record explicitly: + +```rpgle +READ trans nextDs; // priming read into the lookahead buffer +DOW NOT %EOF(trans); + currDs = nextDs; // process the buffered record… + READ trans nextDs; // …while nextDs already holds the lookahead + atEof = %EOF(trans); + // original lookahead field references → nextDs subfields; the cycle filled them + // with '9's / *HIVAL at EOF — replicate if the logic tests for that sentinel + … +ENDDO; +``` + +Verify every lookahead reference and the EOF-sentinel behavior per site. + +## Recipe 7 — FORCE + +`FORCE file` overrode the cycle's next-record selection for one cycle. There is no free-form shortcut: restructure the read loop so the selection logic is explicit (usually a `SELECT` choosing which file to READ next). Document the original selection intent in a `// REVIEW:` note. + +## Recipe 8 — Halt indicators (H1–H9) + +The cycle paused with an operator message at `*GETIN` when an Hx was ON. Rewrite as explicit error handling: set an error flag, write/send a message, and decide terminate-vs-continue in code. If the program relied on the operator's choice, surface that as a behavioral decision the user must make. + +*Analysis powered by iA from [programmers.io](https://programmers.io/ia/)* diff --git a/skills/ia/references/rpg-opcode-map.md b/skills/ia/references/rpg-opcode-map.md new file mode 100644 index 0000000..743a7f4 --- /dev/null +++ b/skills/ia/references/rpg-opcode-map.md @@ -0,0 +1,112 @@ +# Fixed-Form Opcode → Free-Form Map + +Companion to [rpg-conversion.md](rpg-conversion.md). Straight syntax changes convert silently; the **Review** column flags the sites where correctness depends on data types, indicator lifetime, or timing that a human must confirm — mark those with a `// REVIEW:` comment in the output. Two-factor arithmetic note: when factor 1 is blank, the result field is also the first operand (`ADD AMT TOTAL` → `TOTAL += AMT`). + +## Arithmetic + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `ADD f1 f2 res` / `ADD f2 res` | `res = f1 + f2;` / `res += f2;` | — | `H` extender → `EVAL(H)` | +| `SUB`, `MULT`, `DIV` | `-`, `*`, `/` | — | `DIV`+`MVR` pair → review precision | +| `MVR res` (after DIV) | `q = %DIV(a:b); res = %REM(a:b);` | ✔ | Replace the DIV too; integer semantics must match | +| `Z-ADD f2 res` | `res = f2;` | — | Zeroes then adds — plain assignment is equivalent | +| `Z-SUB f2 res` | `res = -f2;` | — | | +| `SQRT f2 res` | `res = %SQRT(f2);` | — | | +| `XFOOT arr res` | `res = %XFOOT(arr);` | — | | +| Result indicators HI/LO/EQ on arithmetic | `IF res > 0;` / `< 0` / `= 0` after the assignment | ✔ | Only if the indicator is actually tested elsewhere | + +## MOVE family + +Always inspect both source and target definitions before converting — this is the highest-risk group. + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `MOVE` same type + length | `res = f2;` | — | | +| `MOVE` char→char, target shorter/longer | `EVALR res = f2;` | ✔ | MOVE right-adjusts; confirm truncation side | +| `MOVE`/`MOVEL` numeric→char | `res = %EDITC(f2:'X');` or `%CHAR` | ✔ | MOVE keeps zone/sign encoding — `%CHAR` does not reproduce a trailing-sign zone | +| `MOVE`/`MOVEL` char→numeric | `res = %DEC(f2:len:dec);` | ✔ | MOVE ignores invalid chars differently; wrap in MONITOR if input is untrusted | +| `MOVEL` left-adjust | `res = f2;` (char) / `%SUBST` | ✔ | | +| `MOVE`/`MOVEL` date↔numeric/char | `%DATE`/`%CHAR`/`%DEC` with a format | ✔ | Confirm the legacy field's implied format (H-spec date options) | +| `MOVEA` non-overlapping | `%SUBARR` / element loop | ✔ | | +| `MOVEA` overlapping ranges or char↔array tricks | emulate byte semantics explicitly | ✔ | No single BIF; verify against data | +| `MHHZO/MHLZO/MLHZO/MLLZO` | `%BITAND`/`%BITOR` surgery | ✔ | Zone/digit manipulation — verify against data | + +## Compare & branch + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `IFxx/ANDxx/ORxx/WHxx/DOWxx/DOUxx` | `IF/AND/OR/WHEN/DOW/DOU` with `= <> < <= > >=` | — | | +| `COMP f1 f2` + HI/LO/EQ | `IF f1 > f2; … ELSEIF …` at the sites that test the indicators | ✔ | `COMP` is not valid in free form; indicator lifetime matters | +| `CASxx f1 f2 SUBR` | `SELECT; WHEN f1 xx f2; EXSR subr; …ENDSL;` | — | Trailing `CAS` (no relation) → `OTHER` | +| `CABxx f1 f2 TAG` | Structured rewrite: loop / guard / early exit | ✔ | See patterns below | +| `GOTO`/`TAG` | `DOW/DOU/ITER/LEAVE/LEAVESR/RETURN/IF` | ✔ | Backward = loop; forward = `IF` skip; branch-to-end = `LEAVE/RETURN/LEAVESR`; crossing edges need a manual restructure | +| `DO n idx` … `ENDDO` | `FOR idx = 1 TO n; … ENDFOR;` | — | `DO` with START/STEP maps to FOR clauses | +| `ITER`, `LEAVE`, `LEAVESR`, `RETRN`→`RETURN` | same | — | | + +**GOTO patterns:** (1) `TAG` at top + unconditional `GOTO` back = `DOW *ON` loop with `LEAVE` at exit tests. (2) Conditional `GOTO` over a block to a later `TAG` = invert the condition into `IF … ENDIF`. (3) Several `CABxx`/`GOTO` to one exit `TAG` = guard clauses with `RETURN`/`LEAVESR`. `GOTO` is legal in fixed-format ILE but not in free form, so it must be restructured, not carried over. + +## Indicators & flow + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `SETON`/`SETOF(F)` with indicators in result columns | `*INxx = *ON;` / `*OFF;` | — | `SETON LR` → `*INLR = *ON;` | +| Conditioning indicators (cols 9–11, `N` prefix) | Wrap statement in `IF *INxx;` / `IF NOT *INxx;` | — | Preserve the exact indicator; renaming is a later modernization step | +| `EXSR` / `BEGSR` / `ENDSR` | unchanged — valid free form | — | Subroutine→procedure conversion is a later step, not part of conversion | +| Halt indicators H1–H9 as targets | explicit error handling | ✔ | See [rpg-cycle-recipes.md](rpg-cycle-recipes.md) Recipe 8 | +| `*INU1`–`*INU8` tests | unchanged | — | External job switches still work in free form | + +## Database & device I/O + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `CHAIN key file __(HI=nf)` | `CHAIN key file; IF NOT %FOUND(file);` | — | Key list: `CHAIN (k1:k2) file;` | +| `READ/READE/READP/READPE __(EQ=eof)` | `READ file; IF %EOF(file);` | — | Program-described files: DS operand mandatory | +| `SETLL` (EQ=exact) / `SETGT` | `SETLL key file; IF %EQUAL(file);` | — | HI slot (no record) → `NOT %FOUND` | +| `WRITE`/`UPDAT`→`UPDATE`/`DELET`→`DELETE`/`UNLCK`→`UNLOCK` | same opcodes | — | Error indicator (LO) → `(E)` extender + `%ERROR` | +| `KLIST`/`KFLD` | inline `(k1:k2:…)` key lists or `%KDS(ds)` | — | One KLIST used many times → consider a DS + `%KDS` | +| `EXCPT`→`EXCEPT name` | replace per the O-spec table in [rpg-conversion.md](rpg-conversion.md) Step 3 | ✔ | `**FREE` forbids O-specs — every `EXCEPT` must become `WRITE`/`UPDATE`/`DELETE` | +| `EXFMT`, `POST`, `REL`, `ACQ`, `NEXT` | valid free-form opcodes | — | Workstation semantics unchanged | +| `FORCE` | eliminate — explicit read selection | ✔ | Cycle-only; Recipe 7 | +| `FEOD`, `OPEN`, `CLOSE` | same | — | | +| `SPECIAL`/`CONSOLE`/`KEYBORD`/`CRT`/`BSCA` devices | no modern equivalent — redesign | ✔ | S/36-era device files; needs a redesign decision | + +## Strings + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `CAT f1 f2` (`:n` blanks) | `res = %TRIMR(f1) + ' … ' + f2;` | ✔ | Blank-count semantics; plain `+` when factor untrimmed | +| `SUBST` | `res = %SUBST(str:start:len);` | — | `P` extender = pad — clear target first | +| `SCAN` | `pos = %SCAN(pat:str{:start});` | — | Found-indicator → `IF pos > 0;` | +| `CHECK`/`CHECKR` | `%CHECK`/`%CHECKR` | — | | +| `XLATE` | `%XLATE(from:to:str)` | — | | +| `TESTN` | `MONITOR; n = %DEC(s:l:d); ON-ERROR; … ENDMON;` | ✔ | TESTN's three indicator outcomes have no single BIF | +| `TESTZ` | zone test — emulate explicitly | ✔ | | +| `BITON`/`BITOF(F)`/`TESTB` | `%BITOR`/`%BITAND`/`%BITNOT` expressions | ✔ | Verify bit masks against usage | + +## Arrays, tables, data areas, dates, misc + +| Fixed form | Free form | Review | Notes | +|---|---|---|---| +| `LOKUP`→`LOOKUP` (EQ) | `idx = %LOOKUP(arg:arr{:start});` | ✔ | HI/LO variants → `%LOOKUPGT/GE/LT/LE`; tables → `%TLOOKUP` | +| `OCCUR` | `%OCCUR(ds) = n;` / `n = %OCCUR(ds);` | — | | +| `SORTA` | `SORTA arr;` | — | Alternating-table sorts need care | +| E-spec compile-time array/table | `DCL-S arr CHAR(n) DIM(m) CTDATA PERRCD(k);` + `**CTDATA arr` section | — | Alternating tables → paired `ALT` definitions | +| `*LIKE DEFINE f1 res` | `DCL-S res LIKE(f1);` | — | | +| `*NAMVAR DEFINE` / `*DTAARA DEFINE` | `DCL-S`/`DCL-DS … DTAARA('LIB/NAME')` | ✔ | Confirm lock semantics with `IN`/`OUT` sites | +| `IN`/`OUT` (data area) | `IN {*LOCK} dtaara;` / `OUT dtaara;` | — | | +| `TIME res` | `%TIME()` / `%DATE()` / `%TIMESTAMP()` | ✔ | 6/12/14-digit numeric targets encode date+time — match the layout | +| `UDATE`, `*DATE`, `UDAY/UMONTH/UYEAR` | `%DATE()` + `%SUBDT`, or keep — still supported | — | H-spec date-format option decides the edit pattern | +| `ADDDUR`/`SUBDUR` | `d2 = d1 + %DAYS(n);` etc.; duration form of SUBDUR → `%DIFF` | — | | +| `EXTRCT` | `%SUBDT(src:*MSECONDS…*YEARS)` | — | | +| `TEST (D/T/Z)` | `TEST(DE) *fmt field;` — unchanged | — | | +| `CALL 'PGM'` + `PARM` list | `DCL-PR pgm EXTPGM('PGM'); … END-PR;` + `CALLP pgm(p1:p2);` | ✔ | Build the PR from the PLIST; `ia_call_parameters` confirms shapes | +| `CALLB` | bound `CALLP` with `EXTPROC` | ✔ | | +| `PLIST`/`PARM` (`*ENTRY`) | `DCL-PI` | — | Order/types identical — interface stability rule | +| `DSPLY` | unchanged | — | | +| `DUMP` | `DUMP(A);` | — | | +| `DEBUG` | drop + note | ✔ | Pre-ILE debugging aid; obsolete | +| `SHTDN` | `IF %SHTDN;` | — | | +| `C/EXEC SQL` … `C+` … `C/END-EXEC` | `EXEC SQL …;` | — | Continuation lines join naturally | +| `FREE` (opcode) | obsolete — redesign | ✔ | OPM opcode (deactivate program) | + +*Analysis powered by iA from [programmers.io](https://programmers.io/ia/)* diff --git a/skills/ia/references/templates/README.md b/skills/ia/references/templates/README.md index 6b52c30..fa1c2e6 100644 --- a/skills/ia/references/templates/README.md +++ b/skills/ia/references/templates/README.md @@ -14,6 +14,10 @@ This directory contains specialized documentation templates for different audien | [`template-operations.md`](template-operations.md) | Operations, Support Team | Runtime, monitoring, troubleshooting | 10 sections + operational readiness | Operations runbooks, support guides | | [`template-testcases.md`](template-testcases.md) | QA / UAT Testers | Manual execution test scripts, traceability | 6 sections + traceability matrix + sign-off | Test case documents, UAT preparation | +**Conversion deliverable template** (used by [rpg-conversion.md](../rpg-conversion.md), not audience-selected): [`template-modernization-summary.md`](template-modernization-summary.md) — the modernization summary report delivered at the end of an RPG fixed-form → free-form conversion. + +**Onboarding deliverable template** (used by [onboarding-guide.md](../onboarding-guide.md), not audience-selected): [`template-onboarding-guide.md`](template-onboarding-guide.md) — the menu-scoped New-Developer Onboarding Guide. Its subject is a `*MENU`, not a program, so it is outside the audience-selection logic below. + --- ## Template Selection Logic diff --git a/skills/ia/references/templates/template-modernization-summary.md b/skills/ia/references/templates/template-modernization-summary.md new file mode 100644 index 0000000..97df069 --- /dev/null +++ b/skills/ia/references/templates/template-modernization-summary.md @@ -0,0 +1,80 @@ +# RPG Modernization Summary — {PROGRAM_NAME} + +**Author:** iA by programmers.io +**Date:** {YYYY-MM-DD} +**Source:** {LIBRARY}/{SRCPF}/{PROGRAM_NAME} · {MEMBER_TYPE} · changed {YYYY-MM-DD HH:MM} · {N} lines +**Converted:** docs/conversions/{PGM}/{PGM}.rpgle ({M} lines) · target **`**FREE`** +**Workflow:** [rpg-conversion.md](../rpg-conversion.md) + +--- + +## 1. Outcome + +| Item | Result | +|---|---| +| Generation | RPG II (S/36) / RPG III / RPG/400 (OPM) / fixed RPG IV | +| OPM → ILE step needed | yes / no | +| End state | fully free-form `**FREE` | +| Cycle program | yes (eliminated — see §4) / no | +| Compile status | **not yet compiled** / compiled clean by user on {date} / errors outstanding | +| Items flagged for review | {K} (see §6) | + +## 2. Changes by specification type + +| Spec | Original | Converted to | Notes | +|---|---|---|---| +| H | {n} H-specs | one `CTL-OPT` | activation group: {DFTACTGRP(*YES) / ACTGRP(...)} | +| F | {n} F-specs | `DCL-F` | {externally described / program-described DISK(reclen) / printer} | +| I | {n} I-specs | {removed (external DDS) / OVERLAY copybook(s) / EXTFLD rename / cycle eliminated} | see §3 | +| O | {n} O-specs | {WRITE/UPDATE/DELETE / DDS PRTF} | see §3 | +| D | {n} D-specs | `DCL-S` / `DCL-DS` / `DCL-C` / `DCL-PR` | | +| C | {n} C-specs | free-form opcodes + BIFs | GOTO/CABxx restructured: {n} | +| `*ENTRY PLIST` | {n} parameters | `DCL-PI` | order/types unchanged | + +**Indicators:** {how *INxx were handled — preserved during conversion; result indicators mapped to %EOF/%FOUND/%ERROR/%EQUAL where they only feed a following test}. + +## 3. Files externalized + +| File (Library) | Device | Described | Action taken | +|---|---|---|---| +| {FILE} ({LIB}) | DISK/PRINTER/WORKSTN | Externally / Program-described | {re-declared `*EXT` / OVERLAY copybook `{FILE}CPY` (new/reused) / DDS PRTF `{NAME}.prtf`} | + +Copybooks created/reused: {list}. DDS members created: {list}. Program-described byte positions preserved (digits = bytes×2−1 for packed): ✅/❌. + +## 4. Cycle elimination *(cycle programs only)* + +How the cycle was made explicit: priming read + `DOW NOT %EOF` loop; control breaks (L1–L9) via saved previous-value comparison in **original total-time source order**; match fields (M1–M9/MR) via {SQL INNER JOIN cursor / CHAIN + %FOUND / two-file merge loop}; 1P/headings hoisted; lookahead/FORCE/halt handling. Recipe references: [rpg-cycle-recipes.md](../rpg-cycle-recipes.md). + +## 5. Interface stability (checked against iA, not the output) + +- Entry parameters match the source PARAMS (name/type/length, in order): ✅/❌ +- All original files present, none added: ✅/❌ +- All callees still called; `CALL`→`CALLP` prototypes match `ia_call_parameters`: ✅/❌ +- Callers unaffected ({n} from `ia_call_hierarchy`): ✅/❌ + +## 6. Items needing developer attention (`// REVIEW:` in the source) + +| # | Source line(s) | What to confirm | +|---|---|---| +| 1 | {NNN} | {e.g. MOVE numeric→char: sign/zone semantics} | +| 2 | {NNN} | {e.g. GOTO web restructured — verify control flow} | + +Common categories: MOVE/MOVEL type or length changes, DIV/MVR precision, restructured GOTO, bit/zone opcodes, device files with no ILE equivalent, each eliminated-cycle block. + +## 7. Compile & test (developer's step on IBM i) + +Compile: `CRTBNDRPG` (or `CRTSQLRPGI` for SQL/embedded-SQL); DDS first (`CRTPRTF`). Record errors and fixes here: + +| Attempt | Error | Fix | +|---|---|---| +| 1 | {message} | {change} | + +Test focus: OPM↔ILE behavioral differences (override/open scope, `RCLRSC`, `*INLR`), byte-position alignment on program-described files, and cycle→linear correctness (control-break totals, matched-record logic). + +## 8. Follow-up modernization opportunities (NOT part of this conversion) + +- {subroutines → procedures; `*INxx` → named boolean variables; RLA → embedded SQL; error handling with MONITOR; qualified data structures; magic numbers → `DCL-C` constants} + +> Conversion preserves the program's behavior construct by construct. It is **not** a certification of behavioral equivalence — compiling and regression-testing on IBM i remain required before the converted program is used. + +*Analysis powered by iA from [programmers.io](https://programmers.io/ia/)* diff --git a/skills/ia/references/templates/template-onboarding-guide.md b/skills/ia/references/templates/template-onboarding-guide.md new file mode 100644 index 0000000..e541e14 --- /dev/null +++ b/skills/ia/references/templates/template-onboarding-guide.md @@ -0,0 +1,127 @@ +# {MENU} — New-Developer Onboarding Guide + +**Application menu:** {MENU} ({LIBRARY}) +**Repository:** {IA_LIBRARY} · metadata collected {COLLECTION_DATE} +**Generated:** {DATE} +**Author:** iA by programmers.io + +> {FRESHNESS_VERDICT — e.g. "All in-scope members were last changed before the metadata collection date; this guide matches the repository." or "⚠ MEMBER changed {DATE}, after the last collection — verify against current source."} + +**Who this is for:** a developer new to this application. It assumes you can program, and explains the IBM i specifics. + +--- + +## 1. What this menu does + +{3–5 sentences: the menu's business purpose, how many options it offers, what kinds of programs sit behind them, and what data they collectively own. Derived from the menu description, the option text, and the file tables below.} + +**At a glance** + +| | | +|---|---| +| Menu options | {N} | +| Programs reached | {N} ({N} directly from the menu, {N} deeper) | +| Data files | {N} | +| Screens (DSPF) | {N} | +| Reports (PRTF) | {N} | +| Batch jobs | {N} | + +--- + +## 2. Start here + +Read these five first, in this order. + +| # | Object | Why it matters | +|---|--------|----------------| +| 1 | {OBJ} | {one line} | +| 2 | {OBJ} | {one line} | +| 3 | {OBJ} | {one line} | +| 4 | {OBJ} | {one line} | +| 5 | {OBJ} | {one line} | + +--- + +## 3. Menu options + +| Option | Runs | Type | What it does | Files touched | +|--------|------|------|--------------|---------------| +| 1 | {PGM} | {ATTR} | {option text} | {N} | + +{If option numbers could not be recovered from the menu's command member, say so here explicitly and state that the order is alphabetical.} + +--- + +## 4. Call map + +```text +{MENU} ({LIBRARY}) +├── 1. {PGM} {role} +│ └── {CALLEE} {role} +└── 2. {PGM} {role} +``` + +{Mark SBMJOB edges explicitly. Text tree only — never Mermaid.} + +--- + +## 5. Program details + +### {PGM} — {role sentence} + +{1–2 sentences: what it does for the business and how a user reaches it.} + +`{ATTR}` · {N} exec lines · {N} subroutines · {N} procedures · {N} SQL statements · source `{SRCFILE}/{MEMBER}` + +| File | Type | Over | Access | Purpose | +|------|------|------|--------|---------| +| {FILE} | PF | | Update | {description} | + +{Repeat per program, in menu-option order, then deeper programs.} + +--- + +## 6. Shared data + +Files touched by more than one program: + +| File | {PGM} | {PGM} | {PGM} | +|------|-------|-------|-------| +| {FILE} | Update | Input | — | + +**What to be careful with** + +- {File updated by N programs → what that means for a change} +- {The most-shared file → the application's spine} + +--- + +## 7. Terms you'll see + +**{Term}** — {2–3 plain sentences}. *Where:* {object/program}. + +{Only terms this scope actually contains.} + +--- + +## 8. Reached but not detailed + +{Omit this section if the cap wasn't hit.} + +| Object | Type | Reached via | Role | +|--------|------|-------------|------| + +--- + +## 9. Where to go next + +| To learn | Ask iA for | +|----------|------------| +| Everything around one object | an object context matrix for {OBJ} | +| A full spec for one program | program documentation for {PGM} | +| What a change would break | a change impact analysis | +| A visual of the whole library | a 3D app map of {LIBRARY} | + +--- + +*Generated by iA by programmers.io from repository {IA_LIBRARY}. Every object, file and access mode in this document comes from the iA repository — nothing is inferred from source reading.* diff --git a/skills/ia/references/tool-catalog.md b/skills/ia/references/tool-catalog.md index ca428ac..85d4f44 100644 --- a/skills/ia/references/tool-catalog.md +++ b/skills/ia/references/tool-catalog.md @@ -1,4 +1,4 @@ -# iA Tool Catalog (51 Tools) +# iA Tool Catalog (54 Tools) **Rule:** Prefer the dedicated `ia_*` tools. @@ -20,6 +20,7 @@ | Tool | Purpose | |------|---------| +| `ia_object_context_matrix` | **One-call full context for any object** — everything it uses and everything that uses it, bucketed into sections (update/input/output files, display + printer files, calls, callers) and enriched with each referenced object's attribute, description, record formats, key fields and select/omit. Replaces `ia_object_references` + `ia_find_object_usages` + one `ia_object_lookup` per referenced object | | `ia_find_object_usages` | Broad where-used: all objects referencing `object_name` (optional type + library filter) | | `ia_object_references` | Inverse: what an object references/contains (modules in SRVPGM, files used) | | `ia_reference_count` | Lightweight: counts of references grouped by type | @@ -90,5 +91,7 @@ | `ia_klist_usage` | KLIST/KFLD key list definitions; `%` wildcards in kfld_name | | `ia_application_area` | Forward: area → objects; Reverse: object → areas (`%` supported) | | `ia_sql_names` | SQL long/short name mapping for **routines** (procedures/functions) | +| `ia_variant_summary` | Cross-library comparison roll-up — counts of IDENTICAL / COSMETIC / LOGIC / SINGLE members. Fixed-size result; call before `ia_member_variants` | +| `ia_member_variants` | Per-member/per-library variant letters + verdict. Same letter = identical source. Defaults to `verdict=LOGIC` | | `ia_sql_table_names` | SQL long↔short name mapping for **tables + columns** (`CREATE TABLE ... FOR SYSTEM NAME`); resolve a long name to its 10-char system name before where-used. Complements `ia_sql_names` | | `ia_program_files` | Program file usage with PREFIX/RENAME; filter by member/library | diff --git a/skills/ia/references/version-comparison.md b/skills/ia/references/version-comparison.md new file mode 100644 index 0000000..f15d6ed --- /dev/null +++ b/skills/ia/references/version-comparison.md @@ -0,0 +1,285 @@ +# Version Comparison Reference + +Use this guide when a user wants to **compare multiple versions of the same program/member** — the copies that exist in different libraries (e.g. yearly source libraries `LIB2024` / `LIB2025` / `LIB2026`, or `DEV` / `TEST` / `PROD`). The deliverable is a **cross-version comparison report** focused on **what changed** and the behavioral impact of each change, optionally followed by a **detailed Excel diff workbook**. + +**Trigger phrases:** "compare the versions of X", "compare X across libraries", "diff the versions of X", "what changed between the versions of X", "version comparison report for X", "how did X evolve". + +--- + +## ⛔ Intent Routing — Compare vs Document + +There is a deliberate split between two workflows that both begin by finding a member in several libraries: + +| Intent | Workflow | Behavior on multiple versions | +|--------|----------|-------------------------------| +| **Document one version** ("document X", "spec for X") | [program-documentation.md](program-documentation.md) | **HARD STOP** — present the version table and ask which **single** version to document. | +| **Compare versions** ("compare the versions of X") | **this doc** | Discover every version, show the version table, then **ask which versions to include** (any subset of two or more; all is the recommended default) **and which output format(s)** the user wants — see Step 1. | + +If the request is ambiguous ("look at X in all the libraries"), ask one clarifying question: *"Do you want a comparison report across the versions, or a spec for one specific version?"* + +--- + +## ⛔ The Report Is About What CHANGED + +This report is not a program spec and not an inventory. Every section must earn its place by describing a **difference**. Three hard rules: + +1. **No "What Stayed the Same" section.** Do not enumerate the identical declarations, subroutines or logic. The diff hunks bound the change by construction; a change-footprint line (Step 6) states the bound in one sentence. +2. **No complexity-metrics table.** IF/DO/SELECT counts, executable-line counts, operation counts and subroutine counts do not belong in this report. If a metric genuinely changed *because of* a change, state it inline in that change's prose. +3. **The Side-by-Side matrix carries only rows that differ.** Drop any aspect whose value is the same in every library ("Files used", "Subroutines: 17 / 17 / 17"). A row that reads the same across all columns is noise. + +Observations & Risks stays — it is the analytical payload. + +--- + +## Step 1 — Discover Every Version + +One call: + +``` +ia_member_lookup(member_name=X) # source_library defaults to *ALL +``` + +This returns one row per library where the member exists, with `SOURCE_LIBRARY`, `SOURCE_FILE`, `MEMBER_TYPE`, total / comment / blank line counts, and the change date/time. + +- **Zero rows** → the member does not exist (Rule Two). Report the negative; suggest `ia_object_lookup('%X%')`. +- **One row** → only one version exists — there is nothing to compare. Tell the user, and offer program documentation instead. +- **Two or more rows** → **show the user the full version list first** (every library, its total source code lines, and change date), then **stop and ask two questions in one interaction** before retrieving any source: + 1. **Which versions should the comparison include?** Offer **all versions** as the recommended default; the user may pick any subset of **two or more**. If they pick only one, that is the *documentation* workflow — see Intent Routing. Never silently drop a version from the set the user chose. + 2. **Which format(s) do they want the final report in?** Offer **Markdown (`.md`)**, **Word (`.docx`)**, **PDF**, and the **side-by-side Excel diff workbook** (libraries laid out horizontally, one row per change block, colored git-style, changed parts only — see Step 8). More than one is fine. Recommend **Word** for the readable report — its converter colors the diff hunks, while PDF renders them as plain monospace. The `.md` is always written regardless, since it is the canonical copy the other formats are generated from. + + Do not start the comparison until both answers are in. + +--- + +## Step 2 — Order the Versions and Pick the Baseline + +- Sort the versions **oldest → newest** by change date. +- The **oldest** version is the **baseline** — usually the **bottom library** in the list (e.g. `LIB2024` under `LIB2025` under `LIB2026`). Compare everything forward from there. +- If change dates are missing or tie, fall back to library sort order (**bottom = baseline**) and say so. When the timestamps are unreliable (e.g. all members loaded into the repository in the same batch), order by **content containment** instead and state that in the report. +- **State the baseline assumption explicitly** — e.g. *"Baseline = LIB2024 (oldest version, changed 2024-…)."* + +--- + +## Step 3 — Retrieve Each Version's Source + +Route by `MEMBER_TYPE` from Step 1 (all versions share the same type): + +| MEMBER_TYPE | Tool | +|-------------|------| +| `RPGLE`, `SQLRPGLE`, `RPG`, `SQLRPG` | `ia_rpg_source(member_name=X, library_name=L)` | +| `CLLE`, `CLP`, `CL` | `ia_cl_source(member_name=X, library_name=L)` | + +Fetch once per version, passing that version's `library_name`. Apply the standard pagination rule (loop `offset` in 10,000-line pages if the member exceeds 10,000 lines). + +Write each version to a working file, **one source line per line, in `SOURCE_RRN` order**: + +``` +{scratch}/{PROGRAM}_{LIBRARY}.txt +``` + +Plain source text is fine when RRNs run 1..N (the normal case). If RRNs are sparse, write `rrntext` instead — the diff builder accepts both. + +--- + +## Step 4 — Compute the Diff with the Bundled Builder + +**Never hand-author the diff.** Hand-typed line numbers drift, and the report and the workbook then disagree. Run the builder once; it computes the alignment with `difflib` and emits both artifacts from that single computation. + +```bash +python .claude/skills/ia/scripts/build_version_diff.py \ + --program ICS100 \ + --src LIB2024={scratch}/ICS100_LIB2024.txt \ + --src LIB2025={scratch}/ICS100_LIB2025.txt \ + --src LIB2026={scratch}/ICS100_LIB2026.txt \ + --emit-meta-template {scratch}/ICS100_meta.json \ + --emit-md {scratch}/ICS100_hunks.md +``` + +`--src` is repeated **oldest first**. The run prints every hunk it found with an id (`LIB2025->LIB2026#1`) and an auto-derived section name, and writes a `--emit-meta-template` JSON skeleton keyed by those ids. + +**Two-pass flow:** + +1. **Pass 1** — run as above. Read the hunks. Fill in the meta JSON: per-step `title` / `intro`, per-hunk `section` (override the auto-derived guess — it is only the nearest `BegSr`/comment banner), `what`, `impact`, `risks`, plus the top-level `description` / `member_type` / `source_file` / `baseline` / `date` / `versions[].headline` / `risks[]`. +2. **Pass 2** — re-run with `--meta {scratch}/ICS100_meta.json` plus `--emit-md` and (when the user chose Excel in Step 1) `--emit-xlsx`. + +Options: `--context N` sets context lines each side of a change (**default 2** — keep it). + +--- + +## Step 5 — The Diff Hunk Format + +The builder emits ```` ```diff ```` fences that render green/red natively in Markdown viewers **and** in the Word export (the docx converter has a dedicated `diff` renderer). Do not reformat them. + +```diff +@@ LIB2025 → LIB2026 · Main Processing Logic · +5 −0 @@ + 72 │ 72 │ PGMNAME = PgmDS.PgmName; + 73 │ 73 │ ++ · │ 74 │ if %parms = 1; ++ · │ 75 │ KCUSTID = %trim(custno); + 74 │ 79 │ DoU EXIT = *On; +``` + +- **Column 1 is the git marker** — `+` added, `−`/`-` removed, space unchanged. It must stay in column 1 or Markdown highlighters lose the colors. +- **Dual gutter** — old library RRN `│` new library RRN. `·` means the line does not exist in that version. +- **Hunk header** names the enclosing RPG section, the way git shows function context. +- **Two lines of real code before and after** every change, so a reviewer sees where it landed. + +For each hunk write a one-line **behavioral impact** — what the program now does differently, or "none — preparatory". Distinguish a *declared-but-unused* addition from one actually wired into the logic. + +--- + +## Step 6 — Assemble the Report + +Follow **Report Structure** below and apply every rule in **Report-Writing Conventions**. Paste the builder's hunks verbatim into the Change Timeline. + +**Change footprint** — one line in the Executive Summary, replacing any "what stayed the same" prose. Derive the numbers from the builder's summary line: + +> **Change footprint:** 3 hunks · +12 −0 lines · 2.7% of the 448-line member touched · 0 subroutines, 0 files and 0 screens added or removed. + +--- + +## Step 7 — Save and Export + +- **Save target (single canonical copy):** + ``` + docs/program-specs/{PROGRAM_NAME}/{PROGRAM_NAME}_Version_Comparison.md + ``` + Create the folder if needed. One copy per program — overwrite in place on a re-run. (This is a distinct deliverable from the five canonical spec DocTypes; it is **not** subject to the program-documentation Step 1.5 gate.) +- **Export** to whichever of Word/PDF the user chose in Step 1, via the bundled converters (`scripts/convert_md_to_docx.py` / `convert_md_to_pdf.py`), written alongside the `.md`. Do not generate formats the user didn't ask for. (The docx converter colors `diff` fences; the PDF converter renders them as plain monospace — which is why Word is the recommended choice.) + +--- + +## Step 8 — Build the Detailed Excel (when requested) + +The Excel workbook is chosen **up front**, in the Step 1 format question — do not ask about it again after the report is saved. If the user selected it, re-run the builder with `--meta` and `--emit-xlsx`; if they didn't, skip this step entirely: + +```bash +python .claude/skills/ia/scripts/build_version_diff.py \ + --program ICS100 \ + --src LIB2024=… --src LIB2025=… --src LIB2026=… \ + --meta {scratch}/ICS100_meta.json \ + --emit-xlsx docs/program-specs/ICS100/ICS100_Version_Diff.xlsx +``` + +The workbook (requires `pip install openpyxl`) has four sheets: + +| Sheet | Contents | +|-------|----------| +| **Summary** | Program header, per-library line counts and Δ, lines added/removed per step, color legend. | +| **Diff (side-by-side)** | The horizontal view: **one row per change block** — a whole diff block per row, never one row per source line. One column per library holds that version's slice of the block (context plus whatever that version has), RRN-prefixed, one source line per line inside the cell. Lines are colored individually via rich text — see the marker table below. A **blank line** means the version has no counterpart there, which keeps all library cells aligned line-for-line, so line *n* of one cell is line *n* of every other. Frozen header and metadata columns, autofilter on hunk / step / section / type. | +| **Change Log** | One row per hunk: step, section, type, old/new line ranges, ± counts, what changed, behavioral impact, risk refs. | +| **Risks** | The Observations & Risks table, split into Introduced vs Pre-existing. | + +Save it next to the `.md` as `{PROGRAM_NAME}_Version_Diff.xlsx`. + +### Reading a Diff-sheet cell + +| Marker | Colour | Means | +|--------|--------|-------| +| `+` | green | added in this library and it survives | +| `~` | green | **rewritten** in this library — this is the new text | +| `~` | red | superseded in the next library — this is the text that was replaced | +| `-` | red | removed in the next library | +| `+` / `~` | amber | added or rewritten here, then rewritten or removed again later — this exact text existed only in this version | +| *(space)* | grey | unchanged context | +| *(blank line)* | — | this library has no counterpart at that position | + +A **rewritten line and its replacement share one row**, so you read the before and after straight across the library columns — this is the one place the workbook deliberately departs from git, which would show a rewrite as a `-` line followed by an unrelated-looking `+` line. Markdown hunks keep the git form. + +**A block changed by more than one step is a single row, not one row per step.** The `Step`, `Type` and `Δ` columns then carry a `;`-separated entry per step, in order, so `insert; modify` with `+6 −0; +4 −2` reads as *"inserted in the first step, then modified in the second."* Left as separate rows the same block would appear twice with near-identical content. + +> **Hunk ids are positional.** `LIB2025->LIB2026#1` means "the first hunk of that step" — if the source changes and a new hunk appears earlier in the member, every later id shifts and the meta narrative silently attaches to the wrong hunk. Re-run pass 1 and re-check the ids whenever the source files change. + +--- + +## Report-Writing Conventions (non-negotiable) + +1. **"Total source code lines", not "Lines".** The line-count column header reads **Total source code lines**. Never just "Lines". +2. **Identify versions by library name, never by year.** Columns and row labels use the **library name** (`LIB2024`, `PRODLIB`, …). A year may appear as descriptive context; the identifying label is always the library. +3. **Every diff carries source line numbers.** The builder's dual-RRN gutter satisfies this. A diff without line numbers is incomplete. +4. **Say "Main Processing Logic", not "Mainline".** Match the section name used in program documentation. +5. **Prefer "usage" over "reference" for variables/parameters/fields** — *"declared but no usage"*, not *"declared but not referenced"*. +6. **State the baseline** up front, so every "added / removed / changed" is unambiguous. +7. **No "What Stayed the Same" section and no complexity table** — see the rules block above. + +**Branding:** `Author: iA by programmers.io`. First mention "iA by programmers.io", thereafter "iA". Close with *"Prepared with iA by programmers.io."* + +--- + +## Report Structure + +Scale depth to the number of versions; with only two versions the "progressive" timeline collapses to a single baseline-vs-new diff. + +```markdown +# {PROGRAM} — Cross-Version Comparison Report + +**Program:** {PROGRAM} ({alt name / description if in source header}) +**Type:** {MEMBER_TYPE} +**Source file:** {SOURCEPF} +**Libraries compared:** {LIB_A} · {LIB_B} · {LIB_C} (oldest → newest) +**Baseline:** {LIB_A} — {why} +**Author:** iA by programmers.io +**Date:** {YYYY-MM-DD} + +## 1. Executive Summary +- One or two sentences: what the program does. +- The theme of the changes across versions. +- Summary table — one row per version, ordered oldest → newest: + + | Library | Total source code lines | Δ vs prior | What changed | + |---------|------------------------:|:----------:|--------------| + | {LIB_A} | {n} | — (baseline) | Baseline. … | + | {LIB_B} | {n} | +{d} | … | + | {LIB_C} | {n} | +{d} | … | +- **Change footprint:** {h} hunks · +{a} −{r} lines · {pct}% of the {n}-line member + touched · {what categories were not added or removed}. +- Net effect statement. + +## 2. Change Timeline (progressive) +For each adjacent pair, oldest → newest: +### {LIB_A} → {LIB_B}: (+{n} lines) + +**Behavioral impact:** +### {LIB_B} → {LIB_C}: (+{n} lines) +- … + +## 3. Side-by-Side (differences only) +One column per **library**. Include only aspects whose value differs somewhere: + +| Aspect | {LIB_A} | {LIB_B} | {LIB_C} | +|--------|---------|---------|---------| +| … | … | … | … | + +## 4. Observations & Risks +### Introduced by these changes +| # | Version | Finding | +### Pre-existing items (present in all versions — informational, not regressions) +| # | Finding | + +## 5. Conclusion +- One-line evolution summary (baseline → … → latest). +- The follow-ups worth raising. + +*Prepared with iA by programmers.io.* +``` + +--- + +## Common Traps + +| Trap | Fix | +|------|-----| +| Starting the comparison without asking which versions and which formats | Step 1 requires showing the version table, then asking for the version set (default: all) **and** the output format(s) (.md / Word / PDF / Excel) before any source is retrieved. | +| Asking the user to pick a **single** version | That's the *documentation* workflow. A comparison needs at least two versions — see Intent Routing. | +| Writing a "What Stayed the Same" section | Removed by design. Use the one-line change footprint instead. | +| Including a complexity-metrics table | Removed by design. Mention a metric only when a change caused it to move. | +| Side-by-Side rows identical in every column | Delete them. The matrix shows differences only. | +| Hand-typing the diff and its line numbers | Run `build_version_diff.py`. Hand-typed RRNs drift and desync the report from the workbook. | +| Reformatting the builder's `diff` fences | Leave the marker in column 1 and the dual gutter intact, or the colors break in Markdown and Word. | +| Trusting the auto-derived hunk section name | It is the nearest `BegSr`/comment banner — often a stale header. Override it in the meta JSON. | +| Labelling columns/rows by year | Use the **library name**; iA tracks libraries, not calendar years. | +| Calling a declared-but-unused addition a "feature" | Separate *declared* from *wired in*. Note when a parameter/field has no usage yet. | +| Not naming the baseline | State the oldest/bottom library as baseline up front. | +| Treating a constant, cross-version issue as a regression | Put unchanged issues under "Pre-existing items", clearly informational. | +| Asking about the Excel after the report is saved | Formats — including the Excel — are chosen up front in the Step 1 question. Build what was chosen; don't re-ask. | +| Saying "Mainline" / "not referenced" | Use "Main Processing Logic" and "usage". | diff --git a/skills/ia/scripts/build_change_estimate.py b/skills/ia/scripts/build_change_estimate.py new file mode 100644 index 0000000..6b141a6 --- /dev/null +++ b/skills/ia/scripts/build_change_estimate.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Build a costed estimate workbook from a Change Impact Assessment markdown. + +The markdown written by the /ia skill is the source of truth. This script reads +its "## Affected Artifacts" table and prices each row, so the workbook can never +disagree with the assessment the user was shown. + +Every rate lives on an Assumptions sheet and the Detail and Summary sheets +reference it by cell, so the workbook stays live: change a rate in Excel and all +totals recalculate without re-running this script. That matters in front of a +client — the conversation moves to the rates, which is negotiable, instead of to +the total, which looks arbitrary. + +Required input table (see references/change-impact-analysis.md §7): + + ## Affected Artifacts + | Library | Object | Type | Attribute | Impact | Lines | Notes | + |---|---|---|---|---|---|---| + | CASELIB | ORDENT | *PGM | RPGLE | CHANGE | 1840 | rewrites CUSTNO | + + Impact is one of CHANGE / RECOMPILE / REBUILD / REVIEW / NONE. + +Requires: pip install openpyxl +Usage: python build_change_estimate.py ASSESSMENT.md [--xlsx] [--narrowing] + [--outdir DIR] +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import re +import sys +from pathlib import Path + +# Section numbering is optional — "## Affected Artifacts" and "## 3. Affected +# Artifacts" must both match, since the report template numbers its sections. +HEADING = re.compile(r"^##\s+(?:\d+(?:\.\d+)*\.?\s+)?affected\s+artifacts\b", re.I) +WANTED = ["library", "object", "type", "attribute", "impact", "lines", "notes"] + +# Assumptions sheet layout — every formula below addresses these by row, so the +# two must move together. +A_PGM_S, A_PGM_M, A_PGM_L = 2, 3, 4 +A_PGM_RECOMPILE = 5 +A_PF, A_LF, A_DSPF, A_PRTF, A_COPYBOOK, A_OTHER, A_REVIEW = 6, 7, 8, 9, 10, 11, 12 +A_DESIGN, A_CUTOVER, A_UNIT, A_SYSTEST, A_UAT, A_DATACONV, A_CONTINGENCY = ( + 14, 15, 16, 17, 18, 19, 20 +) +A_BAND_S, A_BAND_M, A_HOURS_DAY = 22, 23, 24 + +ASSUMPTIONS = [ + ("Program change — S (small)", 1.5, + "hours; line count at or below the S threshold; <= 500 source lines"), + ("Program change — M (medium)", 3.0, "hours; 501-2,000 lines"), + ("Program change — L (large)", 5.0, "hours; > 2,000 lines"), + ("Program — recompile only", 0.15, "hours; level-check rebuild, no code edit"), + ("Physical file / DDS-DDL change", 0.15, "hours"), + ("Logical file / index rebuild", 0.15, "hours each"), + ("Display file (DSPF) change", 1.0, "hours"), + ("Printer file (PRTF) change", 1.0, "hours"), + ("Copybook change", 0.15, "hours"), + ("Other object change", 0.15, "hours"), + ("Review-only item", 0.5, "hours; inspect and confirm no change needed"), + ("", None, ""), + ("Impact review & change design", 2.0, "hours, fixed"), + ("Cutover / implementation", 2.0, "hours, fixed"), + ("Unit test per changed program", 0.5, "hours"), + ("System / regression test", 0.15, "share of development"), + ("UAT support", 0.10, "share of development"), + ("Data conversion", 2.0, "hours; counted only when the change can lose data"), + ("Contingency", 0.12, "share of subtotal; raise deliberately, not by default"), + ("", None, ""), + ("Band threshold — S up to", 500, "source lines"), + ("Band threshold — M up to", 2000, "source lines"), + ("Hours per working day", 8, ""), +] + +# Basis keys are written into the Detail sheet and counted by Summary formulas, +# so they must stay literal and stable. +RATE_ROW = { + "PGM_CHANGE": None, # banded — resolved by formula against the Band column + "PGM_RECOMPILE": A_PGM_RECOMPILE, + "PF_CHANGE": A_PF, + "LF_REBUILD": A_LF, + "DSPF_CHANGE": A_DSPF, + "PRTF_CHANGE": A_PRTF, + "COPYBOOK_CHANGE": A_COPYBOOK, + "OTHER_CHANGE": A_OTHER, + "REVIEW_ONLY": A_REVIEW, + "NO_COST": None, +} + +PROGRAM_TYPES = {"*PGM", "*SRVPGM", "*MODULE"} +PROGRAM_ATTRS = {"RPGLE", "SQLRPGLE", "RPG", "SQLRPG", "RPG38", "RPGIII", + "CLLE", "CLP", "CL", "CBLLE", "CBL", "PGM", "SRVPGM", "MODULE"} +COPYBOOK_ATTRS = {"RPGLEINC", "CPYBK", "COPYBOOK", "INC", "SQLINC"} +LF_ATTRS = {"LF", "INDEX", "VIEW", "LF38", "DDS_LF", "MQT"} +PF_ATTRS = {"PF", "PF-DATA", "PF-SRC", "PF38", "TABLE", "PHYSICAL"} + + +def split_row(line: str) -> list[str]: + """Split a markdown table row, honouring escaped pipes.""" + line = line.strip().replace(r"\|", "\x00") + if line.startswith("|"): + line = line[1:] + if line.endswith("|"): + line = line[:-1] + return [c.replace("\x00", "|").strip() for c in line.split("|")] + + +def is_divider(line: str) -> bool: + cells = split_row(line) + return bool(cells) and all(re.fullmatch(r":?-{2,}:?", c) for c in cells if c != "") + + +def strip_md(text: str) -> str: + """Flatten inline markdown — cells land in Excel as plain text.""" + text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) + text = text.replace("`", "") + # Object types (*PGM) start with an asterisk the emphasis stripper would eat. + text = re.sub(r"\*([A-Z][A-Z0-9]{1,9})\b", "\x01\\1", text) + text = re.sub(r"\*\*|\*", "", text) + return text.replace("\x01", "*").strip() + + +def parse(md_path: Path) -> tuple[str, list[dict]]: + """Return (title, artifact rows) from the Affected Artifacts table.""" + lines = md_path.read_text(encoding="utf-8").splitlines() + title = md_path.stem.replace("_", " ") + rows: list[dict] = [] + in_section = False + i = 0 + + while i < len(lines): + line = lines[i].strip() + + if line.startswith("# ") and not line.startswith("##"): + title = strip_md(line[2:]) + elif line.startswith("## "): + in_section = bool(HEADING.match(line)) + elif in_section and line.startswith("|") and i + 1 < len(lines) \ + and is_divider(lines[i + 1]): + header = [strip_md(c).lower() for c in split_row(line)] + idx = {name: header.index(name) for name in WANTED if name in header} + missing = [n for n in WANTED[:5] if n not in idx] + if missing: + sys.exit(f"error: Affected Artifacts table is missing column(s): " + f"{', '.join(missing)}") + i += 2 + while i < len(lines) and lines[i].strip().startswith("|"): + if not is_divider(lines[i]): + cells = [strip_md(c) for c in split_row(lines[i])] + rows.append({ + name: (cells[pos] if pos < len(cells) else "") + for name, pos in idx.items() + }) + i += 1 + in_section = False + continue + i += 1 + + return title, rows + + +def classify(row: dict) -> str: + """Map an artifact row onto a rate key.""" + impact = row.get("impact", "").upper() + otype = row.get("type", "").upper() + attr = row.get("attribute", "").upper() + + if impact in ("NONE", ""): + return "NO_COST" + if impact == "REVIEW": + return "REVIEW_ONLY" + # A level-check rebuild costs the same whatever the artifact is — without + # this, a DSPF or PF marked RECOMPILE would be priced at the full change + # rate. REBUILD keeps falling through to the type dispatch so that logical + # files still land on LF_REBUILD. + if impact == "RECOMPILE": + return "PGM_RECOMPILE" + + is_program = otype in PROGRAM_TYPES or attr in PROGRAM_ATTRS + if attr in COPYBOOK_ATTRS: + return "COPYBOOK_CHANGE" + if is_program: + return "PGM_CHANGE" if impact == "CHANGE" else "PGM_RECOMPILE" + if attr in LF_ATTRS: + return "LF_REBUILD" + if attr in PF_ATTRS: + return "PF_CHANGE" + if attr == "DSPF": + return "DSPF_CHANGE" + if attr == "PRTF": + return "PRTF_CHANGE" + if otype == "*FILE": + return "PF_CHANGE" + return "OTHER_CHANGE" + + +def to_int(text: str): + digits = re.sub(r"[^0-9]", "", text or "") + return int(digits) if digits else None + + +def build(out: Path, title: str, rows: list[dict], narrowing: bool) -> dict: + try: + from openpyxl import Workbook + from openpyxl.styles import Alignment, Font, PatternFill + from openpyxl.utils import get_column_letter + except ImportError: + sys.exit("openpyxl is required: pip install openpyxl") + + wb = Workbook() + head_fill = PatternFill("solid", fgColor="1F3864") + head_font = Font(bold=True, color="FFFFFF") + bold = Font(bold=True) + dim = Font(italic=True, color="44546A") + money_fill = PatternFill("solid", fgColor="FFF2CC") + + # ---------------------------------------------------------------- Detail + det = wb.create_sheet("Estimate Detail") + headers = ["Library", "Object", "Type", "Attribute", "Impact", + "Lines", "Band", "Basis", "Hours", "Notes"] + det.append(headers) + for c in range(1, len(headers) + 1): + det.cell(row=1, column=c).font = head_font + det.cell(row=1, column=c).fill = head_fill + + counts: dict[str, int] = {} + for n, row in enumerate(rows, start=2): + basis = classify(row) + counts[basis] = counts.get(basis, 0) + 1 + det.cell(row=n, column=1, value=row.get("library", "")) + det.cell(row=n, column=2, value=row.get("object", "")) + det.cell(row=n, column=3, value=row.get("type", "")) + det.cell(row=n, column=4, value=row.get("attribute", "")) + det.cell(row=n, column=5, value=row.get("impact", "")) + det.cell(row=n, column=6, value=to_int(row.get("lines", ""))) + # Band is a formula so editing a threshold in Assumptions re-bands live. + if basis == "PGM_CHANGE": + det.cell(row=n, column=7, value=( + f'=IF($F{n}="","S",' + f'IF($F{n}<=Assumptions!$B${A_BAND_S},"S",' + f'IF($F{n}<=Assumptions!$B${A_BAND_M},"M","L")))' + )) + det.cell(row=n, column=9, value=( + f'=IF($G{n}="S",Assumptions!$B${A_PGM_S},' + f'IF($G{n}="M",Assumptions!$B${A_PGM_M},Assumptions!$B${A_PGM_L}))' + )) + elif basis == "NO_COST": + det.cell(row=n, column=9, value=0) + else: + det.cell(row=n, column=9, + value=f"=Assumptions!$B${RATE_ROW[basis]}") + det.cell(row=n, column=8, value=basis) + det.cell(row=n, column=9).number_format = "0.00" + det.cell(row=n, column=10, value=row.get("notes", "")) + + last = len(rows) + 1 + det.freeze_panes = "A2" + if rows: + det.auto_filter.ref = f"A1:{get_column_letter(len(headers))}{last}" + for c, name in enumerate(headers, start=1): + widest = max([len(name)] + [ + len(str(det.cell(row=r, column=c).value or "")) for r in range(2, last + 1) + ]) if rows else len(name) + det.column_dimensions[get_column_letter(c)].width = min(max(widest + 2, 9), 55) + + span = f"'Estimate Detail'!$H$2:$H${max(last, 2)}" + hours_span = f"'Estimate Detail'!$I$2:$I${max(last, 2)}" + + # ----------------------------------------------------------- Assumptions + asm = wb.create_sheet("Assumptions") + asm.append(["Assumption", "Value", "Notes"]) + for c in range(1, 4): + asm.cell(row=1, column=c).font = head_font + asm.cell(row=1, column=c).fill = head_fill + for label, value, note in ASSUMPTIONS: + asm.append([label, value, note]) + for r in range(2, len(ASSUMPTIONS) + 2): + cell = asm.cell(row=r, column=2) + if cell.value is None: + continue + cell.fill = money_fill + cell.font = bold + cell.number_format = "0%" if r in (A_SYSTEST, A_UAT, A_CONTINGENCY) else "0.00" + for r in (A_BAND_S, A_BAND_M, A_HOURS_DAY): + asm.cell(row=r, column=2).number_format = "0" + asm.column_dimensions["A"].width = 34 + asm.column_dimensions["B"].width = 12 + asm.column_dimensions["C"].width = 52 + note_at = len(ASSUMPTIONS) + 3 + asm.cell(row=note_at, column=1, + value="Every shaded value feeds the Detail and Summary sheets by " + "cell reference. Change one and the totals recalculate — " + "there is no need to regenerate the workbook.").font = dim + asm.merge_cells(start_row=note_at, start_column=1, end_row=note_at, end_column=3) + asm.cell(row=note_at, column=1).alignment = Alignment(wrap_text=True, vertical="top") + asm.row_dimensions[note_at].height = 30 + + # --------------------------------------------------------------- Summary + s = wb.create_sheet("Summary") + s["A1"] = title + s["A1"].font = Font(bold=True, size=14) + s["A2"] = "Change Impact Estimate" + s["A2"].font = dim + s["A4"], s["B4"] = "Generated", _dt.datetime.now().strftime("%Y-%m-%d %H:%M") + s["A5"], s["B5"] = "Author", "iA by programmers.io" + s["A6"], s["B6"] = "Total impacted objects", len(rows) + # No per-basis counts here — the Basis table at the foot of this sheet + # already breaks the total down, and two places to read the same number is + # two places to disagree. + + r = 8 + s.cell(row=r, column=1, value="Phase").font = head_font + s.cell(row=r, column=1).fill = head_fill + s.cell(row=r, column=2, value="Hours").font = head_font + s.cell(row=r, column=2).fill = head_fill + + design, dev, unit, conv = r + 1, r + 2, r + 3, r + 4 + build_sub, systest, uat, cutover, sub, cont, total, days = ( + r + 5, r + 6, r + 7, r + 8, r + 9, r + 10, r + 11, r + 12 + ) + plan = [ + (design, "Analysis & change design", f"=Assumptions!$B${A_DESIGN}"), + (dev, "Development", f"=SUM({hours_span})"), + (unit, "Unit test", + f'=COUNTIF({span},"PGM_CHANGE")*Assumptions!$B${A_UNIT}'), + (conv, "Data conversion", + f"=Assumptions!$B${A_DATACONV}" if narrowing else "0"), + (build_sub, "Build subtotal", f"=SUM(B{design}:B{conv})"), + (systest, "System / regression test", f"=B{dev}*Assumptions!$B${A_SYSTEST}"), + (uat, "UAT support", f"=B{dev}*Assumptions!$B${A_UAT}"), + (cutover, "Cutover / implementation", f"=Assumptions!$B${A_CUTOVER}"), + (sub, "Subtotal", f"=B{build_sub}+SUM(B{systest}:B{cutover})"), + (cont, "Contingency", f"=B{sub}*Assumptions!$B${A_CONTINGENCY}"), + (total, "TOTAL HOURS", f"=B{sub}+B{cont}"), + (days, "TOTAL DAYS", f"=B{total}/Assumptions!$B${A_HOURS_DAY}"), + ] + for at, label, formula in plan: + s.cell(row=at, column=1, value=label) + s.cell(row=at, column=2, value=formula).number_format = "0.00" + if at in (build_sub, sub, total, days): + s.cell(row=at, column=1).font = bold + s.cell(row=at, column=2).font = bold + s.cell(row=total, column=2).fill = money_fill + s.cell(row=days, column=2).fill = money_fill + + r = days + 2 + s.cell(row=r, column=1, value="Basis").font = head_font + s.cell(row=r, column=1).fill = head_fill + s.cell(row=r, column=2, value="Count").font = head_font + s.cell(row=r, column=2).fill = head_fill + s.cell(row=r, column=3, value="Hours").font = head_font + s.cell(row=r, column=3).fill = head_fill + for basis in sorted(counts): + r += 1 + s.cell(row=r, column=1, value=basis) + s.cell(row=r, column=2, value=f'=COUNTIF({span},"{basis}")') + s.cell(row=r, column=3, + value=f'=SUMIF({span},"{basis}",{hours_span})').number_format = "0.00" + + r += 2 + s.cell(row=r, column=1, value=( + "Rates are on the Assumptions sheet and are deliberately lean. Every " + "figure here is a formula — edit a rate and this page updates. Excludes " + "data validation against live data, external interfaces, and anything " + "changed since the last iA metadata build." + )).font = dim + s.merge_cells(start_row=r, start_column=1, end_row=r, end_column=4) + s.cell(row=r, column=1).alignment = Alignment(wrap_text=True, vertical="top") + s.row_dimensions[r].height = 44 + s.column_dimensions["A"].width = 32 + s.column_dimensions["B"].width = 14 + s.column_dimensions["C"].width = 14 + s.column_dimensions["D"].width = 14 + + if "Sheet" in wb.sheetnames: + del wb["Sheet"] + # Assumptions leads: the rates are what a client argues about, so the + # workbook opens on the negotiable page rather than on the total. + wb._sheets = [asm, s, det] + wb.active = 0 + wb.save(out) + return counts + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("markdown", help="the Change Impact Assessment .md") + ap.add_argument("--xlsx", action="store_true", + help="write the estimate workbook (default action)") + ap.add_argument("--narrowing", action="store_true", + help="the change can lose data — adds the data-conversion line") + ap.add_argument("--outdir", help="output directory (default: beside the .md)") + args = ap.parse_args() + + src = Path(args.markdown) + if not src.is_file(): + print(f"error: not found: {src}", file=sys.stderr) + return 1 + + title, rows = parse(src) + if not rows: + print(f"error: no '## Affected Artifacts' table found in {src.name} — " + "see references/change-impact-analysis.md §7 for the required " + "columns", file=sys.stderr) + return 1 + + outdir = Path(args.outdir) if args.outdir else src.parent + outdir.mkdir(parents=True, exist_ok=True) + out = outdir / f"{src.stem.replace('_Change_Impact', '')}_Change_Estimate.xlsx" + + counts = build(out, title, rows, args.narrowing) + print(f"{src.name}: {len(rows)} artifacts") + for basis in sorted(counts): + print(f" {basis:<18} {counts[basis]}") + print(f" xlsx -> {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ia/scripts/build_context_matrix.py b/skills/ia/scripts/build_context_matrix.py new file mode 100644 index 0000000..3770640 --- /dev/null +++ b/skills/ia/scripts/build_context_matrix.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +"""Render an Object Context Matrix markdown report into Excel and/or HTML. + +The markdown written by the /ia skill is the source of truth. This script only +reformats it, so whatever the agent chose to include — extra sections, a +reordered summary, enrichment tables — lands in every download unchanged. It +never queries the repository, which is exactly what keeps the file the user +downloaded identical to the answer they saw on screen. + +Input shape (produced by references/object-context-matrix.md): + + # Object Context Matrix — CUSTMNTR (*PGM, CASELIB) + + ## Files Used In Update Mode(s) (1) + | File Library | File Name | ... | + |---|---|---| + | CASELIB | CUSTMST | ... | + ## Display File(s) (1) + ... + +Outputs: + --xlsx one sheet per ## section, frozen header, autofilter, sized columns; + prose (Summary, Notes, per-section commentary) is carried across too + --html one self-contained dark-themed page: collapsible sections and a live + filter box, mirroring the portal screen. No external assets. + +Requires: pip install openpyxl (only for --xlsx; --html has no dependencies) +Usage: python build_context_matrix.py REPORT.md [--xlsx] [--html] + [--outdir DIR] +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import html +import re +import sys +from pathlib import Path + +MAX_SHEET_NAME = 31 +# Excel forbids these in a sheet name. +_SHEET_BAD = re.compile(r"[\[\]:*?/\\]") +# Headings carry their row count as "Display File(s) — 1"; older reports used +# "Display File(s) (1)". Either form gets dropped from the sheet tab. +_TRAILING_COUNT = re.compile(r"\s*(?:\(\d+\)|[-–—]\s*\d+)\s*$") +# A markdown bullet or numbered item — anything else continues the line above. +_BULLET = re.compile(r"^(?:[-*+]|\d+[.)])\s") + + +def split_row(line: str) -> list[str]: + """Split a markdown table row, honouring escaped pipes.""" + line = line.strip().replace(r"\|", "\x00") + if line.startswith("|"): + line = line[1:] + if line.endswith("|"): + line = line[:-1] + return [c.replace("\x00", "|").strip() for c in line.split("|")] + + +def is_divider(line: str) -> bool: + """True for the |---|---| row under a table header.""" + cells = split_row(line) + return bool(cells) and all(re.fullmatch(r":?-{2,}:?", c) for c in cells if c != "") + + +# IBM i object types (*PGM, *FILE, *MENU, *ALL, *ENTRY…) start with an asterisk, +# which the emphasis stripper below would otherwise eat — turning *PGM into PGM +# in every Object Type column. Park them before stripping, restore after. +_OBJ_TYPE = re.compile(r"\*([A-Z][A-Z0-9]{1,9})\b") +_CODE = re.compile(r"`([^`]*)`") +# Underscores only mark emphasis at a word boundary. Without this guard the +# stripper turns detected_from into detectedfrom and ia_file_dependencies into +# iafiledependencies — silently corrupting every identifier it touches. +_EMPH = re.compile(r"\*\*|\*|(? str: + """Flatten inline markdown — cells land in Excel as plain text.""" + text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) + # Park inline code so its contents pass through untouched. + spans: list[str] = [] + + def hold(m: re.Match) -> str: + spans.append(m.group(1)) + return f"\x02{len(spans) - 1}\x02" + + text = _CODE.sub(hold, text) + text = _OBJ_TYPE.sub("\x01\\1", text) + text = _EMPH.sub("", text) + text = text.replace("\x01", "*") + text = re.sub(r"\x02(\d+)\x02", lambda m: spans[int(m.group(1))], text) + return text.strip() + + +def parse(md_path: Path) -> tuple[str, list[str], list[dict]]: + """Return (title, summary_paragraphs, sections).""" + lines = md_path.read_text(encoding="utf-8").splitlines() + + title = md_path.stem.replace("_", " ") + summary: list[str] = [] + sections: list[dict] = [] + current: dict | None = None + # Markdown hard-wraps prose, so a paragraph arrives as several lines. Only a + # blank line (or a heading/table) ends one — without this, every wrapped + # bullet lands as its own half-sentence row. + fresh = True + i = 0 + + while i < len(lines): + raw = lines[i] + line = raw.strip() + + if line.startswith("# ") and not line.startswith("##"): + title = strip_md(line[2:]) + fresh = True + i += 1 + continue + + if line.startswith("## "): + current = {"heading": strip_md(line[3:]), "header": [], "rows": [], "notes": []} + sections.append(current) + fresh = True + i += 1 + continue + + # A table starts at a pipe row followed by a divider row. + if line.startswith("|") and i + 1 < len(lines) and is_divider(lines[i + 1]): + header = [strip_md(c) for c in split_row(line)] + rows: list[list[str]] = [] + i += 2 + while i < len(lines) and lines[i].strip().startswith("|"): + if not is_divider(lines[i]): + rows.append([strip_md(c) for c in split_row(lines[i])]) + i += 1 + # Pad or trim ragged rows to the header width. + width = len(header) + rows = [(r + [""] * width)[:width] for r in rows] + if current is None: + current = {"heading": "Details", "header": [], "rows": [], "notes": []} + sections.append(current) + if current["header"]: + # A second table under one heading — keep it as its own section. + current = {"heading": current["heading"] + " (cont.)", + "header": [], "rows": [], "notes": []} + sections.append(current) + current["header"] = header + current["rows"] = rows + fresh = True + continue + + if line: + target = summary if current is None else current["notes"] + text = strip_md(line) + if not fresh and target and not _BULLET.match(line): + target[-1] = f"{target[-1]} {text}" + else: + target.append(text) + fresh = False + else: + fresh = True + i += 1 + + return title, summary, [s for s in sections if s["rows"] or s["notes"]] + + +def clip(text: str, cap: int) -> str: + """Trim to cap on a word boundary rather than slicing a word in half.""" + if len(text) <= cap: + return text + cut = text[:cap] + space = cut.rfind(" ") + # Only honour the boundary if it keeps most of the name; otherwise a single + # long word would collapse the tab to nothing. + if space >= cap * 0.6: + cut = cut[:space] + return cut.rstrip(" -–—([") + + +def sheet_name(heading: str, used: set[str]) -> str: + # Drop the trailing row count — Overview lists it anyway, and keeping it + # pushes longer headings past Excel's 31-char cap. + name = _TRAILING_COUNT.sub("", heading) + name = _SHEET_BAD.sub("-", name).strip() or "Section" + name = clip(name, MAX_SHEET_NAME) + base, n = name, 2 + while name.lower() in used: + suffix = f" {n}" + name = clip(base, MAX_SHEET_NAME - len(suffix)) + suffix + n += 1 + used.add(name.lower()) + return name + + +def write_xlsx(out: Path, title: str, summary: list[str], sections: list[dict]) -> None: + try: + from openpyxl import Workbook + from openpyxl.styles import Alignment, Font, PatternFill + from openpyxl.utils import get_column_letter + except ImportError: + sys.exit("openpyxl is required for --xlsx: pip install openpyxl") + + wb = Workbook() + head_fill = PatternFill("solid", fgColor="1F3864") + head_font = Font(bold=True, color="FFFFFF") + + ws = wb.active + ws.title = "Overview" + ws["A1"] = title + ws["A1"].font = Font(bold=True, size=14) + ws["A3"] = "Generated" + ws["B3"] = _dt.datetime.now().strftime("%Y-%m-%d %H:%M") + ws["A4"] = "Author" + ws["B4"] = "iA by programmers.io" + row = 6 + for para in summary: + ws.cell(row=row, column=1, value=para).alignment = Alignment(wrap_text=True) + ws.merge_cells(start_row=row, start_column=1, end_row=row, end_column=7) + row += 1 + row += 1 + ws.cell(row=row, column=1, value="Section").font = head_font + ws.cell(row=row, column=1).fill = head_fill + ws.cell(row=row, column=2, value="Rows").font = head_font + ws.cell(row=row, column=2).fill = head_fill + for sec in sections: + row += 1 + ws.cell(row=row, column=1, value=sec["heading"]) + # Prose-only sections (Summary, Notes) have no rows to count — leaving + # the cell blank reads better than a misleading 0. + ws.cell(row=row, column=2, value=len(sec["rows"]) if sec["rows"] else "") + ws.column_dimensions["A"].width = 46 + ws.column_dimensions["B"].width = 10 + + note_font = Font(italic=True, color="44546A") + + def put_notes(sheet, start: int, notes: list[str], span: int) -> None: + """Lay prose out down column A, merged across the table's width.""" + for n, text in enumerate(notes): + at = start + n + cell = sheet.cell(row=at, column=1, value=text) + cell.alignment = Alignment(wrap_text=True, vertical="top") + cell.font = note_font + if span > 1: + sheet.merge_cells(start_row=at, start_column=1, + end_row=at, end_column=span) + sheet.row_dimensions[at].height = 15 * max(1, (len(text) // (span * 18) + 1)) + + used: set[str] = {"overview"} + for sec in sections: + if not (sec["rows"] or sec["notes"]): + continue + s = wb.create_sheet(sheet_name(sec["heading"], used)) + + if not sec["rows"]: + # Prose-only section — the heading is the sheet's only chrome. + s.cell(row=1, column=1, value=sec["heading"]).font = Font(bold=True, size=12) + s.column_dimensions["A"].width = 110 + put_notes(s, 3, sec["notes"], 1) + continue + + s.append(sec["header"]) + for c in range(1, len(sec["header"]) + 1): + s.cell(row=1, column=c).font = head_font + s.cell(row=1, column=c).fill = head_fill + for r in sec["rows"]: + s.append(r) + s.freeze_panes = "A2" + s.auto_filter.ref = ( + f"A1:{get_column_letter(len(sec['header']))}{len(sec['rows']) + 1}" + ) + for c, name in enumerate(sec["header"], start=1): + longest = max([len(name)] + [len(str(r[c - 1])) for r in sec["rows"]]) + s.column_dimensions[get_column_letter(c)].width = min(max(longest + 2, 10), 60) + # A blank row keeps the prose clear of the autofilter range above it. + put_notes(s, len(sec["rows"]) + 3, sec["notes"], len(sec["header"])) + + wb.save(out) + + +HTML_CSS = """ +:root{--bg:#0f1419;--panel:#161d26;--edge:#263140;--fg:#e6edf3;--dim:#8b98a5; +--accent:#4a9eff;--head:#1c2734} +*{box-sizing:border-box} +body{margin:0;padding:2rem 1.25rem 4rem;background:var(--bg);color:var(--fg); +font:15px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif} +.wrap{max-width:1200px;margin:0 auto} +h1{font-size:1.6rem;margin:0 0 .35rem;font-weight:650} +.sub{color:var(--dim);font-size:.85rem;margin-bottom:1.5rem} +.summary{background:var(--panel);border:1px solid var(--edge);border-left:3px solid +var(--accent);border-radius:6px;padding:.9rem 1.1rem;margin-bottom:1.5rem} +.summary p{margin:.4rem 0} +#filter{width:100%;padding:.6rem .85rem;margin-bottom:1.25rem;background:var(--panel); +border:1px solid var(--edge);border-radius:6px;color:var(--fg);font-size:.9rem} +#filter:focus{outline:none;border-color:var(--accent)} +details{background:var(--panel);border:1px solid var(--edge);border-radius:6px; +margin-bottom:.75rem;overflow:hidden} +summary{padding:.75rem 1.1rem;cursor:pointer;font-weight:600;background:var(--head); +display:flex;align-items:center;gap:.6rem;list-style:none} +summary::-webkit-details-marker{display:none} +summary::before{content:"\\25B8";color:var(--accent);transition:transform .15s} +details[open] summary::before{transform:rotate(90deg)} +.count{background:var(--accent);color:#0f1419;border-radius:10px;padding:.05rem .5rem; +font-size:.75rem;font-weight:700} +.scroll{overflow-x:auto} +table{border-collapse:collapse;width:100%;font-size:.86rem} +th{background:var(--head);text-align:left;padding:.55rem .8rem;white-space:nowrap; +border-bottom:1px solid var(--edge);position:sticky;top:0} +td{padding:.5rem .8rem;border-bottom:1px solid var(--edge);vertical-align:top} +tr:last-child td{border-bottom:none} +tr:hover td{background:#1b2430} +.note{color:var(--dim);padding:.6rem 1.1rem;font-size:.85rem} +footer{margin-top:2.5rem;color:var(--dim);font-size:.8rem;text-align:center} +.hidden{display:none} +@media print{body{background:#fff;color:#000}details{break-inside:avoid} +details>div{display:block!important}#filter{display:none}} +""" + +HTML_JS = """ +var box=document.getElementById('filter'); +box.addEventListener('input',function(){ + var q=box.value.toLowerCase(); + document.querySelectorAll('details').forEach(function(d){ + var hits=0; + d.querySelectorAll('tbody tr').forEach(function(tr){ + var on=!q||tr.textContent.toLowerCase().indexOf(q)>-1; + tr.classList.toggle('hidden',!on); + if(on)hits++; + }); + var tot=d.querySelectorAll('tbody tr').length; + d.classList.toggle('hidden',q&&tot&&!hits); + if(q&&hits)d.open=true; + }); +}); +""" + + +def write_html(out: Path, title: str, summary: list[str], sections: list[dict]) -> None: + e = html.escape + p: list[str] = [] + p.append("") + p.append("") + p.append(f"{e(title)}
") + p.append(f"

{e(title)}

") + p.append("
Generated " + f"{_dt.datetime.now().strftime('%Y-%m-%d %H:%M')} · " + "iA by programmers.io
") + if summary: + p.append("
") + p.extend(f"

{e(s)}

" for s in summary) + p.append("
") + p.append("") + + for sec in sections: + p.append("
" + e(sec["heading"])) + if sec["rows"]: + p.append(f"{len(sec['rows'])}") + p.append("
") + for note in sec["notes"]: + p.append(f"
{e(note)}
") + if sec["rows"]: + p.append("
") + p.extend(f"" for h in sec["header"]) + p.append("") + for r in sec["rows"]: + p.append("" + "".join(f"" for c in r) + "") + p.append("
{e(h)}
{e(c)}
") + p.append("
") + + p.append("
Object Context Matrix · iA by programmers.io
") + p.append(f"
") + out.write_text("".join(p), encoding="utf-8") + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("markdown", help="the Object Context Matrix .md report") + ap.add_argument("--xlsx", action="store_true", help="write the Excel workbook") + ap.add_argument("--html", action="store_true", help="write the interactive page") + ap.add_argument("--outdir", help="output directory (default: beside the .md)") + args = ap.parse_args() + + src = Path(args.markdown) + if not src.is_file(): + return err(f"not found: {src}") + if not (args.xlsx or args.html): + return err("nothing to do — pass --xlsx and/or --html") + + title, summary, sections = parse(src) + if not sections: + return err(f"no '## ' sections with tables found in {src.name} — " + "is this an Object Context Matrix report?") + + outdir = Path(args.outdir) if args.outdir else src.parent + outdir.mkdir(parents=True, exist_ok=True) + + total = sum(len(s["rows"]) for s in sections) + print(f"{src.name}: {len(sections)} sections, {total} rows") + + if args.xlsx: + out = outdir / f"{src.stem}.xlsx" + write_xlsx(out, title, summary, sections) + print(f" xlsx -> {out}") + if args.html: + out = outdir / f"{src.stem}.html" + write_html(out, title, summary, sections) + print(f" html -> {out}") + return 0 + + +def err(msg: str) -> int: + print(f"error: {msg}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ia/scripts/build_member_diff.py b/skills/ia/scripts/build_member_diff.py new file mode 100644 index 0000000..7617642 --- /dev/null +++ b/skills/ia/scripts/build_member_diff.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python3 +"""Build the cross-library member comparison workbook. + +Pulls every member+library row from the iA MCP server over HTTP (never a direct +Db2 connection — that reuses the existing credential path and keeps one access +route to the repository) and writes a three-sheet Excel workbook: + + 1. Overview — run metadata, verdict distribution, data-quality checks + 2. Members — one row per member, one column per library (the deliverable) + 3. Detail — one row per member+library copy + +The point of doing this in a script rather than through the model is context: +a 10,000-member / 6-library repository is ~60,000 rows. The script moves them; +the model only ever sees the small summary. + +Rows come from ia_member_variants and ia_variant_summary. + +Requires: pip install openpyxl +Usage: python build_member_diff.py [--url URL] [--out FILE] [--limit N] +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import sys +import urllib.error +import urllib.request +from collections import OrderedDict, defaultdict + +try: + from openpyxl import Workbook + from openpyxl.styles import Alignment, Border, Font, PatternFill, Side + from openpyxl.utils import get_column_letter +except ImportError: + sys.exit("openpyxl is required: pip install openpyxl") + +DEFAULT_URL = "http://localhost:3010/mcp" + +# SHA-256 of the empty string. If every fingerprint equals this, the hash +# routine never received any data and the whole report is meaningless. +EMPTY_SHA256_12 = "e3b0c44298fc" + +VERDICT_ORDER = {"LOGIC": 0, "COSMETIC": 1, "IDENTICAL": 2, "SINGLE": 3} + +# ── palette ──────────────────────────────────────────────────────────────── +C_HEAD = "1F3B57" +C_BAND = "F2F5F8" +FILL_LOGIC = PatternFill("solid", fgColor="F8CBCB") +FILL_COSMETIC = PatternFill("solid", fgColor="FDE9C4") +FILL_IDENTICAL = PatternFill("solid", fgColor="D7EFD9") +FILL_SINGLE = PatternFill("solid", fgColor="E8EAED") +FILL_WARN = PatternFill("solid", fgColor="FFF3CD") +FILL_BAD = PatternFill("solid", fgColor="F8CBCB") +FILL_OK = PatternFill("solid", fgColor="D7EFD9") +VERDICT_FILL = { + "LOGIC": FILL_LOGIC, + "COSMETIC": FILL_COSMETIC, + "IDENTICAL": FILL_IDENTICAL, + "SINGLE": FILL_SINGLE, +} +THIN = Side(style="thin", color="D6DBE0") +BORDER = Border(left=THIN, right=THIN, top=THIN, bottom=THIN) + + +# ── MCP transport ────────────────────────────────────────────────────────── +def mcp_texts(url: str, tool: str, args: dict) -> list[str]: + """Call one MCP tool and return its text content blocks. + + Transport and error handling only — the two callers disagree about the + payload format, so parsing is left to them. + """ + body = json.dumps({ + "jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": tool, "arguments": args}, + }).encode() + req = urllib.request.Request(url, data=body, headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }) + try: + with urllib.request.urlopen(req, timeout=600) as r: + raw = r.read().decode("utf-8", "replace") + except urllib.error.URLError as e: + sys.exit(f"cannot reach the MCP server at {url}: {e}\n" + "Start the iA MCP server first, then retry.") + + payload = None + for line in raw.splitlines(): # SSE framing + if line.startswith("data:"): + payload = json.loads(line[5:].strip()) + break + if payload is None: + payload = json.loads(raw) + + if "error" in payload: + sys.exit(f"{tool} failed: {payload['error']}") + + result = payload.get("result", {}) + texts = [c.get("text", "") for c in result.get("content", []) + if c.get("type") == "text"] + + # The server reports tool-level problems (validation errors, SQL errors) as + # an isError result carrying a plain-text message. Never treat that as + # "no rows" — an empty report that should have been an error is far worse + # than a crash. + if result.get("isError"): + sys.exit(f"{tool} returned an error:\n " + "\n ".join(texts)[:1500]) + if not texts: + sys.exit(f"{tool} returned no usable content") + return texts + + +def mcp_call(url: str, tool: str, args: dict) -> list[dict]: + """Call an ia_* tool, which answers with a strict JSON envelope.""" + for text in mcp_texts(url, tool, args): + try: + doc = json.loads(text) + except json.JSONDecodeError: + sys.exit(f"{tool} returned a non-JSON response (treating as failure):\n" + f" {text[:1500]}") + if isinstance(doc, dict): + if not doc.get("success", True): + sys.exit(f"{tool} returned failure: {json.dumps(doc)[:800]}") + return doc.get("data", []) or [] + sys.exit(f"{tool} returned no usable content") + + +# ── variant labelling ────────────────────────────────────────────────────── +BASE_LABEL = "Base" + + +def variant_label(rank: int) -> str: + """Base for the prevailing version, then v1, v2, v3 for each divergence. + + Reads as a version history rather than as an arbitrary letter, and needs no + special case past 26 the way A..Z did. + """ + return BASE_LABEL if rank <= 1 else f"v{rank - 1}" + + +def _rank_from_letter(value) -> int | None: + """Recover a rank from the A, B, C ... V27 labels the views produce.""" + if value is None: + return None + text = str(value).strip() + if len(text) > 1 and text[0] in "Vv" and text[1:].isdigit(): + return int(text[1:]) + if len(text) == 1 and text.isalpha(): + return ord(text.upper()) - 64 + return None + + +def normalize_variants(detail: list[dict]) -> list[dict]: + """Put both data paths on the same labels. + + The direct read supplies a numeric rank; the views supply letters. Reducing + them here means the workbook is identical whichever route produced it. + """ + for r in detail: + for rank_key, label_key in (("VARIANT_RANK", "VARIANT"), + ("NORM_VARIANT_RANK", "NORM_VARIANT")): + rank = r.get(rank_key) + if rank is None: + rank = _rank_from_letter(r.get(label_key)) + if rank is not None: + r[label_key] = variant_label(rank) + return detail + + +def summarize(detail: list[dict]) -> list[dict]: + """Roll the detail rows up per verdict. + + Derived from the same rows the sheets are built from rather than from a + second query, so the Overview can never disagree with the Members sheet. + """ + members: dict[str, dict] = {} + for r in detail: + members.setdefault(r["MEMBER_NAME"], r) + + groups: dict[str, list[str]] = defaultdict(list) + for name, meta in members.items(): + groups[meta.get("VERDICT")].append(name) + + out = [] + for verdict, names in groups.items(): + rows = [r for r in detail if r["MEMBER_NAME"] in set(names)] + changed = [str(r.get("MEMBER_CHANGED")) for r in rows if r.get("MEMBER_CHANGED")] + stamped = [str(r.get("FINGERPRINTED")) for r in rows if r.get("FINGERPRINTED")] + out.append({ + "VERDICT": verdict, + "MEMBER_COUNT": len(names), + "COPY_COUNT": len(rows), + "TYPE_DRIFT_COUNT": len({r["MEMBER_NAME"] for r in rows + if r.get("TYPE_DRIFT") == "Y"}), + "PF_DRIFT_COUNT": len({r["MEMBER_NAME"] for r in rows + if r.get("PF_DRIFT") == "Y"}), + "LARGEST_MEMBER": max(rows, key=lambda r: r.get("TOTAL_LINES") or 0 + )["MEMBER_NAME"] if rows else None, + "NEWEST_CHANGE": max(changed) if changed else None, + "OLDEST_FINGERPRINT": min(stamped) if stamped else None, + }) + return out + + +# ── sheet writers ────────────────────────────────────────────────────────── +def _header(ws, row: int, labels: list[str]) -> None: + for col, text in enumerate(labels, start=1): + c = ws.cell(row=row, column=col, value=text) + c.font = Font(bold=True, color="FFFFFF", size=10) + c.fill = PatternFill("solid", fgColor=C_HEAD) + c.alignment = Alignment(horizontal="left", vertical="center", wrap_text=True) + c.border = BORDER + ws.row_dimensions[row].height = 26 + + +def _autosize(ws, widths: dict[int, int]) -> None: + for idx, width in widths.items(): + ws.column_dimensions[get_column_letter(idx)].width = width + + +def sheet_overview(wb, summary, detail, libraries, warnings): + ws = wb.create_sheet("Overview") + ws.sheet_view.showGridLines = False + _autosize(ws, {1: 34, 2: 22, 3: 16, 4: 24, 5: 25, 6: 18, 7: 24, 8: 26}) + + t = ws.cell(row=1, column=1, value="Cross-Library Member Comparison") + t.font = Font(bold=True, size=16, color=C_HEAD) + ws.cell(row=2, column=1, + value="Every source member in the repository, and whether its copies " + "in different libraries are the same.").font = Font(size=10, italic=True, + color="5B6B78") + + total_members = len({r["MEMBER_NAME"] for r in detail}) + row = 4 + for label, value in [ + ("Generated", _dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), + ("Libraries compared", ", ".join(libraries) if libraries else "(none)"), + ("Library count", len(libraries)), + ("Members reported", total_members), + ("Member+library copies", len(detail)), + ]: + ws.cell(row=row, column=1, value=label).font = Font(bold=True, size=10) + ws.cell(row=row, column=2, value=value).font = Font(size=10) + row += 1 + + # ── data-quality block: loud, and first, because a wrong report that looks + # right is worse than no report at all. + row += 1 + ws.cell(row=row, column=1, value="Data quality").font = Font(bold=True, size=12, + color=C_HEAD) + row += 1 + if warnings: + for level, text in warnings: + c = ws.cell(row=row, column=1, value=f"{level}: {text}") + c.fill = FILL_BAD if level == "BLOCKER" else FILL_WARN + c.font = Font(bold=(level == "BLOCKER"), size=10) + c.alignment = Alignment(wrap_text=True, vertical="top") + ws.merge_cells(start_row=row, start_column=1, end_row=row, end_column=8) + ws.row_dimensions[row].height = 42 + row += 1 + else: + c = ws.cell(row=row, column=1, value="OK: no data-quality problems detected.") + c.fill = FILL_OK + c.font = Font(size=10) + row += 1 + + # ── verdict distribution + row += 1 + ws.cell(row=row, column=1, value="Verdict distribution").font = Font(bold=True, size=12, + color=C_HEAD) + row += 1 + _header(ws, row, ["Verdict", "Members", "Copies", + "Member Type Differs\ne.g. RPGLE → SQLRPGLE", + "Source File Differs\ne.g. QRPGSRC → QRPGLESRC", + "Largest Member", "Newest Change", "Oldest Snapshot"]) + ws.row_dimensions[row].height = 34 # two of these headers wrap to a second line + row += 1 + meaning = { + "LOGIC": "the code itself differs — this is the shortlist", + "COSMETIC": "differs only in comments, blanks, indentation or case", + "IDENTICAL": "byte-for-byte the same everywhere", + "SINGLE": "only one copy exists — nothing to compare", + } + for r in sorted(summary, key=lambda x: VERDICT_ORDER.get(x.get("VERDICT"), 9)): + verdict = r.get("VERDICT") + vals = [verdict, r.get("MEMBER_COUNT"), r.get("COPY_COUNT"), + r.get("TYPE_DRIFT_COUNT"), r.get("PF_DRIFT_COUNT"), + r.get("LARGEST_MEMBER"), str(r.get("NEWEST_CHANGE") or ""), + str(r.get("OLDEST_FINGERPRINT") or "")] + for col, v in enumerate(vals, start=1): + c = ws.cell(row=row, column=col, value=v) + c.border = BORDER + c.font = Font(size=10, bold=(col == 1)) + if col == 1: + c.fill = VERDICT_FILL.get(verdict, FILL_SINGLE) + row += 1 + note = ws.cell(row=row, column=1, value=f" {meaning.get(verdict, '')}") + note.font = Font(size=9, italic=True, color="5B6B78") + ws.merge_cells(start_row=row, start_column=1, end_row=row, end_column=8) + row += 1 + + row += 1 + ws.cell(row=row, column=1, + value="How to use this: open the Members sheet, filter Verdict = LOGIC. " + "Those are the members whose code actually differs between " + "libraries. Within each verdict the largest members come " + "first.").font = Font(size=10, italic=True, color="5B6B78") + + +# Three things a first-time reader gets wrong: what the variant labels mean, +# what the two hashes are for, and that Line Count Delta is about length rather +# than sameness. Kept to three lines so the grid stays the focus. +MEMBER_NOTES = [ + "VARIANT LABELS — distinct versions are ranked by how many libraries share each; " + "ties by earliest change date. Rank 1 = Base, then v1, v2, v3. Same label = " + "identical source.", + "VERDICT — LOGIC: the code differs. COSMETIC: comments, blank lines, indentation or " + "case only. IDENTICAL: byte-for-byte match. SINGLE: one copy only.", + "MAX SOURCE LINES — the longest single copy, all lines included. LINE COUNT DELTA — " + "longest minus shortest. Length only: a delta of 0 does not imply a match.", +] + + +def member_max_lines(detail: list[dict]) -> dict[str, int]: + """Longest copy per member. + + Line counts are per copy, so the member-level figure has to be reduced + across the copies. Reading it off the first row gives whichever library + sorted first, which is not the longest copy. + """ + out: dict[str, int] = {} + for r in detail: + value = r.get("TOTAL_LINES") + if value is not None and value > out.get(r["MEMBER_NAME"], -1): + out[r["MEMBER_NAME"]] = value + return out + + +def sheet_members(wb, detail, libraries): + """One row per member, one column per library. The deliverable.""" + ws = wb.create_sheet("Members") + ws.sheet_view.showGridLines = False + + members: "OrderedDict[str, dict]" = OrderedDict() + cells: dict[tuple[str, str], list[str]] = defaultdict(list) + maxlines = member_max_lines(detail) + for r in detail: + name = r["MEMBER_NAME"] + if name not in members: + members[name] = r + cells[(name, r["LIBRARY"])].append(r.get("VARIANT") or "?") + + fixed = ["Member", "Member Type", "Source File", "Library Count", + "Exact Variants", "Verdict", "Member Type Differs", + "Source File Differs", "Max Source Lines", "Line Count Delta"] + ncols = len(fixed) + len(libraries) + + for i, note in enumerate(MEMBER_NOTES, start=1): + c = ws.cell(row=i, column=1, value=note) + c.font = Font(size=9, italic=True, color="43525F") + c.alignment = Alignment(wrap_text=True, vertical="top") + c.fill = PatternFill("solid", fgColor=C_BAND) + ws.merge_cells(start_row=i, start_column=1, end_row=i, end_column=ncols) + ws.row_dimensions[i].height = 18 + + hdr = len(MEMBER_NOTES) + 1 + _header(ws, hdr, fixed + list(libraries)) + + row = hdr + 1 + # Within a verdict, largest member first — the biggest programs are where a + # code difference matters most, so they lead each group. + for name, meta in sorted( + members.items(), + key=lambda kv: (VERDICT_ORDER.get(kv[1].get("VERDICT"), 9), + -(maxlines.get(kv[0]) or 0), kv[0])): + verdict = meta.get("VERDICT") + vals = [name, meta.get("MEMBER_TYPE"), meta.get("SOURCE_FILE"), + meta.get("LIB_COUNT"), meta.get("VARIANT_COUNT"), verdict, + meta.get("TYPE_DRIFT"), meta.get("PF_DRIFT"), + maxlines.get(name), meta.get("LINE_SPREAD")] + for col, v in enumerate(vals, start=1): + c = ws.cell(row=row, column=col, value=v) + c.border = BORDER + c.font = Font(size=10, bold=(col == 1)) + if col == 6: + c.fill = VERDICT_FILL.get(verdict, FILL_SINGLE) + c.font = Font(size=10, bold=True) + for i, lib in enumerate(libraries): + got = cells.get((name, lib)) + c = ws.cell(row=row, column=len(fixed) + 1 + i, + value=", ".join(sorted(set(got))) if got else "") + c.border = BORDER + c.alignment = Alignment(horizontal="center") + c.font = Font(size=10, bold=True) + if got: + # Base is the prevailing version; anything else is a fork. + c.fill = FILL_IDENTICAL if set(got) == {BASE_LABEL} else FILL_LOGIC + row += 1 + + # Freeze the notes, the header and the member name column. + ws.freeze_panes = ws.cell(row=hdr + 1, column=2) + ws.auto_filter.ref = f"A{hdr}:{get_column_letter(ncols)}{row - 1}" + _autosize(ws, {1: 16, 2: 13, 3: 13, 4: 13, 5: 14, 6: 12, 7: 19, 8: 19, + 9: 17, 10: 17}) + for i in range(len(libraries)): + ws.column_dimensions[get_column_letter(len(fixed) + 1 + i)].width = 13 + + +def sheet_detail(wb, detail): + ws = wb.create_sheet("Detail") + ws.sheet_view.showGridLines = False + cols = [("MEMBER_NAME", "Member", 16), ("LIBRARY", "Library", 12), + ("SOURCE_FILE", "Source File", 13), + ("MEMBER_TYPE", "Member Type", 13), + ("VARIANT", "Exact Variant", 14), + ("NORM_VARIANT", "Normalized Variant", 19), + ("VERDICT", "Verdict", 12), ("LIB_COUNT", "Library Count", 13), + ("VARIANT_COUNT", "Exact Variants", 14), + ("NORM_VARIANT_COUNT", "Normalized Variants", 20), + ("TOTAL_LINES", "Source Lines", 13), + ("CODE_LINES", "Code Lines", 12), + ("EXACT_HASH_12", "Exact Hash (12)", 16), + ("NORM_HASH_12", "Normalized Hash (12)", 21), + ("MEMBER_CHANGED", "Member Changed", 21)] + _header(ws, 1, [c[1] for c in cols]) + row = 2 + maxlines = member_max_lines(detail) # same member order as the Members sheet + for r in sorted(detail, key=lambda x: (VERDICT_ORDER.get(x.get("VERDICT"), 9), + -(maxlines.get(x["MEMBER_NAME"]) or 0), + x["MEMBER_NAME"], x["LIBRARY"], + x.get("SOURCE_FILE") or "")): + for col, (key, _, _w) in enumerate(cols, start=1): + v = r.get(key) + c = ws.cell(row=row, column=col, + value=str(v) if key == "MEMBER_CHANGED" and v else v) + c.border = BORDER + c.font = Font(size=10) + if key == "VERDICT": + c.fill = VERDICT_FILL.get(r.get("VERDICT"), FILL_SINGLE) + if key in ("EXACT_HASH_12", "NORM_HASH_12"): + c.font = Font(size=9, name="Consolas") + if row % 2 == 0: + for col in range(1, len(cols) + 1): + if not ws.cell(row=row, column=col).fill.fgColor.rgb.endswith( + (FILL_LOGIC.fgColor.rgb[-6:], FILL_COSMETIC.fgColor.rgb[-6:], + FILL_IDENTICAL.fgColor.rgb[-6:], FILL_SINGLE.fgColor.rgb[-6:])): + ws.cell(row=row, column=col).fill = PatternFill("solid", + fgColor=C_BAND) + row += 1 + ws.freeze_panes = "A2" + ws.auto_filter.ref = f"A1:{get_column_letter(len(cols))}{row - 1}" + _autosize(ws, {i: w for i, (_k, _l, w) in enumerate(cols, start=1)}) + + +# ── data-quality checks ──────────────────────────────────────────────────── +def check_data(detail: list[dict]) -> list[tuple[str, str]]: + """Findings that make the report untrustworthy. Reported, never hidden.""" + out: list[tuple[str, str]] = [] + if not detail: + out.append(("BLOCKER", "No comparison rows at all. The source table is " + "empty — the population program has not been run.")) + return out + + exact = {r.get("EXACT_HASH_12") for r in detail} + norm = {r.get("NORM_HASH_12") for r in detail} + + if exact == {EMPTY_SHA256_12}: + out.append(("BLOCKER", + "Every EXACT_HASH is the SHA-256 of the empty string " + f"({EMPTY_SHA256_12}...). The hash routine is being opened and " + "closed but never fed any data, so every member looks identical " + "to every other. Every verdict on the Members sheet is therefore " + "meaningless. Fix ia_hashAdd (SDS acceptance tests 1-4) and " + "rebuild before using this workbook.")) + elif len(exact) == 1 and len(detail) > 1: + out.append(("BLOCKER", + "Every member has the same EXACT_HASH. That cannot be correct for " + "distinct source, so the hash routine is not working.")) + + if norm == {EMPTY_SHA256_12} and exact != {EMPTY_SHA256_12}: + out.append(("BLOCKER", "Every NORM_HASH is the empty-string digest — the " + "normalized stream is never fed.")) + + # A member whose copies differ in length but are reported IDENTICAL is a + # direct contradiction, and the cheapest proof the hashes are wrong. + contradictions = sorted({ + r["MEMBER_NAME"] for r in detail + if r.get("VERDICT") == "IDENTICAL" and (r.get("LINE_SPREAD") or 0) > 0 + }) + if contradictions: + out.append(("BLOCKER", + f"{len(contradictions)} member(s) are reported IDENTICAL but their " + f"copies have different line counts: {', '.join(contradictions[:8])}" + f"{'...' if len(contradictions) > 8 else ''}. Identical source " + "cannot have different line counts.")) + + libs = {r.get("LIBRARY") for r in detail} + if len(libs) < 2: + out.append(("WARNING", + f"Only one library ({', '.join(sorted(libs))}) has been analysed, " + "so there is nothing to compare across libraries — every member can " + "only be SINGLE. Analyse at least two libraries for this report " + "to answer its question.")) + + stale = {r.get("MEMBER_NAME") for r in detail if r.get("VARIANT") in (None, "", "?")} + if stale: + out.append(("WARNING", f"{len(stale)} row(s) have no variant letter.")) + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--url", default=DEFAULT_URL, help=f"MCP endpoint (default {DEFAULT_URL})") + ap.add_argument("--out", default=None, help="output .xlsx (default member-diff-YYYYMMDD.xlsx)") + ap.add_argument("--limit", type=int, default=100000, help="max rows to pull (default 100000)") + ap.add_argument("--member-type", default="*ALL", help="filter to one member type") + args = ap.parse_args() + + out = args.out or f"member-diff-{_dt.date.today():%Y%m%d}.xlsx" + + print(f"Pulling rows from {args.url} ...") + detail = mcp_call(args.url, "ia_member_variants", + {"verdict": "*ALL", "member_type": args.member_type, + "limit": args.limit}) + summary = mcp_call(args.url, "ia_variant_summary", + {"member_type": args.member_type, "limit": 100}) + normalize_variants(detail) + print(f" {len(detail)} member+library rows, {len(summary)} verdict groups") + + libraries = sorted({r["LIBRARY"] for r in detail if r.get("LIBRARY")}) + warnings = check_data(detail) + for level, text in warnings: + print(f" {level}: {text}") + + wb = Workbook() + wb.remove(wb.active) + sheet_overview(wb, summary, detail, libraries, warnings) + sheet_members(wb, detail, libraries) + sheet_detail(wb, detail) + wb.save(out) + + print(f"\n[OK] {out}") + print(f" Overview · Members ({len({r['MEMBER_NAME'] for r in detail})} rows) " + f"· Detail ({len(detail)} rows)") + if any(l == "BLOCKER" for l, _ in warnings): + print(" NOTE: blocking data-quality problems found — see the Overview sheet.") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ia/scripts/build_onboarding_guide.py b/skills/ia/scripts/build_onboarding_guide.py new file mode 100644 index 0000000..0f063c4 --- /dev/null +++ b/skills/ia/scripts/build_onboarding_guide.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Render a New-Developer Onboarding Guide markdown report into Excel. + +The markdown written by the /ia skill is the source of truth; this script only +reformats it, so the workbook always agrees with what the user read on screen. +It never queries the repository. + +Unlike build_context_matrix.py — which mirrors whatever sections it finds — this +one knows the guide's shape and flattens it into four sheets a reader can filter: + + Menu Options the option -> program table + Program -> File every per-program file table, with a Program column + Shared Data the shared-file matrix + Program Stats the stats line under each program heading + +Sections are found by keyword, not by number, so renumbering the guide's +headings does not break the build. + +Requires: pip install openpyxl +Usage: python build_onboarding_guide.py GUIDE.md [--outdir DIR] +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from build_context_matrix import is_divider, split_row, strip_md # noqa: E402 + +# "### CUSTMNTR — Customer maintenance" → name, role. An en/em dash or a hyphen +# all appear in practice depending on who edited the guide. +_PGM_HEADING = re.compile(r"^([A-Z0-9#@$]{1,10})\s*[-–—]\s*(.*)$") +# "`RPGLE` · 288 exec lines · 17 subroutines · … · source `QRPGLESRC/CUSTMNTR`" +_STAT = re.compile(r"(\d+)\s+(exec lines|subroutines|procedures|SQL statements)") +_ATTR = re.compile(r"^`([^`]+)`") +_SOURCE = re.compile(r"source\s+`([^`]+)`") +STAT_COLUMNS = ["exec lines", "subroutines", "procedures", "SQL statements"] + + +def read_tables(md_path: Path) -> tuple[str, list[dict]]: + """Return (title, blocks) where each block is one heading and its tables. + + A block is {level, heading, tables, lines}: `lines` keeps the prose under the + heading so the stats line can be recovered without a second pass. + """ + lines = md_path.read_text(encoding="utf-8").splitlines() + title = md_path.stem.replace("_", " ") + blocks: list[dict] = [] + current: dict | None = None + i = 0 + + while i < len(lines): + line = lines[i].strip() + + if line.startswith("# ") and not line.startswith("##"): + title = strip_md(line[2:]) + i += 1 + continue + + # "### X" does not start with "## " — the third character is a hash, not + # a space — so the deeper heading must be tested first and separately. + if line.startswith("### ") or line.startswith("## "): + level = 3 if line.startswith("### ") else 2 + text = line[4:] if level == 3 else line[3:] + current = {"level": level, "heading": strip_md(text), + "tables": [], "lines": []} + blocks.append(current) + i += 1 + continue + + if line.startswith("|") and i + 1 < len(lines) and is_divider(lines[i + 1]): + header = [strip_md(c) for c in split_row(line)] + rows: list[list[str]] = [] + i += 2 + while i < len(lines) and lines[i].strip().startswith("|"): + if not is_divider(lines[i]): + rows.append([strip_md(c) for c in split_row(lines[i])]) + i += 1 + width = len(header) + rows = [(r + [""] * width)[:width] for r in rows] + if current is not None and rows: + current["tables"].append({"header": header, "rows": rows}) + continue + + if line and current is not None: + current["lines"].append(line) + i += 1 + + return title, blocks + + +def find(blocks: list[dict], *keywords: str) -> dict | None: + """First level-2 block whose heading mentions every keyword.""" + for b in blocks: + low = b["heading"].lower() + if b["level"] == 2 and all(k in low for k in keywords): + return b + return None + + +def program_blocks(blocks: list[dict]) -> list[dict]: + """Level-3 blocks under the Program details heading.""" + out: list[dict] = [] + inside = False + for b in blocks: + if b["level"] == 2: + inside = "program detail" in b["heading"].lower() + continue + if inside: + out.append(b) + return out + + +def relations(pgms: list[dict]) -> tuple[list[str], list[list[str]]]: + """Flatten every per-program file table into one sheet.""" + header = ["Program", "File", "Type", "Over", "Access", "Purpose"] + rows: list[list[str]] = [] + for b in pgms: + m = _PGM_HEADING.match(b["heading"]) + name = m.group(1) if m else b["heading"] + for table in b["tables"]: + # Index the source columns by name so a guide that adds or reorders + # a column still lands in the right place. + idx = {h.lower(): n for n, h in enumerate(table["header"])} + for r in table["rows"]: + def cell(key: str) -> str: + at = idx.get(key) + return r[at] if at is not None and at < len(r) else "" + rows.append([name, cell("file"), cell("type"), cell("over"), + cell("access"), cell("purpose")]) + return header, rows + + +def stats(pgms: list[dict]) -> tuple[list[str], list[list[str]]]: + header = ["Program", "Role", "Attribute", *STAT_COLUMNS, "Source"] + rows: list[list[str]] = [] + for b in pgms: + m = _PGM_HEADING.match(b["heading"]) + name, role = (m.group(1), m.group(2)) if m else (b["heading"], "") + # The stats line opens with the attribute in backticks and names the + # source member. Keying on "exec lines" instead would drop every CL + # program — CL members carry no complexity row, so they have no such + # count, but they still have an attribute and a source member. + line = next((l for l in b["lines"] + if _ATTR.match(l) and _SOURCE.search(l)), "") + found = {label: value for value, label in _STAT.findall(line)} + attr = _ATTR.match(line) + source = _SOURCE.search(line) + rows.append([name, role, + attr.group(1) if attr else "", + *(found.get(c, "") for c in STAT_COLUMNS), + source.group(1) if source else ""]) + return header, rows + + +def write_xlsx(out: Path, title: str, sheets: list[tuple[str, list[str], list[list[str]]]]) -> None: + try: + from openpyxl import Workbook + from openpyxl.styles import Alignment, Font, PatternFill + from openpyxl.utils import get_column_letter + except ImportError: + sys.exit("openpyxl is required: pip install openpyxl") + + wb = Workbook() + head_fill = PatternFill("solid", fgColor="1F3864") + head_font = Font(bold=True, color="FFFFFF") + + ws = wb.active + ws.title = "Overview" + ws["A1"] = title + ws["A1"].font = Font(bold=True, size=14) + ws["A3"] = "Generated" + ws["B3"] = _dt.datetime.now().strftime("%Y-%m-%d %H:%M") + ws["A4"] = "Author" + ws["B4"] = "iA by programmers.io" + ws["A6"] = "Sheet" + ws["B6"] = "Rows" + for c in ("A6", "B6"): + ws[c].font = head_font + ws[c].fill = head_fill + for n, (name, _, rows) in enumerate(sheets, start=7): + ws.cell(row=n, column=1, value=name) + ws.cell(row=n, column=2, value=len(rows)) + ws.column_dimensions["A"].width = 30 + ws.column_dimensions["B"].width = 10 + + for name, header, rows in sheets: + s = wb.create_sheet(name) + s.append(header) + for c in range(1, len(header) + 1): + s.cell(row=1, column=c).font = head_font + s.cell(row=1, column=c).fill = head_fill + for r in rows: + s.append(r) + s.freeze_panes = "A2" + if rows: + s.auto_filter.ref = f"A1:{get_column_letter(len(header))}{len(rows) + 1}" + for c, col in enumerate(header, start=1): + longest = max([len(col)] + [len(str(r[c - 1])) for r in rows] or [len(col)]) + s.column_dimensions[get_column_letter(c)].width = min(max(longest + 2, 10), 60) + if col.lower() in ("purpose", "role", "why it matters", "what it does"): + for r in range(2, len(rows) + 2): + s.cell(row=r, column=c).alignment = Alignment(wrap_text=True) + + wb.save(out) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("markdown", help="the Onboarding Guide .md report") + ap.add_argument("--outdir", help="output directory (default: beside the .md)") + args = ap.parse_args() + + src = Path(args.markdown) + if not src.is_file(): + return err(f"not found: {src}") + + title, blocks = read_tables(src) + pgms = program_blocks(blocks) + if not pgms: + return err(f"no '### ' program sections under a 'Program details' heading " + f"in {src.name} — is this an Onboarding Guide?") + + sheets: list[tuple[str, list[str], list[list[str]]]] = [] + + options = find(blocks, "menu option") + if options and options["tables"]: + t = options["tables"][0] + sheets.append(("Menu Options", t["header"], t["rows"])) + + header, rows = relations(pgms) + sheets.append(("Program to File", header, rows)) + + shared = find(blocks, "shared data") + if shared and shared["tables"]: + t = shared["tables"][0] + sheets.append(("Shared Data", t["header"], t["rows"])) + + sheets.append(("Program Stats", *stats(pgms))) + + outdir = Path(args.outdir) if args.outdir else src.parent + outdir.mkdir(parents=True, exist_ok=True) + out = outdir / f"{src.stem}.xlsx" + write_xlsx(out, title, sheets) + + print(f"{src.name}: {len(pgms)} programs, {len(rows)} file relations") + for name, _, r in sheets: + print(f" {name}: {len(r)} rows") + print(f" xlsx -> {out}") + return 0 + + +def err(msg: str) -> int: + print(f"error: {msg}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ia/scripts/build_version_diff.py b/skills/ia/scripts/build_version_diff.py new file mode 100644 index 0000000..ebb151d --- /dev/null +++ b/skills/ia/scripts/build_version_diff.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python3 +"""Cross-version diff builder for the /ia version-comparison workflow. + +Aligns the same source member across N libraries (oldest -> newest) and emits: + + * ``--emit-md`` dual-RRN unified diff hunks, ready to paste into the + comparison report (```diff fences, git markers in column 1). + * ``--emit-xlsx`` a workbook whose Diff sheet lays the libraries out + horizontally: one row per whole change block, each library + cell holding that version's slice of the block, padded so + the cells stay line-for-line aligned. Read across to see how + a block evolved; blocks touched by more than one step are + merged into a single row that carries per-step counts. + +The alignment is computed by difflib, never by hand -- the RRNs in the report +and the RRNs in the workbook come from the same computation and cannot drift. + +Usage +----- + python build_version_diff.py --program ICS100 \ + --src LIB2024=ICS100_LIB2024.txt \ + --src LIB2025=ICS100_LIB2025.txt \ + --src LIB2026=ICS100_LIB2026.txt \ + --meta ICS100_meta.json \ + --emit-md hunks.md --emit-xlsx ICS100_Version_Diff.xlsx + +Source files are either ``rrntext`` per line, or plain source (RRN = line +number). ``--emit-meta-template`` writes a JSON skeleton listing every hunk id +so the narrative can be filled in and fed back via ``--meta``. + +Requires openpyxl for --emit-xlsx (``pip install openpyxl``). +""" + +import argparse +import json +import re +import sys +from difflib import SequenceMatcher +from pathlib import Path + +# Diff palette. git tints the whole row and can therefore use very dark text; +# Excel gives a rich-text run no background of its own -- one fill per cell is +# the hard limit -- so here the *text* colour carries the entire signal and has +# to survive at 9pt. These are GitHub's foreground tones (the saturated ones it +# uses for icons and labels), not its on-tint text tones, which muddy together +# into brown at this size. Context is deliberately muted so changes pop. +FILL_CTX = "F6F8FA" +TEXT_ADD = "1A7F37" # added +TEXT_DEL = "CF222E" # removed +TEXT_TRANSIENT = "BF8700" # added/rewritten here, gone or rewritten again next +TEXT_CTX = "57606A" # unchanged context +BRAND = "1F4E79" + +ABSENT = "·" # gutter marker for "line not present in this version" + + +# ── Source loading ───────────────────────────────────────────────────── + + +def read_source(path): + """Return [(rrn, text), ...]. Accepts rrntext, else RRN = line number.""" + raw = Path(path).read_text(encoding="utf-8", errors="replace").split("\n") + if raw and raw[-1] == "": + raw.pop() + if raw and all(re.match(r"^\d+\t", ln) for ln in raw if ln): + out = [] + for ln in raw: + rrn, _, text = ln.partition("\t") + out.append((int(rrn), text)) + return out + return [(i + 1, ln) for i, ln in enumerate(raw)] + + +def key(text): + """Comparison key: trailing blanks are not a change.""" + return text.rstrip() + + +# ── RPG section context (git's "@@ ... @@ funcname") ──────────────────── + +BEGSR_RE = re.compile(r"^\s*(?:begsr|dcl-proc)\s+([*\w]+)", re.I) +BANNER_RE = re.compile(r"^\s*//\s*-*\s*(.*?)\s*-*\**\s*$") + + +def section_at(lines, idx): + """Nearest enclosing subroutine/procedure, else the nearest comment banner.""" + for i in range(idx, -1, -1): + text = lines[i][1] + m = BEGSR_RE.match(text) + if m: + return m.group(1) + m = BANNER_RE.match(text) + if m and m.group(1) and not set(m.group(1)) <= {"-", "*"}: + return m.group(1).rstrip(".") + return "" + + +# ── Pairwise hunks ───────────────────────────────────────────────────── + + +def build_hunks(old, new, context): + """Group the non-equal opcodes into hunks padded by `context` lines.""" + ops = SequenceMatcher( + None, [key(t) for _, t in old], [key(t) for _, t in new], autojunk=False + ).get_opcodes() + + groups, cur = [], [] + for op in ops: + tag, i1, i2, j1, j2 = op + if tag == "equal": + n = i2 - i1 + if cur: + cur.append(("equal", i1, min(i1 + context, i2), j1, min(j1 + context, j2))) + if n > 2 * context: + if cur: + groups.append(cur) + cur = [("equal", max(i1, i2 - context), i2, max(j1, j2 - context), j2)] + elif not cur: + cur = [op] + else: + cur.append(op) + if cur and any(t != "equal" for t, *_ in cur): + groups.append(cur) + return [g for g in groups if any(t != "equal" for t, *_ in g)] + + +def hunk_rows(group, old, new): + """Flatten one hunk into [(marker, old_rrn|None, new_rrn|None, text), ...].""" + rows = [] + for tag, i1, i2, j1, j2 in group: + if tag == "equal": + for oi, nj in zip(range(i1, i2), range(j1, j2)): + rows.append((" ", old[oi][0], new[nj][0], new[nj][1])) + else: + for oi in range(i1, i2): + rows.append(("-", old[oi][0], None, old[oi][1])) + for nj in range(j1, j2): + rows.append(("+", None, new[nj][0], new[nj][1])) + return rows + + +# ── Markdown emission ────────────────────────────────────────────────── + + +def render_md(steps, meta): + """steps: [(prev_lib, cur_lib, [(hunk_id, group, rows), ...]), ...]""" + hmeta = meta.get("hunks", {}) + out = [] + for prev, cur, hunks in steps: + added = sum(1 for _, _, rows in hunks for m, *_ in rows if m == "+") + removed = sum(1 for _, _, rows in hunks for m, *_ in rows if m == "-") + info = hmeta.get(f"{prev}->{cur}", {}) + title = info.get("title", "changes") + delta = f"+{added}" + (f" −{removed}" if removed else "") + out.append(f"### {prev} → {cur}: {title} ({delta} lines)\n") + if info.get("intro"): + out.append(info["intro"] + "\n") + for hid, section, rows in hunks: + a = sum(1 for m, *_ in rows if m == "+") + d = sum(1 for m, *_ in rows if m == "-") + hinfo = hmeta.get(hid, {}) + sect = hinfo.get("section") or section + parts = [f"{prev} → {cur}"] + ([sect] if sect else []) + [f"+{a} −{d}"] + out.append("```diff") + out.append("@@ " + " · ".join(parts) + " @@") + for marker, orrn, nrrn, text in rows: + o = str(orrn) if orrn else ABSENT + n = str(nrrn) if nrrn else ABSENT + out.append(f"{marker}{o:>5} │{n:>5} │ {text}".rstrip()) + out.append("```") + if hinfo.get("impact"): + out.append(f"\n**Behavioral impact:** {hinfo['impact']}\n") + else: + out.append("") + out.append("") + return "\n".join(out) + + +# ── N-way alignment for the workbook ─────────────────────────────────── + + +def align_all(libs, sources): + """Rows of {lib: line_index}; a missing lib key = line absent there. + + A ``replace`` pairs each rewritten line with its replacement on the *same* + master row, so a modified line reads straight across the library columns + instead of appearing as a delete two rows above its own insert. Surplus + lines on either side of the replace fall back to their own rows. + """ + rows = [{libs[0]: i} for i in range(len(sources[libs[0]]))] + for prev, cur in zip(libs, libs[1:]): + old, new = sources[prev], sources[cur] + ops = SequenceMatcher( + None, [key(t) for _, t in old], [key(t) for _, t in new], autojunk=False + ).get_opcodes() + out, p = [], 0 + + def carry_to(oi): + nonlocal p + while p < len(rows) and rows[p].get(prev) != oi: + out.append(rows[p]) + p += 1 + + for tag, i1, i2, j1, j2 in ops: + if tag == "equal": + for oi, nj in zip(range(i1, i2), range(j1, j2)): + carry_to(oi) + if p < len(rows): + rows[p][cur] = nj + out.append(rows[p]) + p += 1 + else: + paired = min(i2 - i1, j2 - j1) if tag == "replace" else 0 + for k, oi in enumerate(range(i1, i2)): + carry_to(oi) + if p < len(rows): + if k < paired: + rows[p][cur] = j1 + k + out.append(rows[p]) + p += 1 + for nj in range(j1 + paired, j2): + out.append({cur: nj}) + out.extend(rows[p:]) + rows = out + return rows + + +def line_state(rows, ri, libs, i, sources): + """(marker, colour) for library libs[i] on master row ri. + + Compares this line against its counterpart in the neighbouring libraries by + *text*, not just presence, so a rewritten line is reported as modified + (``~``) rather than as an unrelated delete + insert. + """ + row = rows[ri] + text = key(sources[libs[i]][row[libs[i]]][1]) + + def against(j): + """'gap' (no counterpart), 'diff' (rewritten) or 'same'.""" + if not 0 <= j < len(libs): + return "same" # no neighbour = nothing changed + other = libs[j] + if other not in row: + return "gap" + return "same" if key(sources[other][row[other]][1]) == text else "diff" + + back, fwd = against(i - 1), against(i + 1) + new_here = back in ("gap", "diff") + superseded = fwd in ("gap", "diff") + + if back == "gap": + marker = "+" + elif back == "diff" or fwd == "diff": + marker = "~" + elif fwd == "gap": + marker = "-" + else: + marker = " " + + if new_here and superseded: + colour = TEXT_TRANSIENT + elif new_here: + colour = TEXT_ADD + elif superseded: + colour = TEXT_DEL + else: + colour = TEXT_CTX + return marker, colour + + +# ── Workbook ─────────────────────────────────────────────────────────── + + +def write_xlsx(path, program, libs, sources, rows, steps, meta, context): + from openpyxl import Workbook + from openpyxl.styles import Alignment, Border, Font, PatternFill, Side + from openpyxl.utils import get_column_letter + + mono = Font(name="Consolas", size=9) + body = Font(name="Calibri", size=10) + head = Font(name="Calibri", size=10, bold=True, color="FFFFFF") + title = Font(name="Calibri", size=14, bold=True, color=BRAND) + head_fill = PatternFill("solid", fgColor=BRAND) + thin = Side(style="thin", color="D0D7DE") + box = Border(left=thin, right=thin, top=thin, bottom=thin) + top = Alignment(vertical="top", wrap_text=True) + + wb = Workbook() + + def header(ws, cols, row=1): + for c, name in enumerate(cols, start=1): + cell = ws.cell(row=row, column=c, value=name) + cell.font, cell.fill, cell.border = head, head_fill, box + cell.alignment = Alignment(vertical="center", wrap_text=True) + ws.freeze_panes = ws.cell(row=row + 1, column=1) + + # ── Summary ──────────────────────────────────────────────────────── + ws = wb.active + ws.title = "Summary" + ws["A1"] = f"{program} — Cross-Version Diff" + ws["A1"].font = title + info = [ + ("Program", program), + ("Description", meta.get("description", "")), + ("Type", meta.get("member_type", "")), + ("Source file", meta.get("source_file", "")), + ("Libraries (oldest → newest)", " → ".join(libs)), + ("Baseline", meta.get("baseline", libs[0])), + ("Author", meta.get("author", "iA by programmers.io")), + ("Date", meta.get("date", "")), + ] + r = 3 + for k, v in info: + ws.cell(row=r, column=1, value=k).font = Font(name="Calibri", size=10, bold=True) + ws.cell(row=r, column=2, value=v).font = body + r += 1 + + r += 1 + ws.cell(row=r, column=1, value="Version summary").font = Font( + name="Calibri", size=11, bold=True, color=BRAND + ) + r += 1 + header(ws, ["Library", "Total source code lines", "Δ vs prior", "Lines added", + "Lines removed", "What changed"], row=r) + ws.freeze_panes = None + vmeta = {v["library"]: v for v in meta.get("versions", [])} + prev_n = None + step_by_cur = {cur: (prev, hunks) for prev, cur, hunks in steps} + r += 1 + for lib in libs: + n = len(sources[lib]) + added = removed = "" + if lib in step_by_cur: + _, hunks = step_by_cur[lib] + added = sum(1 for _, _, rw in hunks for m, *_ in rw if m == "+") + removed = sum(1 for _, _, rw in hunks for m, *_ in rw if m == "-") + vals = [lib, n, "— (baseline)" if prev_n is None else f"{n - prev_n:+d}", + added, removed, vmeta.get(lib, {}).get("headline", "")] + for c, v in enumerate(vals, start=1): + cell = ws.cell(row=r, column=c, value=v) + cell.font, cell.border, cell.alignment = body, box, top + prev_n = n + r += 1 + + r += 1 + ws.cell(row=r, column=1, value="Legend").font = Font( + name="Calibri", size=11, bold=True, color=BRAND + ) + r += 1 + ws.cell(row=r, column=1, value="On the Diff sheet each row is one whole change block; " + "line colours inside a cell read like git. Every library " + "cell in a row is padded so line 1 of one cell is line 1 " + "of the next — read straight across.").font = body + r += 1 + for sample, label, colour in ( + ("+ 15 custno char(10) …", "Added in this library", TEXT_ADD), + ("- 15 custno char(10) …", "Removed in the next library", TEXT_DEL), + ("~ 15 custno char(10) …", "Rewritten — green = the new text, " + "red = the text it replaced (same row)", TEXT_ADD), + ("+ 15 custno char(10) …", "Added here, then rewritten or removed later", + TEXT_TRANSIENT), + (" 15 custno char(10) …", "Unchanged context", TEXT_CTX), + ("", "Blank line = that library has no counterpart here", TEXT_CTX)): + cell = ws.cell(row=r, column=1, value=sample) + cell.font = Font(name="Consolas", size=9, color="FF" + colour) + cell.fill, cell.border = PatternFill("solid", fgColor=FILL_CTX), box + ws.cell(row=r, column=2, value=label).font = body + r += 1 + + for col, w in (("A", 30), ("B", 24), ("C", 16), ("D", 13), ("E", 15), ("F", 70)): + ws.column_dimensions[col].width = w + + # ── Diff (side-by-side) — one whole hunk per row ─────────────────── + from openpyxl.cell.rich_text import CellRichText, TextBlock + from openpyxl.cell.text import InlineFont + + ws = wb.create_sheet("Diff (side-by-side)") + cols = ["Hunk", "Step", "Section", "Type", "Δ"] + list(libs) + ["Behavioral impact"] + header(ws, cols) + + hmeta = meta.get("hunks", {}) + code_fill = PatternFill("solid", fgColor=FILL_CTX) + block_align = Alignment(vertical="top", wrap_text=True) + + # master-row position of every (library, line index) + pos = {(lib, idx): i for i, row in enumerate(rows) for lib, idx in row.items()} + rrn_index = {lib: {rrn: i for i, (rrn, _) in enumerate(src)} + for lib, src in sources.items()} + + # One row per change block. Hunks from different steps that touch the same + # region are merged, so a block never appears twice. + records = [] + for prev, cur, hunks in steps: + for hid, section, hrows in hunks: + spans = [] + for _, orrn, nrrn, _ in hrows: + lib, rrn = (cur, nrrn) if nrrn is not None else (prev, orrn) + idx = rrn_index[lib].get(rrn) + if idx is not None and (lib, idx) in pos: + spans.append(pos[(lib, idx)]) + if not spans: + continue + hinfo = hmeta.get(hid, {}) + records.append({ + "lo": min(spans), "hi": max(spans), + "ids": [hid], + "by_step": {f"{prev} → {cur}": [ + sum(1 for m, *_ in hrows if m == "+"), + sum(1 for m, *_ in hrows if m == "-")]}, + "sections": [hinfo.get("section") or section], + "impacts": [hinfo.get("impact", "")], + }) + + records.sort(key=lambda x: (x["lo"], x["hi"])) + merged = [] + for rec in records: + if merged and rec["lo"] <= merged[-1]["hi"]: + m = merged[-1] + m["hi"] = max(m["hi"], rec["hi"]) + for k in ("ids", "sections", "impacts"): + m[k] += rec[k] + # A block touched by several steps keeps its counts per step, so the + # reader sees "inserted in 2025, then modified in 2026". + for step, (a, d) in rec["by_step"].items(): + tot = m["by_step"].setdefault(step, [0, 0]) + tot[0] += a + tot[1] += d + else: + merged.append(rec) + + def joined(values): + seen = [v for v in dict.fromkeys(values) if v] + return "; ".join(seen) + + def kind_of(a, d): + return "insert" if a and not d else "delete" if d and not a else "modify" + + def run(colour, text): + # Full-opacity ARGB ("FF…"): openpyxl pads a bare 6-digit colour with a + # 00 alpha channel, which strict renderers treat as transparent and + # replace with their own palette. + return TextBlock(InlineFont(rFont="Consolas", sz=9, color="FF" + colour), text) + + out_r = 2 + for rec in merged: + lo, hi = rec["lo"], rec["hi"] + counts = list(rec["by_step"].values()) + for c, v in enumerate([joined(rec["ids"]), + "; ".join(rec["by_step"]), + joined(rec["sections"]), + "; ".join(kind_of(a, d) for a, d in counts), + "; ".join(f"+{a} −{d}" for a, d in counts)], start=1): + cell = ws.cell(row=out_r, column=c, value=v) + cell.font, cell.border, cell.alignment = body, box, top + + # One cell per library: that version's slice of the block, with a blank + # line wherever the version has no counterpart, so the library cells + # stay line-for-line aligned. + for i, lib in enumerate(libs): + blocks = [] + for ri in range(lo, hi + 1): + if lib not in rows[ri]: + # Alignment padding. It gets an explicit font like every + # other run — a bare string run has no rPr, and renderers + # that don't inherit the cell font mis-colour the cell. + blocks.append(run(TEXT_CTX, "\n")) + continue + rrn, text = sources[lib][rows[ri][lib]] + marker, colour = line_state(rows, ri, libs, i, sources) + blocks.append(run(colour, f"{marker}{rrn:>5} {text}\n")) + while blocks and str(blocks[-1]) == "\n": + blocks.pop() + if blocks: + blocks[-1] = TextBlock(blocks[-1].font, str(blocks[-1]).rstrip("\n")) + cell = ws.cell(row=out_r, column=6 + i) + cell.value = CellRichText(blocks) if blocks else None + cell.fill, cell.border, cell.alignment = code_fill, box, block_align + cell.font = mono + + ic = ws.cell(row=out_r, column=len(cols), value=joined(rec["impacts"])) + ic.font, ic.border, ic.alignment = body, box, top + ws.row_dimensions[out_r].height = 12.6 * (hi - lo + 1) + out_r += 1 + + ws.auto_filter.ref = f"A1:{get_column_letter(len(cols))}{out_r - 1}" + ws.freeze_panes = "F2" + for col, w in (("A", 20), ("B", 20), ("C", 24), ("D", 10), ("E", 10)): + ws.column_dimensions[col].width = w + for i in range(len(libs)): + ws.column_dimensions[get_column_letter(6 + i)].width = 58 + ws.column_dimensions[get_column_letter(len(cols))].width = 60 + + # ── Change Log ───────────────────────────────────────────────────── + ws = wb.create_sheet("Change Log") + header(ws, ["Hunk", "Step", "Section", "Type", "Old lines", "New lines", + "+", "−", "What changed", "Behavioral impact", "Risks"]) + r = 2 + for prev, cur, hunks in steps: + for hid, section, hrows in hunks: + a = sum(1 for m, *_ in hrows if m == "+") + d = sum(1 for m, *_ in hrows if m == "-") + orrns = [o for m, o, _, _ in hrows if m == "-" and o] + nrrns = [n for m, _, n, _ in hrows if m == "+" and n] + hinfo = hmeta.get(hid, {}) + kind = "insert" if a and not d else "delete" if d and not a else "modify" + vals = [hid, f"{prev} → {cur}", hinfo.get("section") or section, kind, + f"{min(orrns)}–{max(orrns)}" if orrns else "—", + f"{min(nrrns)}–{max(nrrns)}" if nrrns else "—", + a, d, hinfo.get("what", ""), hinfo.get("impact", ""), + ", ".join(hinfo.get("risks", []))] + for c, v in enumerate(vals, start=1): + cell = ws.cell(row=r, column=c, value=v) + cell.font, cell.border, cell.alignment = body, box, top + r += 1 + for col, w in (("A", 20), ("B", 20), ("C", 22), ("D", 10), ("E", 12), ("F", 12), + ("G", 6), ("H", 6), ("I", 52), ("J", 62), ("K", 12)): + ws.column_dimensions[col].width = w + + # ── Risks ────────────────────────────────────────────────────────── + risks = meta.get("risks", []) + if risks: + ws = wb.create_sheet("Risks") + header(ws, ["#", "Category", "Scope", "Finding"]) + for r, item in enumerate(risks, start=2): + vals = [item.get("id", ""), item.get("category", ""), + item.get("scope", ""), item.get("finding", "")] + for c, v in enumerate(vals, start=1): + cell = ws.cell(row=r, column=c, value=v) + cell.font, cell.border, cell.alignment = body, box, top + for col, w in (("A", 8), ("B", 18), ("C", 20), ("D", 110)): + ws.column_dimensions[col].width = w + + wb.save(path) + + +# ── Driver ───────────────────────────────────────────────────────────── + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--program", required=True) + ap.add_argument("--src", action="append", required=True, metavar="LIB=PATH", + help="repeat, oldest first") + ap.add_argument("--meta", help="JSON narrative (sections, impact, risks)") + ap.add_argument("--context", type=int, default=2, help="context lines (default 2)") + ap.add_argument("--emit-md") + ap.add_argument("--emit-xlsx") + ap.add_argument("--emit-meta-template") + args = ap.parse_args() + + # The report uses a real minus sign; a cp1252 console must not kill the run. + for stream in (sys.stdout, sys.stderr): + try: + stream.reconfigure(encoding="utf-8", errors="replace") + except (AttributeError, ValueError): + pass + + libs, sources = [], {} + for spec in args.src: + lib, _, path = spec.partition("=") + if not path: + ap.error(f"--src expects LIB=PATH, got {spec!r}") + libs.append(lib) + sources[lib] = read_source(path) + if len(libs) < 2: + ap.error("need at least two --src versions to compare") + + meta = json.loads(Path(args.meta).read_text(encoding="utf-8")) if args.meta else {} + + steps = [] + for prev, cur in zip(libs, libs[1:]): + old, new = sources[prev], sources[cur] + hunks = [] + for n, group in enumerate(build_hunks(old, new, args.context), start=1): + rws = hunk_rows(group, old, new) + anchor = next((nrrn for _, _, nrrn, _ in rws if nrrn), None) + idx = next((i for i, (rrn, _) in enumerate(new) if rrn == anchor), 0) + hunks.append((f"{prev}->{cur}#{n}", section_at(new, idx), rws)) + steps.append((prev, cur, hunks)) + + total_add = sum(1 for _, _, h in steps for _, _, rw in h for m, *_ in rw if m == "+") + total_del = sum(1 for _, _, h in steps for _, _, rw in h for m, *_ in rw if m == "-") + n_hunks = sum(len(h) for _, _, h in steps) + print(f"{args.program}: {len(libs)} versions, {n_hunks} hunks, " + f"+{total_add} −{total_del} lines") + for prev, cur, hunks in steps: + for hid, section, rws in hunks: + a = sum(1 for m, *_ in rws if m == "+") + d = sum(1 for m, *_ in rws if m == "-") + print(f" {hid:<24} {section:<24} +{a} −{d}") + + if args.emit_meta_template: + tpl = { + "program": args.program, + "description": "", "member_type": "", "source_file": "", + "baseline": libs[0], "author": "iA by programmers.io", "date": "", + "versions": [{"library": l, "headline": ""} for l in libs], + "hunks": {}, + "risks": [{"id": "R1", "category": "Introduced", "scope": libs[-1], + "finding": ""}], + } + for prev, cur, hunks in steps: + tpl["hunks"][f"{prev}->{cur}"] = {"title": "", "intro": ""} + for hid, section, _ in hunks: + tpl["hunks"][hid] = {"section": section, "what": "", "impact": "", + "risks": []} + Path(args.emit_meta_template).write_text( + json.dumps(tpl, indent=2, ensure_ascii=False), encoding="utf-8") + print(f"wrote {args.emit_meta_template}") + + if args.emit_md: + Path(args.emit_md).write_text(render_md(steps, meta), encoding="utf-8") + print(f"wrote {args.emit_md}") + + if args.emit_xlsx: + rows = align_all(libs, sources) + write_xlsx(args.emit_xlsx, args.program, libs, sources, rows, steps, + meta, args.context) + print(f"wrote {args.emit_xlsx}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ia/scripts/convert_md_to_docx.py b/skills/ia/scripts/convert_md_to_docx.py index 23fa72a..690c1c3 100644 --- a/skills/ia/scripts/convert_md_to_docx.py +++ b/skills/ia/scripts/convert_md_to_docx.py @@ -47,6 +47,16 @@ COLOR_LINK = RGBColor(0x31, 0x82, 0xce) # Link blue COLOR_CODE_TEXT = RGBColor(0x2d, 0x37, 0x48) # Code block text +# Unified-diff palette (GitHub's), used by ```diff fences in comparison reports. +DIFF_ADD_BG_HEX = 'E6FFEC' +DIFF_DEL_BG_HEX = 'FFEBE9' +DIFF_CTX_BG_HEX = 'F6F8FA' +DIFF_HDR_BG_HEX = 'DDF4FF' +COLOR_DIFF_ADD = RGBColor(0x0a, 0x5c, 0x2e) # Added line text +COLOR_DIFF_DEL = RGBColor(0x8b, 0x1a, 0x1a) # Removed line text +COLOR_DIFF_HDR = RGBColor(0x05, 0x50, 0xae) # @@ hunk header +COLOR_DIFF_GUTTER = RGBColor(0x8c, 0x95, 0x9f) # RRN gutter + CODE_BG_HEX = 'F7FAFC' INLINE_CODE_BG_HEX = 'F1F5F9' HEADER_ROW_HEX = '2B6CB0' @@ -426,6 +436,66 @@ def add_mermaid_diagram(doc, code_lines: List[str]) -> bool: # ── Code blocks ──────────────────────────────────────────────────────── +def add_diff_block(doc, lines: List[str]): + """Render a ```diff fence git-style: green additions, red removals. + + Recognises the dual-RRN gutter emitted by build_version_diff.py + (``+ 123 | 129 | code``) and greys the gutter so the code stands out. + """ + for idx, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith('@@'): + bg, fg, bold = DIFF_HDR_BG_HEX, COLOR_DIFF_HDR, True + elif line.startswith('+'): + bg, fg, bold = DIFF_ADD_BG_HEX, COLOR_DIFF_ADD, False + elif line.startswith('-'): + bg, fg, bold = DIFF_DEL_BG_HEX, COLOR_DIFF_DEL, False + else: + bg, fg, bold = DIFF_CTX_BG_HEX, COLOR_CODE_TEXT, False + + p = doc.add_paragraph() + p.paragraph_format.space_before = Pt(0) + p.paragraph_format.space_after = Pt(0) + p.paragraph_format.line_spacing = 1.0 + p.paragraph_format.left_indent = Cm(0.4) + p.paragraph_format.right_indent = Cm(0.4) + + # Split "||" from the code so the numbers recede. + gutter, code = '', line if line else ' ' + if not stripped.startswith('@@'): + bar = line.find('│') + if bar != -1: + bar2 = line.find('│', bar + 1) + if bar2 != -1: + gutter, code = line[:bar2 + 1], line[bar2 + 1:] or ' ' + + if gutter: + run = p.add_run(gutter) + run.font.name = 'Consolas' + run.font.size = Pt(9) + run.font.color.rgb = COLOR_DIFF_GUTTER + run = p.add_run(code) + run.font.name = 'Consolas' + run.font.size = Pt(9.5) + run.font.color.rgb = fg + run.bold = bold + set_paragraph_shading(p, bg) + + pPr = p._p.get_or_add_pPr() + pBdr = OxmlElement('w:pBdr') + for side, on in (('top', idx == 0), ('bottom', idx == len(lines) - 1), + ('left', True), ('right', True)): + if not on: + continue + b = OxmlElement(f'w:{side}') + b.set(qn('w:val'), 'single') + b.set(qn('w:sz'), '4') + b.set(qn('w:space'), '4') + b.set(qn('w:color'), 'CBD5E0') + pBdr.append(b) + pPr.append(pBdr) + + def add_code_block(doc, lines: List[str], language: str = ''): if not lines: # Drop trailing blank lines @@ -974,7 +1044,10 @@ def convert_md_to_docx(md_file: str, docx_file: Optional[str] = None, if cover: add_cover_page(doc, meta, title) add_toc_page(doc) - else: + elif meta_skip: + # Only when a metadata block was consumed — meta_skip then points past + # the H1, so the body loop below never renders it. With no metadata + # block meta_skip is 0 and the body renders the H1 itself. p = doc.add_heading(level=1) parse_inline_formatting(p, title) p.alignment = WD_ALIGN_PARAGRAPH.LEFT @@ -1030,6 +1103,9 @@ def convert_md_to_docx(md_file: str, docx_file: Optional[str] = None, i += 1 # closing fence if lang.lower() == 'mermaid' and add_mermaid_diagram(doc, code_lines): continue + if lang.lower() == 'diff': + add_diff_block(doc, code_lines) + continue add_code_block(doc, code_lines, lang) continue