From 170b03d12179a3f9362ef9bcd6d9bc5e537139d9 Mon Sep 17 00:00:00 2001 From: Isabella Gottardi Date: Thu, 21 May 2026 15:55:05 +0100 Subject: [PATCH 1/5] docs: Add high-level documentation (#33) Signed-off-by: Isabella Gottardi --- README.md | 11 +- docs/README.md | 2 + docs/source/execution_flow.md | 149 +++++++++++++++++++++++++ docs/source/high_level_architecture.md | 83 ++++++++++++++ docs/source/index.md | 2 + mkdocs.yml | 9 ++ 6 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 docs/source/execution_flow.md create mode 100644 docs/source/high_level_architecture.md diff --git a/README.md b/README.md index 04856b4..38d8fbe 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,10 @@ Use the core docs for: - shared CLI guidance: [docs/source/cli.md](docs/source/cli.md) - backend discovery and installation model: [docs/source/backends.md](docs/source/backends.md) - output structure and JSON results: [docs/source/metrics.md](docs/source/metrics.md) -- architecture and repo boundaries: [docs/source/overview.md](docs/source/overview.md) +- architecture and repo boundaries: + [docs/source/overview.md](docs/source/overview.md), + [docs/source/high_level_architecture.md](docs/source/high_level_architecture.md), + and [docs/source/execution_flow.md](docs/source/execution_flow.md) Target-specific, backend-specific, and converter-specific detail belongs in the plugin repo that owns that functionality. @@ -153,9 +156,9 @@ If you need `torch.nn.Module` inputs, install the optional extra: pip install mlia[torch] ``` -The Python API follows the same plugin-based architecture as the CLI: the core package -provides the entry points and shared output structure, while installed plugins -extend what targets and backends are available. +The Python API follows the same plugin-based architecture as the CLI: the core +package provides the entry points and shared output structure, while installed +plugins extend what targets and backends are available. ## Development diff --git a/docs/README.md b/docs/README.md index daa6713..ed2796d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,9 @@ This directory contains the MkDocs content for the `mlia` repository. - `source/index.md`: documentation landing page - `source/overview.md`: role of the core repo in the split MLIA ecosystem +- `source/high_level_architecture.md`: core runtime layers and extension boundaries - `source/plugin_architecture.md`: how the core repo relates to plugin repos +- `source/execution_flow.md`: shared `mlia check` command, API, data, and artifact flow - `source/cli.md`: command-line entry points and common workflows - `source/backends.md`: backend discovery, installation, and responsibility boundaries - `source/metrics.md`: output formats, schema shape, and metrics guidance diff --git a/docs/source/execution_flow.md b/docs/source/execution_flow.md new file mode 100644 index 0000000..103bef0 --- /dev/null +++ b/docs/source/execution_flow.md @@ -0,0 +1,149 @@ + + +# End-to-end Execution Flow + +This page follows a typical `mlia check` run from the command line to final +reports. Plugin-specific collectors, analyzers, converters, and backend tools may +vary, but the core orchestration path is shared. + +Use this page for runtime ordering and failure handling. For the static module +layout, see [High-level Architecture](high_level_architecture.md). For plugin +discovery and extension points, see [Plugin Architecture](plugin_architecture.md). + +## Command flow + +```mermaid +sequenceDiagram + participant User + participant CLI as mlia CLI + participant Commands as mlia.cli.commands + participant API as mlia.api + participant TargetReg as Target registry + participant BackendReg as Backend registry + participant BackendMgr as Backend manager + participant Advisor as Target advisor + participant Workflow as Workflow executor + participant Reports as Output handlers + + User->>CLI: mlia check model --target-profile profile --performance + CLI->>CLI: Parse args and create ExecutionContext + CLI->>Commands: check(ctx, target_profile, model, flags, backends) + Commands->>API: get_advice(target_profile, model, category, context, backends) + API->>TargetReg: Resolve profile and target + TargetReg->>BackendReg: Ensure backend plugins loaded + TargetReg->>TargetReg: Ensure target plugins loaded + API->>API: Validate advice category and backend selection + API->>BackendMgr: ensure_backends_installed(validated_backends) + API->>TargetReg: Get target advisor factory + TargetReg-->>API: Advisor factory + API->>Advisor: Create advisor for target profile and model + API->>Advisor: run(context) + Advisor->>Workflow: Configure collectors/analyzers/producers/events + Workflow->>Workflow: Collect data + Workflow->>Workflow: Analyze data + Workflow->>Workflow: Detect patterns when configured + Workflow->>Reports: Produce advice and publish events + Reports-->>User: Console output, JSON output, files, logs +``` + +## Runtime responsibilities + +| Step | Core owner | What happens | +| --- | --- | --- | +| Parse command | `mlia.cli.main` | Builds an argument parser from core and CLI plugin commands. | +| Build context | `mlia.cli.main.setup_context()` | Creates `ExecutionContext`, resolves output format, output directory, action resolver, and logging. | +| Select advice category | `mlia.cli.commands.check()` | Converts `--compatibility` and `--performance` flags into an advice-category set. | +| Validate target/profile | `mlia.cli.command_validators`, `mlia.target.registry` | Confirms the selected target profile supports the requested category. | +| Validate backends | `mlia.api.get_advice()` | Resolves requested backends against the target and supported backend registry. | +| Ensure backend installation | `mlia.backend.manager` | Checks or installs backends as required, subject to EULA and noninteractive settings. | +| Create advisor | `mlia.api.get_advisor()` | Finds the target from the profile and calls the target's advisor factory. | +| Run workflow | `mlia.core.advisor`, `mlia.core.workflow` | Executes collectors, analyzers, optional pattern analyzers, and advice producers. | +| Produce output | `mlia.core.reporting`, event handlers, output schema helpers | Emits console output, logs, generated files, and JSON-compatible data when requested. | + +## Conceptual data flow + +```mermaid +graph LR + model[Input model] --> command[mlia check] + profile[Target profile TOML] --> command + backendOptions[Backend options] --> command + + command --> context[ExecutionContext] + context --> outputDir[Output directory] + context --> logs[logs/mlia.log] + + model --> collectors[Data collectors] + profile --> advisor[Target advisor] + advisor --> collectors + collectors --> maybeConvert{Converter needed?} + maybeConvert -->|yes| converter[Converter plugin] + converter --> intermediate[Converted/intermediate artifact] + maybeConvert -->|no| backendInput[Backend input artifact] + intermediate --> backendInput + backendInput --> backend[Backend tool or service] + backend --> rawResults[Raw backend output] + rawResults --> analyzers[Data analyzers] + analyzers --> facts[Analyzed facts] + facts --> producers[Advice producers] + producers --> advice[Advice] + advice --> events[Events] + events --> handlers[Event handlers and reporters] + handlers --> reports[Reports and JSON output] + reports --> outputDir +``` + +The exact collector inputs, converted artifact names, backend outputs, and +analyzer facts are plugin-owned. The core guarantees the shared execution +context, workflow stages, event publishing, and output/reporting mechanisms. + +## Failure flow + +MLIA failures normally terminate the command and return a non-zero exit status. +The CLI always reports the run output directory at the end of command execution. +Unless `--output-dir` is set, logs and generated artifacts are written under +`mlia-output` in the current working directory. The main log file is +`mlia-output/logs/mlia.log`. + +In normal CLI output, handled errors are intentionally short: + +```text +Execution finished with error: +Please check the log files in the mlia-output/logs for more details, or enable debug mode (--debug) +This execution of MLIA used output directory: mlia-output +``` + +With `--debug`, MLIA prints debug logging to the console and records debug logs +in `mlia-output/logs/mlia.log`. For unexpected exceptions, `--debug` also shows +the Python traceback. Handled error categories, such as configuration errors, +backend availability errors, and interrupted execution, are still displayed as +concise messages. + +The CLI handles these categories specially: + +| Failure category | User-visible behavior | +| --- | --- | +| Configuration error | Prints the configuration error message directly. | +| Backend unavailable | Prints `Error: Backend is not available.` Some optional backends also include an installation hint, for example `mlia-backend install ""`. | +| Interrupted execution | Prints `Execution has been interrupted`. | +| Internal error | Prints `Internal error: `. | +| Other exception | Prints `Execution finished with error: `, points to the log directory, and shows a traceback when `--debug` is enabled. | + +Inside the workflow, exceptions from collection, analysis, pattern detection, or +advice production are first published as an `ExecutionFailedEvent`. That event +contains the original exception in its `err` field so event handlers can observe +or transform the failure. The default workflow event handler re-raises that +exception, so the CLI error handling described above still decides what the user +sees. + +## API flow + +Python callers can use `mlia.api` directly instead of invoking the CLI. The API +path still validates target profiles, backend selection, backend installation, +and advisor creation. `run_advisor()` additionally supports returning a +standardized JSON-compatible dictionary and optional schema validation modes. + +Use the CLI when you want the standard command-line user experience. Use the API +when another Python tool needs to run MLIA and consume structured results. diff --git a/docs/source/high_level_architecture.md b/docs/source/high_level_architecture.md new file mode 100644 index 0000000..c506182 --- /dev/null +++ b/docs/source/high_level_architecture.md @@ -0,0 +1,83 @@ + + +# High-level Architecture + +MLIA is a Python application and library built around one core package and +installable plugins. The core package owns the shared user experience and +runtime contracts; plugins add target, backend, converter, or CLI functionality +through Python entry points. + +Use this page for the static system shape. For plugin loading details, see +[Plugin Architecture](plugin_architecture.md). For the step-by-step +`mlia check` path, see [End-to-end Execution Flow](execution_flow.md). + +## System view + +```mermaid +graph TD + user[User] --> mainCli[mlia] + user --> backendCli[mlia-backend] + user --> targetCli[mlia-target] + automation[Automation or Python caller] --> api[Python API] + + mainCli --> checkCommand[mlia.cli.commands.check] + mainCli --> cliPlugins[CLI plugin commands] + checkCommand --> api + + backendCli --> backendCommands[backend install/list/uninstall] + targetCli --> targetCommands[target list] + + api --> context[ExecutionContext] + api --> targetRegistry[Target registry] + api --> backendRegistry[Backend registry] + api --> backendManager[Installation manager] + api --> advisorFactory[Target advisor factory] + + targetRegistry --> targetPlugins[Target plugins] + backendRegistry --> backendPlugins[Backend plugins] + advisorFactory --> advisor[InferenceAdvisor] + + advisor --> workflow[DefaultWorkflowExecutor] + workflow --> collectors[Data collectors] + workflow --> analyzers[Data analyzers] + workflow --> patterns[Pattern analyzers] + workflow --> producers[Advice producers] + workflow --> events[Event publisher and handlers] + + collectors --> backends[Backend tools and services] + backendManager --> backends + backendCommands --> backendManager + targetCommands --> targetRegistry + producers --> advice[Advice] + advice --> events + events --> handlers[Event handlers and reporters] + handlers --> reports[Console, JSON, and output files] +``` + +## Main runtime layers + +| Layer | Core modules | Responsibility | +| --- | --- | --- | +| CLI | `mlia.cli.main`, `mlia.cli.commands`, `mlia.cli.options` | Parse `mlia`, `mlia-backend`, and `mlia-target` commands, create `ExecutionContext`, configure logging and output format, and call command functions. | +| Public API | `mlia.api` | Validate inputs, resolve targets and backends, ensure required backends are installed, create advisors, optionally return standardized output. | +| Registries | `mlia.target.registry`, `mlia.backend.registry`, `mlia.plugins.*` | Load plugin entry points once and expose registered target, backend, converter, and CLI capabilities. | +| Backend management | `mlia.backend.manager`, `mlia.backend.install`, `mlia.backend.config` | Show backend status, install/uninstall backends, resolve dependencies, and provide backend configuration metadata. | +| Execution context | `mlia.core.context` | Carry advice category, config parameters, output directory, action resolver, event publisher, handlers, and output format. | +| Advisor/workflow | `mlia.core.advisor`, `mlia.core.workflow` | Build and run the target-specific workflow stages. | +| Reporting/output | `mlia.core.reporting`, `mlia.core.reporters`, `mlia.core.output_schema`, `mlia.core.output_validation` | Produce human-readable and JSON-compatible results and validate standardized output. | + +## Ownership boundaries + +The core package should not need target-specific conditionals for every new +hardware family or converter. Instead: + +- target plugins register target metadata and advisor factories +- backend plugins register backend configuration and installation metadata +- converter plugins register named converter callables +- CLI plugins can append commands to the command list + +This keeps core orchestration stable while allowing target, backend, and +converter packages to evolve independently. diff --git a/docs/source/index.md b/docs/source/index.md index 104d8b5..1b8c632 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -16,7 +16,9 @@ see the individual plugin repositories. ## Contents - [Overview](overview.md) +- [High-level Architecture](high_level_architecture.md) - [Plugin Architecture](plugin_architecture.md) +- [End-to-end Execution Flow](execution_flow.md) - [CLI](cli.md) - [API](api.md) - [Backends](backends.md) diff --git a/mkdocs.yml b/mkdocs.yml index 700652d..88ca6e5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,14 +25,23 @@ markdown_extensions: - attr_list - def_list - footnotes + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid - tables - toc: permalink: true +extra_javascript: + - https://unpkg.com/mermaid@10.9.1/dist/mermaid.min.js + nav: - Home: index.md - Overview: overview.md + - High-level Architecture: high_level_architecture.md - Plugin Architecture: plugin_architecture.md + - End-to-end Execution Flow: execution_flow.md - CLI: cli.md - API: api.md - Backends: backends.md From 9a30209aa309333141c7b674d00e9f288157821e Mon Sep 17 00:00:00 2001 From: Christopher Sidebottom Date: Mon, 25 May 2026 15:22:02 +0100 Subject: [PATCH 2/5] ci: Gate release publishing on tags (#34) --- .github/workflows/release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29cf9f2..c0fdd1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,12 @@ jobs: publish_release: name: Publish Release + if: >- + ${{ + always() && + needs.build_and_test_wheel.result == 'success' && + (github.repository_owner != 'arm' || github.ref_type == 'tag') + }} needs: build_and_test_wheel runs-on: ${{ fromJson(vars.UBUNTU_RUNNER || '["ubuntu-24.04-arm"]') }} steps: From 6c005c4bded99eb30969dcf738c168ec6a6b0602 Mon Sep 17 00:00:00 2001 From: Christopher Sidebottom Date: Tue, 26 May 2026 10:09:52 +0100 Subject: [PATCH 3/5] build: Add public repo sync workflow (#8) * build: Add public repo sync workflow This enables syncing `main` and any tags to the public repo. * fix: Use workflow wizardry not ai debug * fix: Indentation and point to main of new repo * fix: Ensure we dont run publish flow without configuration This also ensures we only run against tags in the open source repo. --- .github/workflows/release.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c0fdd1e..0567782 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ on: - main tags: - "v*.*.*" + workflow_dispatch: permissions: contents: read @@ -68,7 +69,10 @@ jobs: (github.repository_owner != 'arm' || github.ref_type == 'tag') }} needs: build_and_test_wheel + if: ${{ github.repository_owner != 'arm' || github.ref_type == 'tag' }} runs-on: ${{ fromJson(vars.UBUNTU_RUNNER || '["ubuntu-24.04-arm"]') }} + env: + PUBLISH_PYPI_HOST: ${{ secrets.PUBLISH_PYPI_HOST || '' }} steps: - uses: actions/checkout@v4 with: @@ -77,8 +81,17 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - run: uv build - - run: uv publish dist/* + - if: ${{ env.PUBLISH_PYPI_HOST != '' }} + run: uv publish dist/* env: - UV_PUBLISH_URL: ${{ secrets.PUBLISH_PYPI_HOST }} + UV_PUBLISH_URL: ${{ env.PUBLISH_PYPI_HOST }} UV_PUBLISH_USERNAME: ${{ secrets.INTERNAL_PYPI_USERNAME || secrets.PUBLISH_PYPI_USERNAME }} UV_PUBLISH_PASSWORD: ${{ secrets.INTERNAL_PYPI_PASSWORD || secrets.PUBLISH_PYPI_PASSWORD }} + + sync_to_public_repo: + needs: build_and_test_wheel + if: ${{ github.event.repository.private }} + uses: Arm-Debug/github-workflow-wizardry/.github/workflows/sync-to-public-repo.yml@main + with: + public_repo: arm/mlia + secrets: inherit From bf46c3b8df8b2e5a5449c6c8fd1de66adb02c92f Mon Sep 17 00:00:00 2001 From: Christopher Sidebottom Date: Tue, 26 May 2026 10:34:53 +0100 Subject: [PATCH 4/5] fix: Remove duplicate if-block from release.yml (#37) --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0567782..a1fc5e5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,6 @@ jobs: (github.repository_owner != 'arm' || github.ref_type == 'tag') }} needs: build_and_test_wheel - if: ${{ github.repository_owner != 'arm' || github.ref_type == 'tag' }} runs-on: ${{ fromJson(vars.UBUNTU_RUNNER || '["ubuntu-24.04-arm"]') }} env: PUBLISH_PYPI_HOST: ${{ secrets.PUBLISH_PYPI_HOST || '' }} From f8abc8d34ea26dc2f6f2fbe0c24c1d7b8ee28707 Mon Sep 17 00:00:00 2001 From: jedgar-arm Date: Tue, 26 May 2026 14:30:36 +0100 Subject: [PATCH 5/5] feat: Add availability-aware standardized performance metrics (#32) Signed-off-by: James Edgar --- docs/source/metrics.md | 34 +++ .../.openspec.yaml | 2 + .../.openspec.yaml.license | 2 + .../design.md | 139 ++++++++++ .../design.md.license | 2 + .../proposal.md | 42 +++ .../proposal.md.license | 2 + .../standard-performance-metrics/spec.md | 231 ++++++++++++++++ .../spec.md.license | 2 + .../add-standard-performance-metrics/tasks.md | 75 +++++ .../tasks.md.license | 2 + pyproject.toml | 1 + src/mlia/core/output_schema.py | 124 ++++++++- src/mlia/core/output_validation.py | 2 +- .../resources/mlia-output-schema-1.1.0.json | 260 ++++++++++++++++++ .../mlia-output-schema-1.1.0.json.license | 2 + tests/test_core_output_schema.py | 132 +++++++++ tests/test_core_output_validation.py | 133 +++++++++ 18 files changed, 1182 insertions(+), 5 deletions(-) create mode 100644 openspec/changes/add-standard-performance-metrics/.openspec.yaml create mode 100644 openspec/changes/add-standard-performance-metrics/.openspec.yaml.license create mode 100644 openspec/changes/add-standard-performance-metrics/design.md create mode 100644 openspec/changes/add-standard-performance-metrics/design.md.license create mode 100644 openspec/changes/add-standard-performance-metrics/proposal.md create mode 100644 openspec/changes/add-standard-performance-metrics/proposal.md.license create mode 100644 openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md create mode 100644 openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md.license create mode 100644 openspec/changes/add-standard-performance-metrics/tasks.md create mode 100644 openspec/changes/add-standard-performance-metrics/tasks.md.license create mode 100644 src/mlia/resources/mlia-output-schema-1.1.0.json create mode 100644 src/mlia/resources/mlia-output-schema-1.1.0.json.license diff --git a/docs/source/metrics.md b/docs/source/metrics.md index 3d6d02f..901d759 100644 --- a/docs/source/metrics.md +++ b/docs/source/metrics.md @@ -56,6 +56,40 @@ A simplified example: The exact contents of the output depend on the installed plugins, but the high-level structure stays stable. +### Metric availability + +From schema version `1.1.0`, result metrics can be represented in two ways: + +- numeric metrics with `name`, `value`, and `unit` +- unavailable metric entries with `name`, `unit`, `availability`, and `reason` + +For numeric metrics, omitted `availability` means that the value is available. +Unavailable metric entries do not contain a placeholder `value`. + +This explicit availability marker is currently limited to the standardized +performance fields added for this work: `accelerator_operator_percentage`, +`inferences_per_second`, `cpu_utilization`, `target_utilization`, +`peak_activation_memory`, and `average_memory`. It is not a complete availability +map for every possible consumer field. + +Example metric entries: + +```json +[ + { + "name": "inferences_per_second", + "value": 4830.9, + "unit": "inferences/s" + }, + { + "name": "cpu_utilization", + "unit": "%", + "availability": "unavailable", + "reason": "CPU utilization data is not available." + } +] +``` + ## How to read a result A recommended reading order is: diff --git a/openspec/changes/add-standard-performance-metrics/.openspec.yaml b/openspec/changes/add-standard-performance-metrics/.openspec.yaml new file mode 100644 index 0000000..93831bd --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-05-13 diff --git a/openspec/changes/add-standard-performance-metrics/.openspec.yaml.license b/openspec/changes/add-standard-performance-metrics/.openspec.yaml.license new file mode 100644 index 0000000..a22d9ae --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/.openspec.yaml.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright 2026, Arm Limited and/or its affiliates. +SPDX-License-Identifier: Apache-2.0 diff --git a/openspec/changes/add-standard-performance-metrics/design.md b/openspec/changes/add-standard-performance-metrics/design.md new file mode 100644 index 0000000..c41a81a --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/design.md @@ -0,0 +1,139 @@ +## Context + +Consumer input asks MLIA to expose supported model performance fields in standardized JSON output. MLIA's standardized output schema is the implementation contract, and this change updates it to schema version `1.1.0`. External consumer schemas may map from MLIA output, but they are not the implementation schema for this public core change. + +This change is intentionally narrow. It covers the supported fields called out by the current work items: + +- known limitations and interpretation notes +- percentage of operators executed on the accelerator +- inference throughput +- CPU utilization +- target utilization +- peak activation memory +- average memory footprint + +MLIA already has schema locations for much of the broader requirement summary: analysis metadata is represented by top-level fields such as `timestamp`, `tool`, `target`, `model`, `backends`, and `context`; compatibility is represented by compatibility results, status, checks, warnings, and errors; and latency can already be emitted as a numeric result metric by backend plugins. This change should preserve those existing outputs while adding the missing supported fields. Compatibility, version, target, and evaluation-mode output may require consumer-specific mapping because MLIA deliberately keeps richer structured data than some report formats. Latency remains an existing plugin-provided metric in this change; current plugin naming and unit differences should be documented as a partial mapping rather than normalized by the core change. + +The implementation should use a metric-by-metric mapping rather than assume every standardized field is new. Existing MLIA metrics that already satisfy a requirement should be documented with their existing names. Genuinely missing fields should be added with standard MLIA metric names and units. Throughput needs particular care because current plugins may already emit throughput-like metrics with backend-specific names. + +The Vela/Corstone "all stats" work is related, but it is plugin-owned and should be specified separately in the owning plugin repository. Optional memory-profile fields such as model weight memory also belong to that later plugin-owned stats-completeness work, not this core standard-fields change. + +## Goals / Non-Goals + +**Goals:** + +- Add the requested supported fields to MLIA standardized JSON output. +- Define the MLIA standardized metric mapping for the standardized performance metric set. +- Use schema-valid MLIA output locations rather than introducing a parallel consumer-shaped report. +- Put global single-inference numeric metrics under `results[*].metrics`. +- Explicitly mark requested fields without values rather than fabricating values. +- Define clear missing-data behavior for each added field. +- Add unit tests and representative JSON coverage for the new fields. +- Keep existing standardized JSON reporting and Python API collection workflows valid. + +**Non-Goals:** + +- Do not add measured values for power, energy, hardware-measured runtime, or batch performance fields when backend source data is absent. +- Do not make private target names, private product context, or consumer-specific requirements part of public MLIA artifacts. +- Do not implement backend-specific Vela/Corstone stats completeness in core MLIA. +- Do not fix plugin-specific bugs as part of the core change unless they block a neutral representative payload. + +## Decisions + +### MLIA standardized output remains authoritative + +MLIA will represent the requested fields through the existing standardized output schema. Numeric global single-inference values belong in `results[*].metrics`. Notes and compatibility-adjacent information must use schema-valid non-metric locations, such as result warnings, checks, entities, or a documented extension if an existing field is not suitable. + +Known limitations and interpretation notes should be represented as result warnings. A consumer that emits a consumer-shaped report can summarize or join those warnings into `analysisLimitations`. + +Alternative considered: emit a separate consumer-shaped JSON document. That would duplicate MLIA's existing standardized output path and make reporting and API behavior harder to keep consistent. + +### Result-level metrics are limited to global values + +Accelerator operator percentage, inference throughput, CPU utilization, target utilization, peak activation memory, and average memory footprint are global result-level values and should be emitted under `results[*].metrics`. + +Accelerator operator percentage is an operator-placement metric, not a measured runtime percentage. The standard MLIA metric name for this field is `accelerator_operator_percentage`. Its mapping documentation should make the placement semantics clear. The requirement context refers to compatibility-style pass, fail, and partial status, which is produced by compatibility analysis rather than performance analysis. Plugins should therefore populate this metric from compatibility/operator-placement data when that data is available, and should not infer it from a performance-only payload. + +Operator-level or layer-level values are outside this core change unless they are needed as source data for one of the requested result-level metrics. + +### Metric semantics are explicit + +Inference throughput is derived from suitable latency data when available, but the backend or plugin owns the calculation because it owns the latency semantics. It uses the standard MLIA metric name `inferences_per_second` and unit `inferences/s`. CPU utilization uses the standard MLIA metric name `cpu_utilization`; MLIA must emit it as an availability-aware metric entry when no backend source can provide a supported value. Target utilization uses the standard MLIA metric name `target_utilization` and is calculated from suitable cycle data as `(compute_cycles / total_cycles) * 100 if total_cycles else 0.0`. Peak activation memory uses the standard MLIA metric name `peak_activation_memory` and means the highest activation memory usage during a single inference run. Average memory uses the standard MLIA metric name `average_memory` and means average memory usage over the full measurement window, not peak, allocation total, or instantaneous memory. The field tickets require these memory metrics to be added to result metrics; backends should emit numeric values where they own suitable source data and use explicit unavailable entries only where that source data is absent. + +Percentage metrics in this field set use values in the range `0..100` with unit `%`. Backends and plugins should normalize source values into that convention before emitting standardized output. + +Memory metrics in this field set use unit `bytes`. This follows existing MLIA metric conventions; consumers that emit consumer-shaped reports can map `bytes` to `B`. + +### Standard names are generic MLIA names + +Metric names and unit constants added by this work should describe the MLIA standardized output contract, not the first consuming workflow. + +The identifiers selected here are MLIA standardized JSON metric names. They should be generic enough for multiple consumers, and MLIA should not emit consumer-specific field paths directly. + +For existing MLIA metrics, the implementation should decide case by case whether to keep the existing name, rename it, or add an alias. Existing output should not be renamed blindly. + +### Scoped MLIA mapping + +This mapping is intentionally limited to the fields needed for the current standardized performance metrics requirement. It is not a full mapping to any consumer-specific schema. + +| Scope | MLIA standardized output | Notes | +| --- | --- | --- | +| Analysis metadata | top-level `timestamp`, `tool`, `target`, `model`, `backends`, and `context` | Preserve existing structured metadata. | +| Compatibility | compatibility `results[*].status`, `checks`, and `entities` | Preserve the richer MLIA compatibility model, including non-boolean statuses. | +| Known limitations and interpretation notes | `results[*].warnings` | Keep notes result-scoped. | +| Single-inference latency | existing backend metric names | Preserve existing plugin output rather than defining a new standard latency metric in this change. | +| Inference throughput | metric `inferences_per_second`, unit `inferences/s` | Backend or plugin owns the latency semantics needed to calculate throughput. | +| Accelerator operator percentage | metric `accelerator_operator_percentage`, unit `%` | Operator-placement metric, not a measured runtime percentage. | +| CPU utilization | metric `cpu_utilization`, unit `%`, or an unavailable metric entry | Emit `unavailable` unless a backend can provide a real value. | +| Target utilization | metric `target_utilization`, unit `%` | Generic MLIA metric name for target utilization derived from suitable cycle data. | +| Peak activation memory | metric `peak_activation_memory`, unit `bytes`, or an unavailable metric entry | Emit numeric values only where the plugin owns suitable source data. | +| Average memory | metric `average_memory`, unit `bytes`, or an unavailable metric entry | Must mean average memory over the measurement window. | +| Scoped metric fields without values | availability-aware metric entries with `availability`, `unit`, and `reason` | MLIA must not fabricate values; markers are limited to the current standardized performance field set. | + +Fields outside this mapping, including optional memory-profile and memory-traffic statistics, remain outside this core change. Existing backend-specific metrics should be preserved where already emitted, but new or standardized completeness work should be handled by a later plugin-owned statistics-completeness change if required. + +### Missing data is represented in metrics + +If MLIA or the selected backend cannot provide the source data for a requested field, the output must make that state clear using the chosen schema-valid representation. MLIA must not approximate or fabricate values. + +Every metric field added or standardized by this change must be represented in `results[*].metrics` either as a numeric metric or as an availability-aware metric entry. This includes `accelerator_operator_percentage`, because it is covered by the MLIA work items even though it is sourced differently from ordinary performance metrics. Existing metadata, compatibility, and latency mappings should remain in their normal MLIA locations or be documented as partial mappings rather than represented as placeholder metric entries. This does not require MLIA to cover every possible consumer field. + +This bounded scope should be documented as a limitation for MLIA output consumers so that users do not assume availability markers are comprehensive for every possible consumer field. + +MLIA should extend the metric schema to support availability-aware metric entries. Supported metrics keep the existing numeric shape with `name`, `value`, and `unit`. Explicit `availability` is optional for supported metrics; if `value` is present and `availability` is omitted, consumers should treat the metric as available. + +Added or standardized metric fields without values are represented in `results[*].metrics` with an explicit availability state, `unit`, and a short human-readable `reason`, without a fake numeric `value`. The initial implementation should use `unavailable` as the only non-value state. Add `unsupported` later only if implementation evidence shows MLIA needs to distinguish fields the selected backend cannot support from values that are supported in principle but unavailable for a particular run. Unit remains mandatory for availability entries in this contract, with canonical units defined in the mapping table; revisit only if implementation exposes a real unitless field. + +The JSON schema should model this as a single metric array with two valid metric entry shapes: numeric entries with `value`, and non-value availability entries with `availability` and `reason`. This keeps consumers on one result-level metric list while making the absence of a numeric value explicit. + +Alternative considered: put non-value fields in result checks or extensions. That avoids changing the metric schema, but it splits requested field availability away from the metric list consumers naturally inspect. Given the expected limited current consumer set, an availability-aware metric model is the cleaner contract as long as schema-version, compatibility notes, and tests are handled deliberately. + +This change should bump the MLIA output schema version from `1.0.0` to `1.1.0`. Keeping `1.0.0` while adding non-value metric entries would silently change the contract for consumers that assume every metric entry has a numeric `value`. + +### Plugin extraction stays plugin-owned + +Core MLIA defines the output placement, validation expectations, standard metric names, units, and a shared helper for ensuring the added standard metrics are represented. Target and backend plugins remain responsible for extracting backend-specific source values and mapping them into the core output shape. + +Plugins should call the core helper when constructing performance results. The helper should preserve supplied numeric metric values and add availability-aware entries for added or standardized metrics that are missing. The initial implementation should not run this helper automatically across all core reporting paths; plugin call sites should opt in explicitly. This keeps the common contract in core while avoiding duplicate unavailable-fill logic in each plugin. + +The initial core `mlia` PR should define the shared contract and helper behavior only. Plugin-owned integration tasks are expected follow-up work, but they should be specified and reviewed in the owning plugin repositories rather than being part of the initial core PR. + +The first representative plugin integration should use the Ethos-U Vela performance path. This proves the helper against a real backend with throughput, memory, and cycle data without expanding the initial implementation to every plugin. The Vela work should have a small `mlia-ethos-u` OpenSpec change that covers helper integration, backend source extraction, plugin-owned validation, and the result-level metrics overwrite fix if it blocks a trustworthy representative payload. Other plugin integrations should follow once the core shape is stable. + +## Risks / Trade-offs + +- The metric schema change means consumers must not assume every metric entry has a numeric `value`; the `1.1.0` schema version is the compatibility signal for that change. +- Availability states can become over-modeled; add distinct states only when implementation needs the distinction. +- Existing tests and plugin output code may need updates where they assume every metric has `name`, `value`, and `unit`. +- The design input suggested `results[*].details` for compatibility percentage, but the current MLIA schema does not define that field. +- Existing plugins already emit latency-like metrics with different names and at least one unit inconsistency risk, so latency should remain a documented partial mapping unless a separate standardization decision is made. +- Some requested values may not be available for every backend. Tests must cover missing-data behavior as well as supported values. +- Plugin-owned extraction may be needed before every requested field can be populated for every target. +- Using Vela as the first representative integration does not prove every plugin is complete; follow-up plugin work remains necessary. + +## Follow-Up Work + +- Create and implement a small `mlia-ethos-u` OpenSpec change for the Vela representative integration. +- Specify and implement the Vela/Corstone "all stats" work in the owning plugin repository. +- Track plugin-specific issues, such as result-level metrics being overwritten by breakdown metrics, in the owning plugin repository. If that overwrite issue blocks the Vela representative payload, fix it as part of the Vela integration rather than deferring it. +- Add plugin-specific payload tests once the core output placement is agreed. diff --git a/openspec/changes/add-standard-performance-metrics/design.md.license b/openspec/changes/add-standard-performance-metrics/design.md.license new file mode 100644 index 0000000..a22d9ae --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/design.md.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright 2026, Arm Limited and/or its affiliates. +SPDX-License-Identifier: Apache-2.0 diff --git a/openspec/changes/add-standard-performance-metrics/proposal.md b/openspec/changes/add-standard-performance-metrics/proposal.md new file mode 100644 index 0000000..a46658a --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/proposal.md @@ -0,0 +1,42 @@ +## Why + +MLIA already emits standardized JSON output, but several standardized performance fields are either missing from JSON or only available in stdout. The immediate requirement is to add those supported fields to MLIA standardized output without inventing metrics that MLIA cannot produce. + +## What Changes + +- Add known limitations and notes on interpreting results to standardized JSON output. +- Add percentage of operators executed on the accelerator to standardized JSON output. +- Add inference throughput to result-level metrics, derived from latency when latency is available. +- Add CPU utilization to result-level metrics when backend source data is available; otherwise represent it explicitly as unavailable. +- Add target utilization to result-level metrics when the required cycle data is available. +- Add peak activation memory to result-level metrics when a source value is available, or as an unavailable metric entry otherwise. +- Add average memory footprint to result-level metrics when a source value is available, or as an unavailable metric entry otherwise. +- Preserve existing compatibility, analysis metadata, and single-inference latency output while adding the missing fields. +- Explicitly represent requested fields without backend values as unavailable, without fabricating power, energy, hardware-measured runtime, or batch metrics. +- Define missing-data behavior for each new field. +- Update tests and sample or representative JSON payloads for the new fields. + +## Out of Scope + +- Do not add measured values for power, energy, hardware-measured runtime data, or batch latency and throughput in this change unless a backend provides suitable source data. +- Do not implement the Vela/Corstone "all stats" work here; that belongs in the owning plugin repository. +- Do not fix plugin-specific output bugs here except where a neutral fixture is needed to prove the core output contract. +- Do not include plugin-owned integration tasks in the initial core `mlia` PR; those should be specified in the owning plugin repositories once the core contract is stable. +- Do not replace MLIA's standardized output schema with a consumer-specific report schema. + +## Capabilities + +### New Capabilities + +- `standard-performance-metrics`: Adds the supported standardized performance fields to MLIA standardized JSON output. + +### Modified Capabilities + +- None. + +## Impact + +- Core standardized output model, schema helpers, and JSON validation paths. +- JSON reporting and Python API collection paths that return standardized output. +- Tests for each added field, missing-data behavior, schema validation, and one representative output payload. +- Plugin follow-up work may be needed where a backend owns the source value extraction. diff --git a/openspec/changes/add-standard-performance-metrics/proposal.md.license b/openspec/changes/add-standard-performance-metrics/proposal.md.license new file mode 100644 index 0000000..a22d9ae --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/proposal.md.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright 2026, Arm Limited and/or its affiliates. +SPDX-License-Identifier: Apache-2.0 diff --git a/openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md b/openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md new file mode 100644 index 0000000..5372315 --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md @@ -0,0 +1,231 @@ +## ADDED Requirements + +### Requirement: Standardized performance fields + +MLIA SHALL add the supported standardized performance fields to standardized JSON output using MLIA's standardized output schema. + +#### Scenario: Existing supported fields preserved + +- **WHEN** MLIA emits standardized performance output +- **THEN** existing compatibility, analysis metadata, and single-inference latency output remain available through MLIA's standardized output schema. + +#### Scenario: Existing latency mapping documented + +- **WHEN** MLIA documents the standardized performance metric set +- **THEN** existing backend-provided latency metrics are documented as a partial mapping unless a separate standard latency name and unit are defined. + +#### Scenario: Existing compatibility mapping documented + +- **WHEN** MLIA documents the standardized performance metric set +- **THEN** existing compatibility output is documented as a richer MLIA-native structure with status, check, and entity output. + +#### Scenario: Existing analyzer version mapping documented + +- **WHEN** MLIA documents the standardized performance metric set +- **THEN** `tool.version` and `backends[*].version` remain separate in MLIA standardized output. + +#### Scenario: Existing target mapping documented + +- **WHEN** MLIA documents the standardized performance metric set +- **THEN** MLIA's structured `target` output remains available in MLIA standardized output. + +#### Scenario: Existing evaluation type mapping documented + +- **WHEN** MLIA documents the standardized performance metric set +- **THEN** MLIA `results[*].mode` remains available without requiring it to match an external consumer enum. + +#### Scenario: Interpretation notes emitted + +- **WHEN** a result has known limitations or notes needed to interpret performance data +- **THEN** MLIA includes those notes in standardized JSON output using `results[*].warnings`. + +#### Scenario: Accelerator operator percentage emitted + +- **WHEN** MLIA can determine the percentage of operators executed on the accelerator from compatibility or operator-placement data +- **THEN** MLIA emits `accelerator_operator_percentage` as a result-level metric under `results[*].metrics` with a `0..100` value and unit `%`. + +#### Scenario: Inference throughput emitted + +- **WHEN** a backend or plugin can provide inference throughput from suitable source data +- **THEN** MLIA emits `inferences_per_second` as a result-level metric under `results[*].metrics` with unit `inferences/s`. + +#### Scenario: Target utilization emitted + +- **WHEN** a backend or plugin can provide target utilization from suitable source data +- **THEN** MLIA emits `target_utilization` as a result-level metric under `results[*].metrics` with a `0..100` value and unit `%`, calculated as `(compute_cycles / total_cycles) * 100 if total_cycles else 0.0`. + +#### Scenario: CPU utilization emitted + +- **WHEN** a backend or plugin can provide CPU utilization from suitable source data +- **THEN** MLIA emits `cpu_utilization` as a result-level metric under `results[*].metrics` with a `0..100` value and unit `%`. + +#### Scenario: CPU utilization unavailable + +- **WHEN** MLIA or the selected backend cannot provide CPU utilization source data +- **THEN** MLIA emits `cpu_utilization` as an availability-aware metric entry by default with `availability` set to `unavailable`, unit `%`, and a reason. + +### Requirement: Standard metric mapping + +MLIA SHALL define stable MLIA standardized-output names and units for string-keyed metrics in the standardized performance metric set. + +#### Scenario: MLIA metric names are documented + +- **WHEN** MLIA documents the standardized performance metric set +- **THEN** the mapping records the MLIA standardized JSON metric name, unit, and source ownership. + +#### Scenario: Existing metric satisfies requirement + +- **WHEN** an existing MLIA metric already satisfies a standardized performance requirement +- **THEN** MLIA documents that existing metric name and unit in the mapping rather than adding a duplicate metric name without justification. + +#### Scenario: New metric is added + +- **WHEN** MLIA adds a new metric for the standardized performance metric set +- **THEN** the metric uses a generic MLIA standardized-output name and unit. + +#### Scenario: Existing metric overlaps requirement + +- **WHEN** an existing MLIA metric overlaps a standardized performance field +- **THEN** MLIA decides case by case whether to keep the existing name, rename it, or add an alias rather than renaming existing output blindly. + +#### Scenario: Percentage metric unit convention + +- **WHEN** MLIA emits a percentage metric in the standardized performance field set +- **THEN** the metric value uses the range `0..100` and unit `%`. + +#### Scenario: Peak activation memory emitted + +- **WHEN** peak activation memory is available for a single inference run +- **THEN** MLIA emits `peak_activation_memory` as a result-level metric under `results[*].metrics` with unit `bytes`. + +#### Scenario: Peak activation memory unavailable + +- **WHEN** MLIA or the selected backend cannot provide supported peak activation memory source data +- **THEN** MLIA emits `peak_activation_memory` as an availability-aware metric entry with unit `bytes` and a reason. + +#### Scenario: Average memory footprint emitted + +- **WHEN** average memory usage over the full measurement window is available for a single inference run +- **THEN** MLIA emits `average_memory` as a result-level metric under `results[*].metrics` with unit `bytes`. + +#### Scenario: Average memory footprint unavailable + +- **WHEN** MLIA or the selected backend cannot provide supported average memory source data +- **THEN** MLIA emits `average_memory` as an availability-aware metric entry with unit `bytes` and a reason. + +### Requirement: Missing data behavior + +MLIA SHALL define and test missing-data behavior for each added field. + +#### Scenario: Source data is missing + +- **WHEN** MLIA or the selected backend cannot provide the source data for an added field +- **THEN** MLIA does not fabricate a value and represents the missing-data state as an availability-aware metric entry. + +#### Scenario: Added metric field is always represented + +- **WHEN** MLIA emits standardized output for a result covered by this change +- **THEN** each added or standardized metric field is present in `results[*].metrics` either as a numeric metric or as an availability-aware metric entry. + +#### Scenario: Core helper fills unavailable metric entries + +- **WHEN** a plugin passes performance result metrics through the core helper for added standard performance fields +- **THEN** supplied numeric values are preserved and missing added standard metrics are filled as availability-aware metric entries. + +#### Scenario: Core helper is called explicitly + +- **WHEN** a performance result is not passed through the core helper +- **THEN** core reporting does not automatically add availability-aware metric entries. + +#### Scenario: Plugin integration is follow-up work + +- **WHEN** the initial core `mlia` PR is implemented +- **THEN** it defines the shared output contract and helper behavior without including plugin-owned integration tasks. + +#### Scenario: Representative plugin integration uses Vela + +- **WHEN** the initial implementation proves plugin integration +- **THEN** it uses the Ethos-U Vela performance path as the representative plugin call site. + +#### Scenario: Vela integration has plugin OpenSpec + +- **WHEN** Vela helper integration is planned +- **THEN** the plugin-owned behavior is specified in a small `mlia-ethos-u` OpenSpec change. + +#### Scenario: Vela representative payload preserves result metrics + +- **WHEN** the Vela representative integration emits performance output +- **THEN** result-level metrics are preserved and are not overwritten by breakdown metric construction. + +#### Scenario: Metric value is available + +- **WHEN** MLIA emits a supported metric value +- **THEN** the metric keeps the existing numeric shape with `name`, `value`, and `unit`, and omitted `availability` means available. + +#### Scenario: Metric value has no numeric value + +- **WHEN** MLIA emits a standardized performance metric without a numeric value +- **THEN** MLIA emits a metric entry with `availability` set to `unavailable`, `unit`, and reason, without a fake numeric `value`. + +#### Scenario: Metric array supports both metric shapes + +- **WHEN** MLIA validates `results[*].metrics` +- **THEN** the schema accepts numeric metric entries and non-value availability entries in the same metrics array. + +#### Scenario: Additional non-value states are deferred + +- **WHEN** implementation needs to distinguish metrics that are unsupported from metrics that are supported in principle but unavailable for a particular run +- **THEN** MLIA adds a separate availability state in a later change rather than requiring it in the initial implementation. + +### Requirement: Non-value fields are explicit + +MLIA SHALL explicitly represent standardized performance fields that do not have numeric values without fabricating metric values. + +#### Scenario: Non-value field is in the standardized performance field set + +- **WHEN** a metric field added or standardized by this change cannot be produced by MLIA or the selected backend +- **THEN** MLIA marks that field using an availability-aware metric entry rather than omitting it silently or emitting a placeholder numeric value. + +#### Scenario: Ticket-covered optional design input field is unavailable + +- **WHEN** `accelerator_operator_percentage` cannot be produced by MLIA or the selected backend +- **THEN** MLIA marks that field using an availability-aware metric entry even though the corresponding design-input row is not marked required. + +#### Scenario: Existing mapped fields do not get metric placeholders + +- **WHEN** an existing metadata, compatibility, or latency field only maps partially to the design input +- **THEN** MLIA keeps the field in its normal standardized output location and does not emit a placeholder metric entry for that mapping gap. + +#### Scenario: Field is outside the standardized performance field set + +- **WHEN** a field is not part of the standardized performance field set for this work +- **THEN** MLIA does not need to represent that field as part of this change. + +#### Scenario: Optional memory-profile stats belong to later plugin work + +- **WHEN** a field such as `memoryProfile.modelWeightMemory` belongs to Vela/Corstone statistics-completeness work +- **THEN** MLIA does not need to represent that field as part of this core standard-fields change. + +#### Scenario: Non-value marker scope is documented + +- **WHEN** MLIA documents the added standardized performance fields +- **THEN** the documentation states that non-value markers are limited to the standardized performance field set for this change and are not comprehensive for every possible consumer field. + +#### Scenario: Availability-aware metric contract is documented + +- **WHEN** MLIA documents the added standardized performance fields +- **THEN** the documentation explains that metrics may be numeric values or explicit availability entries. + +### Requirement: Existing reporting remains valid + +MLIA SHALL add the requested fields without regressing existing standardized JSON reporting or Python API collection workflows. + +#### Scenario: Schema version identifies availability-aware metrics + +- **WHEN** MLIA emits standardized output with availability-aware metric entries +- **THEN** the output uses MLIA output schema version `1.1.0`. + +#### Scenario: Standardized output validates + +- **WHEN** MLIA emits standardized output containing the added fields +- **THEN** the output validates against MLIA's standardized output schema. diff --git a/openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md.license b/openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md.license new file mode 100644 index 0000000..a22d9ae --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/specs/standard-performance-metrics/spec.md.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright 2026, Arm Limited and/or its affiliates. +SPDX-License-Identifier: Apache-2.0 diff --git a/openspec/changes/add-standard-performance-metrics/tasks.md b/openspec/changes/add-standard-performance-metrics/tasks.md new file mode 100644 index 0000000..969b855 --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/tasks.md @@ -0,0 +1,75 @@ +## 1. Output Placement + +- [x] 1.1 Confirm `results[*].warnings` as the schema-valid location for known limitations and interpretation notes. +- [x] 1.2 Confirm that accelerator operator percentage is a result-level metric under `results[*].metrics`. +- [x] 1.3 Confirm that inference throughput, CPU utilization, target utilization, peak activation memory, and average memory footprint are result-level metrics under `results[*].metrics`. +- [x] 1.4 Update the metric schema shape so `results[*].metrics` can contain supported numeric metrics and explicit non-value metric entries. +- [x] 1.5 Model numeric metric entries and non-value availability entries as two valid shapes in the same `results[*].metrics` array. + +## 2. Field Semantics + +- [x] 2.1 Define the metric-by-metric mapping for the standardized performance metric set, including existing latency and throughput-like metrics. +- [x] 2.2 Define generic MLIA standardized metric names and units for any newly added metrics. +- [x] 2.3 Record MLIA standardized JSON metric names, units, and source ownership in the mapping. +- [x] 2.4 Prefer generic MLIA names for genuinely new metric names. +- [x] 2.5 Document existing backend-provided latency metrics as a partial mapping rather than adding a new standard latency metric in this change. +- [x] 2.6 Document existing compatibility output as MLIA-native status, check, and entity output. +- [x] 2.7 Document `tool.version` and `backends[*].version` as separate MLIA standardized output fields. +- [x] 2.8 Document MLIA's structured `target` output. +- [x] 2.9 Document MLIA `results[*].mode` without requiring it to match an external enum. +- [x] 2.10 Define inference throughput as `inferences_per_second` with unit `inferences/s`, a backend/plugin-provided result-level metric derived from suitable latency data where applicable. +- [x] 2.11 Define CPU utilization as `cpu_utilization` with unit `%`, emitted as `unavailable` when no backend source can provide a real value. +- [x] 2.12 Define target utilization as `target_utilization`, calculated as `(compute_cycles / total_cycles) * 100 if total_cycles else 0.0` from suitable cycle data. +- [x] 2.13 Define peak activation memory as `peak_activation_memory`, meaning the highest activation memory usage during a single inference run. +- [x] 2.14 Define average memory as `average_memory`, meaning average memory usage over the full measurement window rather than peak, allocation total, or instantaneous memory. +- [x] 2.15 Define accelerator operator percentage as an operator-placement metric, not a measured runtime percentage. +- [x] 2.16 Define percentage metrics as `0..100` values with unit `%`. +- [x] 2.17 Define memory metrics as using unit `bytes`. +- [x] 2.18 Define missing-data behavior for each new field. +- [x] 2.19 Define the added or standardized metric field set that must always be represented as numeric metrics or explicit non-value markers, including `accelerator_operator_percentage`. +- [x] 2.20 Define non-value field behavior for those metric fields without expanding scope to existing metadata, compatibility, latency mappings, or every possible consumer field. +- [x] 2.21 Use `unavailable` as the only initial non-value availability state. +- [x] 2.22 Defer an `unsupported` state unless implementation evidence shows a separate state is needed. +- [x] 2.23 Define the required and optional fields for non-numeric metric availability entries; `unit` and `reason` are required for this contract unless implementation exposes a real exception. +- [x] 2.24 Document that supported metrics may omit `availability`, and omitted availability means available when `value` is present. + +## 3. Core Implementation + +- [x] 3.1 Add the chosen `results[*].warnings` output representation for known limitations and interpretation notes. +- [x] 3.2 Add or document `accelerator_operator_percentage` as the result-level metric name for percentage of operators executed on the accelerator. +- [x] 3.3 Add or document `cpu_utilization` as the result-level metric name for CPU utilization, including unavailable output when no backend source is available. +- [x] 3.4 Add or document `target_utilization` as the result-level metric name for target utilization. +- [x] 3.5 Add or document `inferences_per_second` as the result-level metric name for inference throughput. +- [x] 3.6 Add or document `average_memory` as the result-level metric name for average memory footprint. +- [x] 3.7 Add or document `peak_activation_memory` as the result-level metric name for peak activation memory. +- [x] 3.8 Add shared metric name and unit constants where the implementation introduces or standardizes string-keyed metric names. +- [x] 3.9 Add a core helper that preserves supplied numeric values and fills missing added standard metrics as availability-aware entries. +- [x] 3.10 Document that plugins should call the core helper explicitly when constructing performance results that should include the added standard metrics. +- [x] 3.11 Ensure JSON reporting and Python API collection preserve the added fields. +- [x] 3.12 Preserve existing compatibility, analysis metadata, and single-inference latency output. +- [x] 3.13 Document that non-value markers are limited to the standardized performance field set for this change. +- [x] 3.14 Bump the MLIA output schema version from `1.0.0` to `1.1.0` for availability-aware metric entries. +- [x] 3.15 Add compatibility documentation explaining that `1.1.0` metrics may be numeric values or non-value availability entries. + +## 4. Validation + +- [x] 4.1 Add unit tests for each new field. +- [x] 4.2 Add tests for missing-data behavior for each new field. +- [x] 4.3 Add schema validation coverage for representative standardized output. +- [x] 4.4 Check whether maintained sample JSON or representative fixtures need + updating. No canonical committed sample JSON artifact was identified for this + slice, so representative JSON shape is covered by tests, schema validation, + and documentation instead of adding a large generated output file. +- [x] 4.5 Validate the Ethos-U Vela performance path as the representative plugin integration where source data is available. +- [x] 4.6 Add coverage for non-value metric representation without fabricated values. +- [x] 4.7 Check user-facing documentation or release notes describe the non-value marker scope limitation. + +## 5. Plugin Follow-Up Coordination + +- [x] 5.1 Record any backend-specific extraction work needed to populate the new fields. +- [x] 5.2 Create a small `mlia-ethos-u` OpenSpec change for the Vela representative integration. +- [x] 5.3 Record the Vela result-level metrics overwrite issue as plugin-owned and fix it in the Vela integration branch because it blocks a trustworthy representative payload. +- [x] 5.4 Record Vela/Corstone all-stats completeness as separate future plugin-owned work. +- [x] 5.5 Keep plugin-owned integration tasks out of the initial core `mlia` PR; specify them in the owning plugin repositories once the core contract is stable. +- [x] 5.6 Track any private plugin integration in the owning private repository rather than in this public core PR. +- [x] 5.7 Track the plugin call-site changes in the owning plugin repositories rather than in this core PR. diff --git a/openspec/changes/add-standard-performance-metrics/tasks.md.license b/openspec/changes/add-standard-performance-metrics/tasks.md.license new file mode 100644 index 0000000..a22d9ae --- /dev/null +++ b/openspec/changes/add-standard-performance-metrics/tasks.md.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright 2026, Arm Limited and/or its affiliates. +SPDX-License-Identifier: Apache-2.0 diff --git a/pyproject.toml b/pyproject.toml index d26f7e5..6be9f97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,7 @@ test = [ "pytest~=9.0", "pytest-cov~=7.0", "flaky~=3.8.1", + "jsonschema~=4.26", "hatchling>=1.27.0" ] dev = [ diff --git a/src/mlia/core/output_schema.py b/src/mlia/core/output_schema.py index 4a30542..8ebdc44 100644 --- a/src/mlia/core/output_schema.py +++ b/src/mlia/core/output_schema.py @@ -13,7 +13,7 @@ from uuid import uuid4 # Schema version for standardized output -SCHEMA_VERSION = "1.0.0" +SCHEMA_VERSION = "1.1.0" # Target schema version TARGET_SCHEMA_VERSION = "1.0.0" @@ -97,6 +97,66 @@ class AdviceSeverity(str, Enum): ERROR = "error" +class MetricAvailability(str, Enum): + """Metric availability enumeration.""" + + UNAVAILABLE = "unavailable" + + +@dataclass(frozen=True) +class StandardPerformanceMetric: + """Definition for a standardized performance metric.""" + + name: str + unit: str + unavailable_reason: str + + +METRIC_NAME_ACCELERATOR_OPERATOR_PERCENTAGE = "accelerator_operator_percentage" +METRIC_NAME_INFERENCES_PER_SECOND = "inferences_per_second" +METRIC_NAME_CPU_UTILIZATION = "cpu_utilization" +METRIC_NAME_TARGET_UTILIZATION = "target_utilization" +METRIC_NAME_PEAK_ACTIVATION_MEMORY = "peak_activation_memory" +METRIC_NAME_AVERAGE_MEMORY = "average_memory" + +UNIT_PERCENT = "%" +UNIT_INFERENCES_PER_SECOND = "inferences/s" +UNIT_BYTES = "bytes" + +STANDARD_PERFORMANCE_METRICS = ( + StandardPerformanceMetric( + name=METRIC_NAME_ACCELERATOR_OPERATOR_PERCENTAGE, + unit=UNIT_PERCENT, + unavailable_reason="Accelerator operator placement data is not available.", + ), + StandardPerformanceMetric( + name=METRIC_NAME_INFERENCES_PER_SECOND, + unit=UNIT_INFERENCES_PER_SECOND, + unavailable_reason="Inference throughput data is not available.", + ), + StandardPerformanceMetric( + name=METRIC_NAME_CPU_UTILIZATION, + unit=UNIT_PERCENT, + unavailable_reason="CPU utilization data is not available.", + ), + StandardPerformanceMetric( + name=METRIC_NAME_TARGET_UTILIZATION, + unit=UNIT_PERCENT, + unavailable_reason="Target utilization data is not available.", + ), + StandardPerformanceMetric( + name=METRIC_NAME_PEAK_ACTIVATION_MEMORY, + unit=UNIT_BYTES, + unavailable_reason="Peak activation memory data is not available.", + ), + StandardPerformanceMetric( + name=METRIC_NAME_AVERAGE_MEMORY, + unit=UNIT_BYTES, + unavailable_reason="Average memory data is not available.", + ), +) + + @dataclass(frozen=True) class Tool: """Tool information.""" @@ -288,21 +348,49 @@ class Metric: """Metric information.""" name: str - value: float + value: float | int | None unit: str aggregation: str | None = None samples: int | None = None qualifiers: dict[str, Any] = field(default_factory=dict) + availability: MetricAvailability | None = None + reason: str | None = None + + def __post_init__(self) -> None: + """Validate that the metric matches one of the supported schema shapes.""" + if self.value is not None: + if self.availability is not None or self.reason is not None: + raise ValueError( + "Metric cannot combine a numeric value with availability fields." + ) + return + + if self.availability is None: + raise ValueError("Metric must include a value or availability.") + + if not self.reason: + raise ValueError("Unavailable metrics must include a reason.") + + if self.aggregation is not None or self.samples is not None: + raise ValueError( + "Unavailable metrics cannot include aggregation or samples." + ) def to_dict(self) -> dict[str, Any]: """Convert to dictionary.""" - result = {"name": self.name, "value": self.value, "unit": self.unit} + result: dict[str, Any] = {"name": self.name, "unit": self.unit} + if self.value is not None: + result["value"] = self.value if self.aggregation is not None: result["aggregation"] = self.aggregation if self.samples is not None: result["samples"] = self.samples if self.qualifiers: result["qualifiers"] = self.qualifiers + if self.availability is not None: + result["availability"] = self.availability.value + if self.reason is not None: + result["reason"] = self.reason return result @classmethod @@ -310,12 +398,40 @@ def from_dict(cls, data: dict[str, Any]) -> Metric: """Create from dictionary.""" return cls( name=data["name"], - value=data["value"], + value=data.get("value"), unit=data["unit"], aggregation=data.get("aggregation"), samples=data.get("samples"), qualifiers=data.get("qualifiers", {}), + availability=( + MetricAvailability(data["availability"]) + if "availability" in data + else None + ), + reason=data.get("reason"), + ) + + +def ensure_standard_performance_metrics(metrics: list[Metric]) -> list[Metric]: + """Add unavailable entries for standard performance metrics. + + Plugin performance collectors should add the standard metrics they can + report, then call this helper to include unavailable entries for the + remaining standard metrics. + """ + existing_names = {metric.name for metric in metrics} + missing_metrics = [ + Metric( + name=definition.name, + value=None, + unit=definition.unit, + availability=MetricAvailability.UNAVAILABLE, + reason=definition.unavailable_reason, ) + for definition in STANDARD_PERFORMANCE_METRICS + if definition.name not in existing_names + ] + return [*metrics, *missing_metrics] @dataclass(frozen=True) diff --git a/src/mlia/core/output_validation.py b/src/mlia/core/output_validation.py index e4b168a..b789e13 100644 --- a/src/mlia/core/output_validation.py +++ b/src/mlia/core/output_validation.py @@ -42,7 +42,7 @@ def load_target_schema() -> dict[str, Any]: schema_path = ( Path(__file__).parent.parent / "resources" - / f"mlia-target-schema-{schema.SCHEMA_VERSION}.json" + / f"mlia-target-schema-{schema.TARGET_SCHEMA_VERSION}.json" ) if not schema_path.exists(): raise FileNotFoundError(f"Schema file not found: {schema_path}") diff --git a/src/mlia/resources/mlia-output-schema-1.1.0.json b/src/mlia/resources/mlia-output-schema-1.1.0.json new file mode 100644 index 0000000..5e22af5 --- /dev/null +++ b/src/mlia/resources/mlia-output-schema-1.1.0.json @@ -0,0 +1,260 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.arm.com/mlia/output-schema-1.1.0.json", + "title": "MLIA Unified Output Schema (v1.1.0)", + "type": "object", + "required": [ + "schema_version", + "run_id", + "timestamp", + "tool", + "target", + "model", + "context", + "backends", + "results" + ], + "properties": { + "schema_version": { + "type": "string", + "const": "1.1.0" + }, + "run_id": { "type": "string", "format": "uuid" }, + "timestamp": { "type": "string", "format": "date-time" }, + + "tool": { + "type": "object", + "required": ["name", "version"], + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" } + }, + "additionalProperties": false + }, + + "backends": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "name", "version", "configuration"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "version": { "type": "string" }, + "impl": { "type": "object" }, + "configuration": { + "type": "object", + "description": "Backend configuration or options." + } + }, + "additionalProperties": false + } + }, + + "target": { + "$ref": "https://schemas.arm.com/mlia/target-1.0.0.json" + }, + + "model": { + "type": "object", + "required": ["name", "format", "hash"], + "properties": { + "name": { "type": "string" }, + "format": { + "type": "string", + "description": "Model file format", + "examples": ["tflite", "onnx", "vgf", "tosa"] + }, + "hash": { + "type": "string", + "pattern": "^[A-Fa-f0-9]{64}$", + "description": "SHA-256 hash of the model file." + }, + "size_bytes": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false + }, + + "context": { + "type": "object", + "properties": { + "runtime_configuration": { "type": "object" }, + "git": { + "type": "object", + "properties": { "mlia_commit": { "type": "string" } }, + "additionalProperties": false + }, + "notes": { "type": "string" }, + "cli_arguments": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + }, + + "results": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "status", "producer"], + "properties": { + "kind": { "enum": ["compatibility", "performance"] }, + "producer": { "type": "string" }, + "status": { + "type": "string", + "enum": ["ok", "partial", "incompatible", "failed", "skipped"] + }, + "warnings": { "type": "array", "items": { "type": "string" } }, + "errors": { "type": "array", "items": { "type": "string" } }, + "metrics": { "type": "array", "items": { "$ref": "#/$defs/metric" } }, + "breakdowns": { "type": "array", "items": { "$ref": "#/$defs/breakdown" } }, + "mode": { + "type": "string", + "enum": ["measured", "simulated", "predicted"] + }, + "checks": { "type": "array", "items": { "$ref": "#/$defs/check" } }, + "entities": { "type": "array", "items": { "$ref": "#/$defs/entity" } } + }, + "additionalProperties": false + } + }, + + "extensions": { "type": "object", "additionalProperties": true } + }, + + "$defs": { + "operator_identifier": { + "type": "object", + "required": ["scope", "name", "location"], + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for this operator instance." + }, + "scope": { + "enum": ["operator", "operator_chain"], + "description": "Granularity level." + }, + "name": { "type": "string" }, + "location": { "type": "string" } + }, + "additionalProperties": false + }, + + "qualifiers_common": { + "type": "object", + "description": "Lightweight context qualifiers; only shared keys constrained.", + "properties": { + "placement": { + "type": "string", + "enum": ["NPU", "NX", "CPU", "GPU", "DSP", "Unknown"] + }, + "phase": { + "type": "string", + "enum": ["before", "after"] + } + }, + "patternProperties": { + "^[a-zA-Z_][a-zA-Z0-9_\\-]*$": { + "type": ["string", "number", "boolean", "array", "object"] + } + }, + "additionalProperties": true + }, + + "metric": { + "oneOf": [ + { "$ref": "#/$defs/numeric_metric" }, + { "$ref": "#/$defs/unavailable_metric" } + ] + }, + + "numeric_metric": { + "type": "object", + "required": ["name", "value", "unit"], + "properties": { + "name": { "type": "string" }, + "value": { "type": "number" }, + "unit": { "type": "string" }, + "aggregation": { "type": "string" }, + "samples": { "type": "integer" }, + "qualifiers": { "$ref": "#/$defs/qualifiers_common" } + }, + "additionalProperties": false + }, + + "unavailable_metric": { + "type": "object", + "required": ["name", "unit", "availability", "reason"], + "properties": { + "name": { "type": "string" }, + "unit": { "type": "string" }, + "availability": { "const": "unavailable" }, + "reason": { "type": "string", "minLength": 1 }, + "qualifiers": { "$ref": "#/$defs/qualifiers_common" } + }, + "additionalProperties": false + }, + + "breakdown": { + "type": "object", + "required": ["scope", "name", "location", "metrics"], + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for this operator instance." + }, + "scope": { + "enum": ["operator", "operator_chain"], + "description": "Granularity level." + }, + "name": { "type": "string" }, + "location": { "type": "string" }, + "metrics": { + "type": "array", + "items": { "$ref": "#/$defs/metric" } + }, + "qualifiers": { "$ref": "#/$defs/qualifiers_common" } + }, + "additionalProperties": false + }, + + "check": { + "type": "object", + "required": ["id", "status"], + "properties": { + "id": { "type": "string" }, + "status": { "enum": ["pass", "fail", "partial"] }, + "details": { "type": "object" } + }, + "additionalProperties": false + }, + + "entity": { + "type": "object", + "required": ["scope", "name", "location", "placement"], + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for this operator instance." + }, + "scope": { + "enum": ["operator", "operator_chain"], + "description": "Granularity level." + }, + "name": { "type": "string" }, + "location": { "type": "string" }, + "placement": { + "type": "string", + "description": "Execution placement." + }, + "attributes": { "type": "object" } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false +} diff --git a/src/mlia/resources/mlia-output-schema-1.1.0.json.license b/src/mlia/resources/mlia-output-schema-1.1.0.json.license new file mode 100644 index 0000000..a22d9ae --- /dev/null +++ b/src/mlia/resources/mlia-output-schema-1.1.0.json.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: Copyright 2026, Arm Limited and/or its affiliates. +SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_core_output_schema.py b/tests/test_core_output_schema.py index 1654b04..9d447b5 100644 --- a/tests/test_core_output_schema.py +++ b/tests/test_core_output_schema.py @@ -4,6 +4,7 @@ import tempfile from pathlib import Path +from typing import Any import pytest @@ -218,6 +219,23 @@ def test_to_dict(self) -> None: "qualifiers": {"key": "value"}, } + def test_unavailable_to_dict(self) -> None: + """Test conversion of unavailable metric to dictionary.""" + metric = schema.Metric( + name="metric_name", + value=None, + unit="metric_unit", + availability=schema.MetricAvailability.UNAVAILABLE, + reason="Metric is unavailable.", + ) + + assert metric.to_dict() == { + "name": "metric_name", + "unit": "metric_unit", + "availability": "unavailable", + "reason": "Metric is unavailable.", + } + def test_from_dict(self) -> None: """Test creation from dictionary.""" data = {"name": "inference_time", "value": 10.5, "unit": "ms"} @@ -225,6 +243,120 @@ def test_from_dict(self) -> None: assert metric.name == "inference_time" assert metric.value == 10.5 + def test_unavailable_from_dict(self) -> None: + """Test creation of unavailable metric from dictionary.""" + metric = schema.Metric.from_dict( + { + "name": schema.METRIC_NAME_CPU_UTILIZATION, + "unit": schema.UNIT_PERCENT, + "availability": "unavailable", + "reason": "CPU utilization data is not available.", + } + ) + + assert metric.name == schema.METRIC_NAME_CPU_UTILIZATION + assert metric.value is None + assert metric.availability == schema.MetricAvailability.UNAVAILABLE + assert metric.reason == "CPU utilization data is not available." + + @pytest.mark.parametrize( + ("metric_kwargs", "match"), + [ + ( + { + "value": 10.5, + "availability": schema.MetricAvailability.UNAVAILABLE, + "reason": "Metric is unavailable.", + }, + "cannot combine", + ), + ({"value": None}, "must include a value or availability"), + ( + {"value": None, "availability": schema.MetricAvailability.UNAVAILABLE}, + "must include a reason", + ), + ( + { + "value": None, + "availability": schema.MetricAvailability.UNAVAILABLE, + "reason": "Metric is unavailable.", + "aggregation": "sum", + }, + "cannot include aggregation or samples", + ), + ], + ) + def test_metric_init_rejects_invalid_shapes( + self, metric_kwargs: dict[str, Any], match: str + ) -> None: + """Metric construction should reject shapes not allowed by the schema.""" + with pytest.raises(ValueError, match=match): + schema.Metric(name="metric_name", unit="metric_unit", **metric_kwargs) + + +class TestEnsureStandardPerformanceMetrics: + """Test standard performance metric helpers.""" + + def test_ensure_standard_performance_metrics_preserves_supplied_metric( + self, + ) -> None: + """Existing metrics should be preserved.""" + throughput = schema.Metric( + name=schema.METRIC_NAME_INFERENCES_PER_SECOND, + value=42.0, + unit=schema.UNIT_INFERENCES_PER_SECOND, + ) + + metrics = schema.ensure_standard_performance_metrics([throughput]) + + assert metrics[0] == throughput + assert metrics[0].to_dict() == { + "name": schema.METRIC_NAME_INFERENCES_PER_SECOND, + "value": 42.0, + "unit": schema.UNIT_INFERENCES_PER_SECOND, + } + + def test_ensure_standard_performance_metrics_fills_missing_metrics( + self, + ) -> None: + """Missing standard metrics should be represented as unavailable.""" + throughput = schema.Metric( + name=schema.METRIC_NAME_INFERENCES_PER_SECOND, + value=42.0, + unit=schema.UNIT_INFERENCES_PER_SECOND, + ) + + metrics = schema.ensure_standard_performance_metrics([throughput]) + by_name = {metric.name: metric for metric in metrics} + + assert set(by_name) == { + definition.name for definition in schema.STANDARD_PERFORMANCE_METRICS + } + assert by_name[schema.METRIC_NAME_INFERENCES_PER_SECOND].value == 42.0 + unavailable_metric = by_name[schema.METRIC_NAME_CPU_UTILIZATION] + assert unavailable_metric.value is None + assert unavailable_metric.availability == schema.MetricAvailability.UNAVAILABLE + assert unavailable_metric.unit == schema.UNIT_PERCENT + assert unavailable_metric.reason + + def test_ensure_standard_performance_metrics_does_not_mutate_input( + self, + ) -> None: + """Helper should return a new list without mutating caller state.""" + metrics = [ + schema.Metric( + name=schema.METRIC_NAME_INFERENCES_PER_SECOND, + value=42.0, + unit=schema.UNIT_INFERENCES_PER_SECOND, + ) + ] + + filled_metrics = schema.ensure_standard_performance_metrics(metrics) + + assert filled_metrics is not metrics + assert len(metrics) == 1 + assert len(filled_metrics) == len(schema.STANDARD_PERFORMANCE_METRICS) + class TestOperatorIdentifier: """Test OperatorIdentifier class.""" diff --git a/tests/test_core_output_validation.py b/tests/test_core_output_validation.py index f2407e7..dcbddab 100644 --- a/tests/test_core_output_validation.py +++ b/tests/test_core_output_validation.py @@ -11,7 +11,9 @@ import pytest +import mlia.core.output_schema as schema from mlia.core.output_validation import ( + JSONSCHEMA_AVAILABLE, SchemaValidationError, _build_schema_registry, collect_validation_errors, @@ -49,6 +51,50 @@ } +def _valid_standardized_output(metric: dict[str, Any]) -> dict[str, Any]: + """Build a valid standardized output payload containing one metric.""" + return { + "schema_version": schema.SCHEMA_VERSION, + "run_id": "550e8400-e29b-41d4-a716-446655440000", + "timestamp": "2025-12-29T10:30:00Z", + "tool": {"name": "MLIA", "version": "1.0.0"}, + "target": { + "profile_name": "ethos-u55-256", + "target_type": "corstone-300", + "components": [{"type": "npu", "family": "ethos-u"}], + "configuration": {}, + }, + "model": { + "name": "model.tflite", + "format": "tflite", + "hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + "context": {}, + "backends": [ + {"id": "vela", "name": "vela", "version": "3.0.0", "configuration": {}} + ], + "results": [ + { + "kind": "performance", + "status": "ok", + "producer": "vela", + "metrics": [metric], + } + ], + } + + +def _valid_standardized_output_with_result( + result: dict[str, Any], +) -> dict[str, Any]: + """Build a valid standardized output payload containing one result.""" + output = _valid_standardized_output( + {"name": "inference_time", "value": 1.0, "unit": "ms"} + ) + output["results"] = [result] + return output + + def test_load_schema(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test load_schema function.""" @@ -183,6 +229,93 @@ def with_resources( assert registry["resources"][1][0] == "target-schema" +def test_loaded_schema_requires_current_schema_version() -> None: + """The loaded JSON Schema should require the current schema version.""" + loaded_schema = load_schema() + + assert loaded_schema["properties"]["schema_version"] == { + "type": "string", + "const": schema.SCHEMA_VERSION, + } + + +@pytest.mark.skipif(not JSONSCHEMA_AVAILABLE, reason="jsonschema is not available") +def test_jsonschema_accepts_unavailable_metric_entry() -> None: + """Schema validation should accept unavailable metric entries.""" + validate_standardized_output( + _valid_standardized_output( + { + "name": schema.METRIC_NAME_CPU_UTILIZATION, + "unit": schema.UNIT_PERCENT, + "availability": "unavailable", + "reason": "CPU utilization data is not available.", + } + ) + ) + + +@pytest.mark.skipif(not JSONSCHEMA_AVAILABLE, reason="jsonschema is not available") +def test_jsonschema_rejects_mismatched_schema_version() -> None: + """Schema 1.1.0 should reject payloads that claim another schema version.""" + output = _valid_standardized_output( + {"name": "inference_time", "value": 1.0, "unit": "ms"} + ) + output["schema_version"] = "1.0.0" + + with pytest.raises(SchemaValidationError, match="Schema validation failed"): + validate_standardized_output(output) + + +@pytest.mark.skipif(not JSONSCHEMA_AVAILABLE, reason="jsonschema is not available") +def test_jsonschema_rejects_unavailable_metric_with_fake_value() -> None: + """Unavailable metric entries should not contain fabricated values.""" + with pytest.raises(SchemaValidationError, match="Schema validation failed"): + validate_standardized_output( + _valid_standardized_output( + { + "name": schema.METRIC_NAME_CPU_UTILIZATION, + "value": 0.0, + "unit": schema.UNIT_PERCENT, + "availability": "unavailable", + "reason": "CPU utilization data is not available.", + } + ) + ) + + +@pytest.mark.skipif(not JSONSCHEMA_AVAILABLE, reason="jsonschema is not available") +def test_jsonschema_accepts_result_with_breakdown_and_entity() -> None: + """Schema validation should accept breakdowns and entities.""" + validate_standardized_output( + _valid_standardized_output_with_result( + { + "kind": "performance", + "status": "ok", + "producer": "backend", + "metrics": [{"name": "inference_time", "value": 1.0, "unit": "ms"}], + "breakdowns": [ + { + "scope": "operator", + "name": "CONV_2D", + "location": "model/conv", + "metrics": [ + {"name": "npu_cycles", "value": 1000, "unit": "cycles"} + ], + } + ], + "entities": [ + { + "scope": "operator", + "name": "CONV_2D", + "location": "model/conv", + "placement": "NPU", + } + ], + } + ) + ) + + def test_validate_with_jsonschema_raises_on_collected_errors( monkeypatch: pytest.MonkeyPatch, ) -> None: