diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d466455..f4b978f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,7 +10,8 @@ ## New Features - +* Added an `--inplace` flag to the `generate-config` CLI command: patches `--default` directly instead of printing to stdout, only filling in values it's missing so existing comments, field order and formatting survive untouched. +* `generate-config` now renders whole-number values (e.g. peak/rated power) as underscore-grouped ints (`1_736_680`) instead of floats (`1736680.0`), avoiding spurious diffs. ## Bug Fixes diff --git a/src/frequenz/gridpool/cli/__main__.py b/src/frequenz/gridpool/cli/__main__.py index a631dee..dee11c9 100644 --- a/src/frequenz/gridpool/cli/__main__.py +++ b/src/frequenz/gridpool/cli/__main__.py @@ -4,14 +4,16 @@ """CLI tool for gridpool functionality.""" import os +import tempfile from pathlib import Path import asyncclick as click from frequenz.client.assets import AssetsApiClient from frequenz.client.common.microgrid import MicrogridId -from frequenz.gridpool import ComponentGraphGenerator, load_configs +from frequenz.gridpool import ComponentGraphGenerator, MicrogridConfig, load_configs from frequenz.gridpool.cli._dump_config import dump_map +from frequenz.gridpool.cli._patch_config import patch_file from frequenz.gridpool.cli._render_graph import ComponentGraphRenderer, RenderOptions @@ -123,23 +125,39 @@ async def render_graph(microgrid_id: int, output: str, show: bool) -> None: default=None, help="Config file whose values override the Assets API (highest precedence).", ) +@click.option( + "--inplace", + is_flag=True, + default=False, + help="Patch --default in place instead of printing to stdout. Preserves " + "existing comments, ordering and formatting in that file; only fills in " + "values it is missing. Requires --default.", +) async def generate_config( microgrid_ids: tuple[int, ...], default_file: Path | None, override_file: Path | None, + inplace: bool, ) -> None: """Generate microgrid config from the Assets API as TOML. Derives metadata, formulas and component IDs for the given microgrid IDs and prints the result as dotted-key TOML to stdout. - `--default` and `--override` each take a config file and are layered with the - Assets API by precedence: `--default` < Assets API < `--override`. So a file - passed as `--override` keeps its values where it has them (the API only fills - gaps), while a file passed as `--default` is overridden by the API. If no - microgrid IDs are given, they are taken from the supplied files. Files are - only read; redirect stdout to save the result. + `--default` and `--override` each take a config file, layered with the Assets + API by precedence: `--default` < Assets API < `--override`. If no microgrid + IDs are given, they are taken from the supplied files. Files are only read; + redirect stdout to save the result. + + With `--inplace`, `--default` is patched directly instead: candidate values + come from the Assets API (with `--override` layered on top), and only + leaves `--default` is missing are added, preserving its existing comments, + field order and formatting. If no microgrid IDs are given, every microgrid + already in `--default` is processed. """ + if inplace and default_file is None: + raise click.ClickException("--inplace requires --default.") + url = os.environ.get("ASSETS_API_URL") key = os.environ.get("ASSETS_API_AUTH_KEY") secret = os.environ.get("ASSETS_API_SIGN_SECRET") @@ -148,18 +166,46 @@ async def generate_config( "ASSETS_API_URL, ASSETS_API_AUTH_KEY, ASSETS_API_SIGN_SECRET must be set." ) + ids = list(dict.fromkeys(microgrid_ids)) or None + if inplace and ids is None: + assert default_file is not None + ids = sorted(int(mid) for mid in MicrogridConfig.load_from_file(default_file)) + async with AssetsApiClient(url, auth_key=key, sign_secret=secret) as client: - configs = await load_configs( - default_files=default_file, - assets_client=client, - override_files=override_file, - microgrid_ids=list(dict.fromkeys(microgrid_ids)) or None, - ) + if inplace: + # default_file is the patch target here, not a merge input. + configs = await load_configs( + assets_client=client, + override_files=override_file, + microgrid_ids=ids, + ) + else: + configs = await load_configs( + default_files=default_file, + assets_client=client, + override_files=override_file, + microgrid_ids=ids, + ) if not configs: raise click.ClickException("No microgrids could be loaded; nothing to write.") - click.echo(dump_map(configs), nl=False) + if inplace: + assert default_file is not None + patched = patch_file(default_file, configs) + fd, tmp_name = tempfile.mkstemp( + dir=default_file.parent, prefix=f".{default_file.name}." + ) + try: + with os.fdopen(fd, "w") as tmp_file: + tmp_file.write(patched) + os.replace(tmp_name, default_file) + except BaseException: + os.remove(tmp_name) + raise + click.echo(f"Patched {default_file}", err=True) + else: + click.echo(dump_map(configs), nl=False) def main() -> None: diff --git a/src/frequenz/gridpool/cli/_dump_config.py b/src/frequenz/gridpool/cli/_dump_config.py index 4c56cb4..eaa86fc 100644 --- a/src/frequenz/gridpool/cli/_dump_config.py +++ b/src/frequenz/gridpool/cli/_dump_config.py @@ -18,6 +18,7 @@ from typing import Any import tomlkit +from tomlkit.items import Integer, Trivia from frequenz.gridpool import MicrogridConfig @@ -27,6 +28,24 @@ def _is_empty(value: Any) -> bool: return value is None or value == {} or value == [] +def _format_value(value: Any) -> Any: + """Render whole-number floats and ints as underscore-grouped ints, e.g. `1_736_680`. + + Args: + value: The value about to be written to the TOML document. + + Returns: + The formatted value, or `value` itself if it is not a whole number. + """ + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, int) and not isinstance(value, bool): + # tomlkit.integer() just does int(raw), dropping underscores; build + # the Integer item directly instead. + return Integer(value, Trivia(), f"{value:_d}") + return value + + def _iter_leaves( prefix: list[str], data: dict[str, Any] ) -> list[tuple[list[str], Any]]: @@ -73,5 +92,5 @@ def dump_map(configs: dict[str, MicrogridConfig]) -> str: if doc.body: doc.add(tomlkit.nl()) for path, value in leaves: - doc.append(tomlkit.key(path), value) + doc.append(tomlkit.key(path), _format_value(value)) return tomlkit.dumps(doc) diff --git a/src/frequenz/gridpool/cli/_patch_config.py b/src/frequenz/gridpool/cli/_patch_config.py new file mode 100644 index 0000000..807fa85 --- /dev/null +++ b/src/frequenz/gridpool/cli/_patch_config.py @@ -0,0 +1,166 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Patch existing dotted-key TOML files with new microgrid config values. + +Unlike `dump_map`, which rebuilds a TOML document from scratch, this module +only adds missing leaves and missing microgrid entries. Values already on +disk always win, so comments, field order and number formatting survive. +""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import tomlkit +from tomlkit import TOMLDocument + +from frequenz.gridpool import MicrogridConfig + +from ._dump_config import _format_value, _iter_leaves + + +def patch_file(path: Path, configs: dict[str, MicrogridConfig]) -> str: + """Patch the TOML file at `path` with any leaves missing from `configs`. + + Args: + path: Path to the existing TOML file to patch. + configs: Mapping from microgrid ID (as string) to `MicrogridConfig`. + + Returns: + The patched TOML text; the caller is responsible for writing it back. + """ + return patch_text(path.read_text(), configs) + + +def patch_text(original: str, configs: dict[str, MicrogridConfig]) -> str: + """Patch dotted-key TOML text with any leaves missing from `configs`. + + Args: + original: The existing TOML text to patch. + configs: Mapping from microgrid ID (as string) to `MicrogridConfig`. + + Returns: + The patched TOML text. + """ + doc = tomlkit.parse(original) + schema = MicrogridConfig.Schema() + + # Leaves needing a whole new sub-table can't be inserted via item assignment + # without tomlkit falling back to bracket-header syntax, so they are spliced + # into the rendered text instead. + orphans: dict[str, list[tuple[list[str], Any]]] = {} + + for mid, cfg in configs.items(): + dumped = schema.dump(cfg) + assert isinstance(dumped, dict) + leaves = _iter_leaves([], dumped) + if not leaves: + continue + + if mid not in doc: + _append_new_entry(doc, mid, leaves) + continue + + for path, value in leaves: + if _leaf_exists(doc, mid, path): + continue + if not _insert_leaf(doc, mid, path, value): + orphans.setdefault(mid, []).append((path, value)) + + text = tomlkit.dumps(doc) + if orphans: + text = _splice_orphans(text, orphans) + return text + + +def _leaf_exists(doc: TOMLDocument, mid: str, path: list[str]) -> bool: + """Whether the dotted key `mid.path...` already has a value in `doc`.""" + node: Any = doc + for key in (mid, *path): + if not isinstance(node, Mapping) or key not in node: + return False + node = node[key] + return True + + +def _insert_leaf(doc: TOMLDocument, mid: str, path: list[str], value: Any) -> bool: + """Insert a leaf directly onto its existing parent table, if there is one. + + Args: + doc: The document being patched, mutated in place. + mid: Microgrid ID the leaf belongs to. + path: Dotted key path under `mid` for the leaf. + value: The leaf's value. + + Returns: + `True` if the leaf was inserted, `False` if a whole new sub-table is + needed instead (left for the caller to handle). + """ + node: Any = doc + matched = 0 + for key in (mid, *path[:-1]): + nxt = node[key] if isinstance(node, Mapping) and key in node else None + if not isinstance(nxt, Mapping): + break + node = nxt + matched += 1 + + full_path = [mid, *path] + if matched != len(full_path) - 1: + return False + node[full_path[-1]] = _format_value(value) + return True + + +def _append_new_entry( + doc: TOMLDocument, mid: str, leaves: list[tuple[list[str], Any]] +) -> None: + """Append a brand-new microgrid entry at the end of the document. + + Args: + doc: The document being patched, mutated in place. + mid: Microgrid ID of the new entry. + leaves: Flattened `(path, value)` pairs for the new entry, in + dataclass field order. + """ + if doc.body: + doc.add(tomlkit.nl()) + for path, value in leaves: + doc.append(tomlkit.key([mid, *path]), _format_value(value)) + + +def _render_lines(mid: str, leaves: list[tuple[list[str], Any]]) -> list[str]: + """Render `(path, value)` leaves as standalone `mid.path = value` lines.""" + tmp = tomlkit.document() + for path, value in leaves: + tmp.append(tomlkit.key([mid, *path]), _format_value(value)) + return tomlkit.dumps(tmp).splitlines(keepends=True) + + +def _splice_orphans(text: str, orphans: dict[str, list[tuple[list[str], Any]]]) -> str: + """Insert each microgrid's orphaned leaves right after its own last line. + + Args: + text: The already-rendered document text. + orphans: Mapping from microgrid ID to its `(path, value)` leaves + that need a brand-new sub-table. + + Returns: + `text` with the orphaned leaves inserted. + """ + lines = text.splitlines(keepends=True) + for mid, leaves in orphans.items(): + insert_at = _last_line_index_for_mid(lines, mid) + new_lines = _render_lines(mid, leaves) + lines[insert_at + 1 : insert_at + 1] = new_lines + return "".join(lines) + + +def _last_line_index_for_mid(lines: list[str], mid: str) -> int: + """Index of the last line belonging to `mid` (its key starts with `mid.`).""" + prefix = f"{mid}." + for idx in range(len(lines) - 1, -1, -1): + if lines[idx].lstrip().startswith(prefix): + return idx + raise ValueError(f"No existing line found for microgrid {mid!r}.") diff --git a/tests/test_dump_config.py b/tests/test_dump_config.py index 9e7938b..edc547b 100644 --- a/tests/test_dump_config.py +++ b/tests/test_dump_config.py @@ -7,7 +7,7 @@ from frequenz.gridpool import MicrogridConfig from frequenz.gridpool.cli._dump_config import dump_map -from frequenz.gridpool.config.microgrid import ComponentTypeConfig, Metadata +from frequenz.gridpool.config.microgrid import ComponentTypeConfig, Metadata, PVConfig def test_dump_map_round_trips() -> None: @@ -42,3 +42,20 @@ def test_dump_map_omits_empty_and_none() -> None: def test_dump_map_empty() -> None: """An empty mapping serializes to an empty string.""" assert dump_map({}) == "" + + +def test_dump_map_renders_whole_floats_as_underscored_ints() -> None: + """Whole-number float fields (e.g. peak/rated power) render as `1_736_680`, not `1736680.0`.""" + configs = { + "10": MicrogridConfig( + meta=Metadata(microgrid_id=10, latitude=52.5), + pv={"1": PVConfig(peak_power=1_736_680.0, rated_power=1_400_000.0)}, + ) + } + + text = dump_map(configs) + + assert "10.pv.1.peak_power = 1_736_680\n" in text + assert "10.pv.1.rated_power = 1_400_000\n" in text + # Genuinely fractional floats are left alone. + assert "10.meta.latitude = 52.5\n" in text diff --git a/tests/test_patch_config.py b/tests/test_patch_config.py new file mode 100644 index 0000000..36d2d2d --- /dev/null +++ b/tests/test_patch_config.py @@ -0,0 +1,123 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Tests for in-place patching of existing dotted-key TOML config files.""" + +from frequenz.gridpool.cli._patch_config import patch_text +from frequenz.gridpool.config.microgrid import ( + ComponentTypeConfig, + Metadata, + MicrogridConfig, + PVConfig, +) + +_ORIGINAL = """# EID 6 TOML configuration + +40.meta.name = "Bona - Auf dem Aurain" #grid_side True +40.meta.gid = 6 +40.meta.enterprise_id = 6 +40.meta.microgrid_id = 40 +40.meta.latitude = 50.39567065 +40.meta.longitude = 8.083947042665976 +40.ctype.grid.meter = [87] +40.pv.1.peak_power = 616_140 +40.pv.1.rated_power = 480_000 # https://example.com/technischedaten +""" + + +def test_patch_is_a_noop_when_nothing_changed() -> None: + """Patching with values already on disk leaves the file byte-identical.""" + configs = { + "40": MicrogridConfig( + meta=Metadata(microgrid_id=40, latitude=50.39567065), + pv={"1": PVConfig(peak_power=616_140.0, rated_power=480_000.0)}, + ) + } + + assert patch_text(_ORIGINAL, configs) == _ORIGINAL + + +def test_patch_inserts_missing_leaf_next_to_existing_table() -> None: + """A missing leaf under an existing table is inserted; everything else is untouched.""" + configs = { + "40": MicrogridConfig(meta=Metadata(microgrid_id=40, altitude=45.5)), + } + + patched = patch_text(_ORIGINAL, configs) + + lines = patched.splitlines() + assert 'name = "Bona - Auf dem Aurain" #grid_side True' in lines[2] + # A genuinely fractional value is left alone. + assert lines[3] == "40.meta.altitude = 45.5" + # Untouched lines are unchanged, including comments. + assert ( + "40.pv.1.rated_power = 480_000 # https://example.com/technischedaten" in patched + ) + assert "# EID 6 TOML configuration" in patched + + +def test_patch_appends_new_microgrid_at_the_end() -> None: + """A microgrid id absent from the file is appended, blank-line separated.""" + configs = { + "9999": MicrogridConfig(meta=Metadata(microgrid_id=9999, name="Brand New")), + } + + patched = patch_text(_ORIGINAL, configs) + + assert patched.startswith(_ORIGINAL) + assert patched[len(_ORIGINAL) :] == ( + '\n9999.meta.microgrid_id = 9_999\n9999.meta.name = "Brand New"\n' + ) + + +def test_patch_inserts_new_subtable_next_to_its_microgrid() -> None: + """A brand-new sub-table for an existing id lands next to that id's other lines.""" + configs = { + "40": MicrogridConfig( + meta=Metadata(microgrid_id=40), + pv={"2": PVConfig(peak_power=50_000.0)}, + ), + } + + patched = patch_text(_ORIGINAL, configs) + + assert patched == _ORIGINAL + "40.pv.2.peak_power = 50_000\n" + + +def test_patch_inserts_new_subtables_for_multiple_microgrids() -> None: + """Each microgrid's new sub-table lands next to its own lines, not all at the end.""" + original = _ORIGINAL + '\n41.meta.name = "Other Grid"\n41.meta.microgrid_id = 41\n' + configs = { + "40": MicrogridConfig( + meta=Metadata(microgrid_id=40), + pv={"2": PVConfig(peak_power=50_000.0)}, + ), + "41": MicrogridConfig( + meta=Metadata(microgrid_id=41), + ctype={"grid": ComponentTypeConfig(meter=[1])}, + ), + } + + patched = patch_text(original, configs) + + lines = patched.splitlines() + assert lines[ + lines.index( + "40.pv.1.rated_power = 480_000 # https://example.com/technischedaten" + ) + + 1 + ] == ("40.pv.2.peak_power = 50_000") + assert lines[-1] == "41.ctype.grid.meter = [1]" + + +def test_patch_formats_new_numeric_leaves() -> None: + """Newly inserted numeric leaves go through the same underscore formatting.""" + configs = { + "5555": MicrogridConfig( + meta=Metadata(microgrid_id=5555, enterprise_id=1_234_567) + ), + } + + patched = patch_text(_ORIGINAL, configs) + + assert "5555.meta.enterprise_id = 1_234_567\n" in patched