From 9dd779ad2b4a2c80473ec4cdfa364555bd5f58c5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 4 Dec 2025 11:27:27 +0100 Subject: [PATCH 01/59] docs: new section describing commitments Signed-off-by: F.N. Claessen --- documentation/concepts/commitments.rst | 169 +++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 documentation/concepts/commitments.rst diff --git a/documentation/concepts/commitments.rst b/documentation/concepts/commitments.rst new file mode 100644 index 0000000000..5521f6ac1d --- /dev/null +++ b/documentation/concepts/commitments.rst @@ -0,0 +1,169 @@ +Commitments +=========== + +Overview +-------- + +A **Commitment** is the central economic abstraction used by FlexMeasures to +express *soft constraints, preferences and market positions* in the scheduler. + +A commitment describes: + +- a **baseline quantity** over time (the target or assumed position), and +- marginal prices for **upwards** and **downwards deviations** from that baseline. + +The scheduler converts all provided commitments into terms in the optimization +objective function so that the solver *minimizes the total deviation cost* +across the schedule horizon. Absolute physical limits (for example generator or +line capacities) are *not* modelled as commitments — those are enforced as +Pyomo constraints. + +Key properties +-------------- + +Each Commitment has the following important attributes (high level): + +- ``name`` — a logical string identifier (e.g. ``"energy"``, ``"production peak"``). +- ``device`` — optional: restricts the commitment to a single device; otherwise + it is an EMS/site-level commitment. +- ``index`` — the DatetimeIndex (time grid) on which the series are defined. +- ``quantity`` — the baseline Series (per slot or per group). +- ``upwards_deviation_price`` — Series defining marginal cost/reward for upward deviations. +- ``downwards_deviation_price`` — Series defining marginal cost/reward for downward deviations. +- ``_type`` — grouping indicator: ``'each'`` or ``'any'`` (see Grouping below). + +Sign convention (flows vs stocks) +-------------------------------- + +- **Flow commitments** (e.g. power/energy flows): + + - A *positive* baseline quantity denotes **consumption**. + + - Actual > baseline → *upwards* deviation (more consumption). + - Actual < baseline → *downwards* deviation (less consumption). + - A *negative* baseline quantity denotes **production** (feed-in). + + - Actual less negative (i.e. closer to zero) → *upwards* deviation (less production). + - Actual more negative → *downwards* deviation (more production). + +- **Stock commitments** (e.g. state of charge for storage): + + - ``quantity`` is the target stock level; deviations above/below that target + are priced via the upwards/downwards price series. + +Soft vs hard semantics +---------------------- + +Commitments in FlexMeasures are **soft** by design: they represent economic +penalties or rewards that the optimizer considers when building schedules. +Hard operational constraints (such as physical power limits or strict device +interlocks) are expressed separately as Pyomo constraints in the scheduling +model. If a “hard” behaviour is required from a commitment, assign very large +penalty prices, but prefer modelling non-negotiable limits as Pyomo constraints. + +Converting flex-context fields into commitments +----------------------------------------------- + +Users may supply preferences and price fields in the ``flex-context``. The +scheduler translates the relevant fields into one or more `Commitment` objects +before calling the optimizer. + +Typical translations include: + +- tariffs (``consumption-price``, ``production-price``) → an ``"energy"`` FlowCommitment with zero baseline so net consumption/production is priced; +- peak/excess limits (``site-peak-production``, ``site-peak-production-price``, etc.) → dedicated peak FlowCommitment(s); +- storage-related fields (``soc-minima``, ``soc-minima-breach-price``, etc.) → StockCommitment(s). + +A short example +--------------- + +Below is a compact example showing how the scheduler conceptually creates an +``"energy"`` flow commitment from a (per-slot) tariff: + +.. code-block:: python + + from pandas import Series, date_range + from flexmeasures.data.models.planning import FlowCommitment + + index = date_range(start="2025-01-01 00:00", periods=24, freq="H") + # zero baseline → the asset may consume or produce; deviations are priced. + baseline = Series(0.0, index=index) + + # consumption and production tariffs (per kWh) + consumption_price = Series(0.20, index=index) # 0.20 EUR/kWh for consumption + production_price = Series(-0.05, index=index) # -0.05 EUR/kWh reward for production + + energy_commitment = FlowCommitment( + name="energy", + index=index, + quantity=baseline, + upwards_deviation_price=consumption_price, + downwards_deviation_price=production_price, + _type="each" + ) + +The scheduler sets up such commitments (site-level and device-level) and, together with any prior commitments, hands them to the linear optimizer. + +Examples (commitments commonly derived from flex-context) +-------------------------------------------------------- + +The examples below map the most common `flex-context` semantics to the +commitments the scheduler constructs. + +1. **Energy (tariff)** + + - *Fields used*: ``consumption-price``, ``production-price``. + - *Commitment*: Flow commitment named ``"energy"`` with zero baseline and + the two price series as upwards/downwards deviation prices. + +2. **Peak consumption** + + - *Fields used*: ``site-peak-consumption`` (baseline) and ``site-peak-consumption-price`` (upwards-deviation price); the downwards price is set to ``0``. + - *Commitment*: Flow commitment named ``"consumption peak"``; positive baseline + values denote the prior consumption peak associated with sunk costs, and the upwards price penalises going beyond that baseline. + +3. **Peak production / peak feed-in** + + - *Fields used*: ``site-peak-production`` (baseline) and ``site-peak-production-price`` (downwards-deviation price); the upwards price is set to ``0``. + - *Commitment*: Flow commitment named ``"production peak"``; negative baseline + values denote the prior production peak associated with sunk costs, and the downwards price penalises going beyond that baseline. + +4. **Consumption capacity** + + - *Fields used*: ``site-consumption-capacity`` (baseline), and ``site-consumption-breach-price`` (upwards-deviation price); the downwards price is set to ``0``. + - *Commitment*: Flow commitment named ``"consumption breach"``; positive baseline + values denote the allowed consumption limit and the upwards price penalises going + beyond that limit. + +5. **Production capacity** + + - *Fields used*: ``site-production-capacity`` (baseline) and ``site-production-breach-price`` (downwards-deviation price); the upwards price is set to ``0``. + - *Commitment*: Flow commitment named ``"production breach"``; negative baseline + values denote the allowed production limit and the downwards price penalises going + beyond that limit. + +6. **SOC minima / maxima (storage preferences)** + + - *Fields used*: ``soc-minima``, ``soc-minima-breach-price``, ``soc-maxima`` and ``soc-maxima-breach-price``. + - *Commitment*: StockCommitment(s) that price deviations below minima or + above maxima. Hard storage capacities are set through ``soc-min`` and ``soc-max`` instead and are modelled as Pyomo constraints. + +7. **Power bands per device** + + - *Fields used*: ``consumption-capacity`` and ``production-capacity`` (baselines), ``consumption-breach-price`` (upwards-deviation price, with 0 downwards) and ``production-breach-price`` (downwards-deviation price, with 0 upwards). + - *Commitment*: FlowCommitment with either baseline and corresponding prices. + +Grouping across time and devices +-------------------------------- + +- ``_type == 'each'``: penalise deviations per time slot (default for time series). +- ``_type == 'any'``: treat the whole commitment horizon as one group (useful + for peak-style penalties where only the maximum over the window should be + counted). + +.. note:: + + Near-term feature: support for **grouping over devices** is planned and + documented here. When enabled, grouping over devices lets you express + soft constraints that aggregate deviations across a set of devices, + for example, an intermediate capacity constraint from a feeder shared by a group of devices (via **flow commitments**), or multiple power-to-heat devices that feed a shared thermal buffer (via **stock commitments**). From 13e93086fd3c0b928ce6d3fcc6e3c764d58a9a0e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 4 Dec 2025 13:44:08 +0100 Subject: [PATCH 02/59] docs: rewrite the overview in commitments section Signed-off-by: F.N. Claessen --- documentation/concepts/commitments.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/documentation/concepts/commitments.rst b/documentation/concepts/commitments.rst index 5521f6ac1d..24d3143ad1 100644 --- a/documentation/concepts/commitments.rst +++ b/documentation/concepts/commitments.rst @@ -4,17 +4,19 @@ Commitments Overview -------- -A **Commitment** is the central economic abstraction used by FlexMeasures to -express *soft constraints, preferences and market positions* in the scheduler. +A **Commitment** is the economic abstraction FlexMeasures uses to express +market positions and soft constraints (preferences) inside the scheduler. +Commitments are converted to linear objective terms; all non-negotiable +operational limits are modelled separately as Pyomo constraints. A commitment describes: -- a **baseline quantity** over time (the target or assumed position), and +- a **baseline quantity** over time (the contracted or preferred position), and - marginal prices for **upwards** and **downwards deviations** from that baseline. The scheduler converts all provided commitments into terms in the optimization objective function so that the solver *minimizes the total deviation cost* -across the schedule horizon. Absolute physical limits (for example generator or +across the schedule horizon. Absolute physical limitations (for example generator or line capacities) are *not* modelled as commitments — those are enforced as Pyomo constraints. From 117919806ca403cb2873c5eb2db509a6c0c8d1d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 4 Dec 2025 13:52:05 +0100 Subject: [PATCH 03/59] docs: keep ems variables, but explain them as representing the site context Signed-off-by: F.N. Claessen --- documentation/concepts/device_scheduler.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/documentation/concepts/device_scheduler.rst b/documentation/concepts/device_scheduler.rst index 7d85e57415..3b87ac7963 100644 --- a/documentation/concepts/device_scheduler.rst +++ b/documentation/concepts/device_scheduler.rst @@ -5,8 +5,8 @@ Storage device scheduler: Linear model Introduction -------------- -This generic storage device scheduler is able to handle an EMS with multiple devices, with various types of constraints on the EMS level and on the device level, -and with multiple market commitments on the EMS level. +This generic storage device scheduler is able to handle a site with multiple devices, with various types of constraints on the site level and on the device level, +and with multiple market commitments on the site level. A typical example is a house with many devices. The commitments are assumed to be with regard to the flow of energy to the device (positive for consumption, negative for production). In practice, this generic scheduler is used in the **StorageScheduler** to schedule a storage device. @@ -45,9 +45,9 @@ Symbol Variable in the Code :math:`\epsilon(d,j)` efficiencies Stock energy losses. :math:`P_{max}(d,j)` device_derivative_max Maximum flow of device :math:`d` during time period :math:`j`. :math:`P_{min}(d,j)` device_derivative_min Minimum flow of device :math:`d` during time period :math:`j`. -:math:`P^{ems}_{min}(j)` ems_derivative_min Minimum flow of the EMS during time period :math:`j`. -:math:`P^{ems}_{max}(j)` ems_derivative_max Maximum flow of the EMS during time period :math:`j`. -:math:`Commitment(c,j)` commitment_quantity Commitment c (at EMS level) over time step :math:`j`. +:math:`P^{ems}_{min}(j)` ems_derivative_min Minimum flow of the site's grid connection point during time period :math:`j`. +:math:`P^{ems}_{max}(j)` ems_derivative_max Maximum flow of the site's grid connection point during time period :math:`j`. +:math:`Commitment(c,j)` commitment_quantity Commitment c (at site level) over time step :math:`j`. :math:`M` M Large constant number, upper bound of :math:`Power_{up}(d,j)` and :math:`|Power_{down}(d,j)|`. :math:`D(d,j)` stock_delta Explicit energy gain or loss of device :math:`d` during time period :math:`j`. ================================ ================================================ ============================================================================================================== @@ -58,8 +58,8 @@ Variables ================================ ================================================ ============================================================================================================== Symbol Variable in the Code Description ================================ ================================================ ============================================================================================================== -:math:`\Delta_{up}(c,j)` commitment_upwards_deviation Upwards deviation from the power commitment :math:`c` of the EMS during time period :math:`j`. -:math:`\Delta_{down}(c,j)` commitment_downwards_deviation Downwards deviation from the power commitment :math:`c` of the EMS during time period :math:`j`. +:math:`\Delta_{up}(c,j)` commitment_upwards_deviation Upwards deviation from the power commitment :math:`c` of the site during time period :math:`j`. +:math:`\Delta_{down}(c,j)` commitment_downwards_deviation Downwards deviation from the power commitment :math:`c` of the site during time period :math:`j`. :math:`\Delta Stock(d,j)` n/a Change of stock of device :math:`d` at the end of time period :math:`j`. :math:`P_{up}(d,j)` device_power_up Upwards power of device :math:`d` during time period :math:`j`. :math:`P_{down}(d,j)` device_power_down Downwards power of device :math:`d` during time period :math:`j`. From 1b103601a5d080f892cec0590d0aae61a8486e4b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 4 Dec 2025 13:55:24 +0100 Subject: [PATCH 04/59] docs: cross-reference the commitments section and the linear problem formulation Signed-off-by: F.N. Claessen --- documentation/concepts/commitments.rst | 8 ++++++++ documentation/concepts/device_scheduler.rst | 1 + 2 files changed, 9 insertions(+) diff --git a/documentation/concepts/commitments.rst b/documentation/concepts/commitments.rst index 24d3143ad1..d21094e522 100644 --- a/documentation/concepts/commitments.rst +++ b/documentation/concepts/commitments.rst @@ -1,3 +1,5 @@ +.. _commitments: + Commitments =========== @@ -169,3 +171,9 @@ Grouping across time and devices documented here. When enabled, grouping over devices lets you express soft constraints that aggregate deviations across a set of devices, for example, an intermediate capacity constraint from a feeder shared by a group of devices (via **flow commitments**), or multiple power-to-heat devices that feed a shared thermal buffer (via **stock commitments**). + + +Advanced: mathematical formulation +---------------------------------- + +For a compact formulation of how commitments enter the optimization problem, see :ref:``. diff --git a/documentation/concepts/device_scheduler.rst b/documentation/concepts/device_scheduler.rst index 3b87ac7963..cb36d0dc46 100644 --- a/documentation/concepts/device_scheduler.rst +++ b/documentation/concepts/device_scheduler.rst @@ -11,6 +11,7 @@ and with multiple market commitments on the site level. A typical example is a house with many devices. The commitments are assumed to be with regard to the flow of energy to the device (positive for consumption, negative for production). In practice, this generic scheduler is used in the **StorageScheduler** to schedule a storage device. The solver minimizes the costs of deviating from the commitments. +For a more detailed explanation of commitments in FlexMeasures, see :ref:``. From 6f6e52b0290a4af369b4cc1f898b8ca30dd07fd8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 8 Dec 2025 15:54:12 +0100 Subject: [PATCH 05/59] fix: cross-references Signed-off-by: F.N. Claessen --- documentation/concepts/commitments.rst | 2 +- documentation/concepts/device_scheduler.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/concepts/commitments.rst b/documentation/concepts/commitments.rst index d21094e522..11ab4cc955 100644 --- a/documentation/concepts/commitments.rst +++ b/documentation/concepts/commitments.rst @@ -176,4 +176,4 @@ Grouping across time and devices Advanced: mathematical formulation ---------------------------------- -For a compact formulation of how commitments enter the optimization problem, see :ref:``. +For a compact formulation of how commitments enter the optimization problem, see :ref:`storage_device_scheduler`. diff --git a/documentation/concepts/device_scheduler.rst b/documentation/concepts/device_scheduler.rst index cb36d0dc46..289f40cfc8 100644 --- a/documentation/concepts/device_scheduler.rst +++ b/documentation/concepts/device_scheduler.rst @@ -11,7 +11,7 @@ and with multiple market commitments on the site level. A typical example is a house with many devices. The commitments are assumed to be with regard to the flow of energy to the device (positive for consumption, negative for production). In practice, this generic scheduler is used in the **StorageScheduler** to schedule a storage device. The solver minimizes the costs of deviating from the commitments. -For a more detailed explanation of commitments in FlexMeasures, see :ref:``. +For a more detailed explanation of commitments in FlexMeasures, see :ref:`commitments`. From 359937985193a7cf4e018df6e29b079e081eeaee Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 8 Dec 2025 15:54:26 +0100 Subject: [PATCH 06/59] docs: add commitments section to index Signed-off-by: F.N. Claessen --- documentation/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/index.rst b/documentation/index.rst index 337beb2585..748b160395 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -189,6 +189,7 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum concepts/flexibility concepts/data-model concepts/security_auth + concepts/commitments concepts/device_scheduler From 84fbd5a4f731417f9d4eb1f26937c050abb9562e Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Fri, 23 Jan 2026 11:42:18 +0100 Subject: [PATCH 07/59] feat: add a util function for printing out commitments in a tabulated format --- flexmeasures/data/models/planning/utils.py | 30 ++++++++++++++++++++++ flexmeasures/ui/static/openapi-specs.json | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 6ba41dd20e..30fe7b457c 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -8,6 +8,7 @@ from pandas.tseries.frequencies import to_offset import numpy as np import timely_beliefs as tb +from tabulate import tabulate from flexmeasures.data.models.planning.exceptions import UnknownPricesException from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -559,3 +560,32 @@ def initialize_device_commitment( stock_commitment["device"] = device stock_commitment["class"] = StockCommitment return stock_commitment + + +def print_commitments(flow_commitments): + """ + Pretty-print a list of FlowCommitment objects as tabulated pandas DataFrames. + + For each FlowCommitment, a DataFrame indexed by time is created containing + the commitment name, device values, group index, quantity, and any available + upward or downward deviation prices. Each commitment is printed separately + in a readable table format, making this function suitable for debugging, + logging, and interactive inspection. + """ + for fc in flow_commitments: + + df = pd.DataFrame(index=fc.device.index) + + df["commitment"] = fc.name + df["device"] = fc.device + df["group"] = fc.group + df["quantity"] = fc.quantity + + if hasattr(fc, "upwards_deviation_price"): + df["up_price"] = fc.upwards_deviation_price + + if hasattr(fc, "downwards_deviation_price"): + df["down_price"] = fc.downwards_deviation_price + + if not df.empty: + print(tabulate(df, headers=df.columns, tablefmt="fancy_grid")) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 89629bb1cf..8c2dee2f83 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.31.0" + "version": "0.30.0" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", From 90177bfcfdacd28134763969c07cc3cb90ca78c4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 23 Jan 2026 12:54:01 +0100 Subject: [PATCH 08/59] refactor: move pretty printing method to class Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/__init__.py | 28 +++++++++++++++++ flexmeasures/data/models/planning/utils.py | 30 ------------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py index 636ff9e2b5..9a18f990f0 100644 --- a/flexmeasures/data/models/planning/__init__.py +++ b/flexmeasures/data/models/planning/__init__.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta +from tabulate import tabulate from typing import Any, Dict, List, Type, Union import pandas as pd @@ -340,6 +341,33 @@ def __post_init__(self): ) self.group = self.group.rename("group") + def pretty_print(self): + """ + Pretty-print a list of FlowCommitment objects as tabulated pandas DataFrames. + + For each FlowCommitment, a DataFrame indexed by time is created containing + the commitment name, device values, group index, quantity, and any available + upward or downward deviation prices. Each commitment is printed separately + in a readable table format, making this function suitable for debugging, + logging, and interactive inspection. + """ + df = self.to_frame() + df = pd.DataFrame(index=df.device.index) + + df["commitment"] = self.name + df["device"] = self.device + df["group"] = self.group + df["quantity"] = self.quantity + + if hasattr(self, "upwards_deviation_price"): + df["up_price"] = self.upwards_deviation_price + + if hasattr(self, "downwards_deviation_price"): + df["down_price"] = self.downwards_deviation_price + + if not df.empty: + print(tabulate(df, headers=df.columns, tablefmt="fancy_grid")) + def to_frame(self) -> pd.DataFrame: """Contains all info apart from the name.""" return pd.concat( diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 30fe7b457c..6ba41dd20e 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -8,7 +8,6 @@ from pandas.tseries.frequencies import to_offset import numpy as np import timely_beliefs as tb -from tabulate import tabulate from flexmeasures.data.models.planning.exceptions import UnknownPricesException from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -560,32 +559,3 @@ def initialize_device_commitment( stock_commitment["device"] = device stock_commitment["class"] = StockCommitment return stock_commitment - - -def print_commitments(flow_commitments): - """ - Pretty-print a list of FlowCommitment objects as tabulated pandas DataFrames. - - For each FlowCommitment, a DataFrame indexed by time is created containing - the commitment name, device values, group index, quantity, and any available - upward or downward deviation prices. Each commitment is printed separately - in a readable table format, making this function suitable for debugging, - logging, and interactive inspection. - """ - for fc in flow_commitments: - - df = pd.DataFrame(index=fc.device.index) - - df["commitment"] = fc.name - df["device"] = fc.device - df["group"] = fc.group - df["quantity"] = fc.quantity - - if hasattr(fc, "upwards_deviation_price"): - df["up_price"] = fc.upwards_deviation_price - - if hasattr(fc, "downwards_deviation_price"): - df["down_price"] = fc.downwards_deviation_price - - if not df.empty: - print(tabulate(df, headers=df.columns, tablefmt="fancy_grid")) From 9f34676c64ec49f145533f01cdf8d785965f12ab Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 23 Jan 2026 12:54:46 +0100 Subject: [PATCH 09/59] feat: Commitment supports device groups Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/__init__.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py index 9a18f990f0..df7e6dbd3f 100644 --- a/flexmeasures/data/models/planning/__init__.py +++ b/flexmeasures/data/models/planning/__init__.py @@ -281,6 +281,7 @@ class Commitment: name: str device: pd.Series = None + device_group: pd.Series = None index: pd.DatetimeIndex = field(repr=False, default=None) _type: str = field(repr=False, default="each") group: pd.Series = field(init=False) @@ -340,6 +341,36 @@ def __post_init__(self): "downwards deviation price" ) self.group = self.group.rename("group") + self._init_device_group() + + def _init_device_group(self): + # EMS-level commitment + if self.device is None: + self.device_group = pd.Series({"EMS": 0}, name="device_group") + return + + # Extract device universe + if isinstance(self.device, pd.Series): + devices = self.device.unique() + else: + devices = [self.device] + + devices = list(devices) + + # Default: one group per device (backwards compatible) + if self.device_group is None: + self.device_group = pd.Series( + range(len(devices)), index=devices, name="device_group" + ) + else: + # Validate custom grouping + missing = set(devices) - set(self.device_group.index) + if missing: + raise ValueError( + f"device_group missing assignments for devices: {missing}" + ) + self.device_group = self.device_group.loc[devices] + self.device_group.name = "device_group" def pretty_print(self): """ @@ -370,7 +401,7 @@ def pretty_print(self): def to_frame(self) -> pd.DataFrame: """Contains all info apart from the name.""" - return pd.concat( + df = pd.concat( [ self.device, self.quantity, @@ -381,6 +412,13 @@ def to_frame(self) -> pd.DataFrame: ], axis=1, ) + # map device → device_group + if self.device is not None: + df["device_group"] = self.device.map(self.device_group) + else: + df["device_group"] = 0 + + return df class FlowCommitment(Commitment): From 43107c827933cd071619f2332e1ea432a679f382 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 23 Jan 2026 12:55:47 +0100 Subject: [PATCH 10/59] feat: start testing device grouping Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_commitments.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 flexmeasures/data/models/planning/tests/test_commitments.py diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py new file mode 100644 index 0000000000..e4bd2ba2f7 --- /dev/null +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -0,0 +1,59 @@ +import pandas as pd + +from flexmeasures.data.models.planning import StockCommitment +from flexmeasures.data.models.planning.utils import initialize_index + + +def test_multi_feed(): + start = pd.Timestamp("2026-01-01T00:00+01") + end = pd.Timestamp("2026-01-02T00:00+01") + resolution = pd.Timedelta("PT1H") + index = (initialize_index(start=start, end=end, resolution=resolution),) + + device_group = pd.Series( + { + "gas boiler": "shared thermal buffer", + "heat pump power": "shared thermal buffer", + "battery power": "battery SoC", + } + ) + + max_thermal_soc = "100 kWh" + breach_price = "1000 EUR/kWh" + + commitment_a = StockCommitment( + name="buffer max", + index=index, + quantity=max_thermal_soc, + upwards_deviation_price=breach_price, + downwards_deviation_price=0, + device=pd.Series( + "gas boiler", index=index + ), # per-slot device resolution happens elsewhere + device_group=device_group, + ) + commitment_b = StockCommitment( + name="buffer max", + index=index, + quantity=max_thermal_soc, + upwards_deviation_price=breach_price, + downwards_deviation_price=0, + device=pd.Series( + "heat pump power", index=index + ), # per-slot device resolution happens elsewhere + device_group=device_group, + ) + commitment_c = StockCommitment( + name="buffer max", + index=index, + quantity=max_thermal_soc, + upwards_deviation_price=breach_price, + downwards_deviation_price=0, + device=pd.Series( + "battery power", index=index + ), # per-slot device resolution happens elsewhere + device_group=device_group, + ) + assert commitment_a.device_group[0] == "shared thermal buffer" + assert commitment_b.device_group[0] == "shared thermal buffer" + assert commitment_c.device_group[0] == "battery SoC" From b90d7f0da8d9036c535e0f397af740c0e3b1adcb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 23 Jan 2026 14:35:35 +0100 Subject: [PATCH 11/59] dev: test multi-feed Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_commitments.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index e4bd2ba2f7..9472b845eb 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -57,3 +57,104 @@ def test_multi_feed(): assert commitment_a.device_group[0] == "shared thermal buffer" assert commitment_b.device_group[0] == "shared thermal buffer" assert commitment_c.device_group[0] == "battery SoC" + + +import pandas as pd +import numpy as np + +from flexmeasures.data.models.planning import StockCommitment +from flexmeasures.data.models.planning.utils import initialize_index +from flexmeasures.data.models.planning.linear_optimization import device_scheduler + + +def test_multi_feed_device_scheduler_shared_buffer(): + # ---- time setup + start = pd.Timestamp("2026-01-01T00:00+01") + end = pd.Timestamp("2026-01-02T00:00+01") + resolution = pd.Timedelta("PT1H") + index = initialize_index(start=start, end=end, resolution=resolution) + + # ---- three devices + devices = ["gas boiler", "heat pump power", "battery power"] + + # ---- device grouping + device_group = pd.Series( + { + "gas boiler": "shared thermal buffer", + "heat pump power": "shared thermal buffer", + "battery power": "battery SoC", + } + ) + + # ---- trivial device constraints (stocks unconstrained individually) + device_constraints = [] + for _ in devices: + df = pd.DataFrame( + { + "min": -np.inf, + "max": np.inf, + "equals": np.nan, + "derivative min": -np.inf, + "derivative max": np.inf, + "derivative equals": np.nan, + }, + index=index, + ) + device_constraints.append(df) + + # ---- no EMS-level constraints + ems_constraints = pd.DataFrame( + { + "derivative min": -np.inf, + "derivative max": np.inf, + }, + index=index, + ) + + # ---- shared buffer max = 100 (soft) + max_soc = 100.0 + breach_price = 1_000.0 + + commitments = [] + for dev in devices: + commitments.append( + StockCommitment( + name="buffer max", + index=index, + quantity=max_soc, + upwards_deviation_price=breach_price, + downwards_deviation_price=0, + device=pd.Series(dev, index=index), + device_group=device_group, + ) + ) + + # ---- run scheduler + planned_power, planned_costs, results, model = device_scheduler( + device_constraints=device_constraints, + ems_constraints=ems_constraints, + commitments=commitments, + initial_stock=0, + ) + + # ---- sanity: model solved + assert results.solver.termination_condition in ("optimal", "locallyOptimal") + + # ---- key assertion: exactly TWO commitment groups + # - one for "shared thermal buffer" + # - one for "battery SoC" + # + # i.e. NOT three (which would indicate per-device baselines) + commitment_groups = set(commitments[0].device_group.values) + breakpoint() + assert commitment_groups == { + "shared thermal buffer", + "battery SoC", + } + + # ---- key behavioural check: + # total commitment cost should be <= 1 breach per group per timestep + # + # If baselines were duplicated, cost would be ~2x for the shared buffer. + expected_max_cost = len(index) * breach_price * 2 + assert planned_costs <= expected_max_cost From 41799816ea8d9f2c44bcaa42828764a96781d6da Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Mon, 26 Jan 2026 15:47:14 +0100 Subject: [PATCH 12/59] update the ids of devices to be integers --- .../models/planning/tests/test_commitments.py | 77 ++----------------- 1 file changed, 6 insertions(+), 71 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 9472b845eb..4d856c733c 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -1,64 +1,3 @@ -import pandas as pd - -from flexmeasures.data.models.planning import StockCommitment -from flexmeasures.data.models.planning.utils import initialize_index - - -def test_multi_feed(): - start = pd.Timestamp("2026-01-01T00:00+01") - end = pd.Timestamp("2026-01-02T00:00+01") - resolution = pd.Timedelta("PT1H") - index = (initialize_index(start=start, end=end, resolution=resolution),) - - device_group = pd.Series( - { - "gas boiler": "shared thermal buffer", - "heat pump power": "shared thermal buffer", - "battery power": "battery SoC", - } - ) - - max_thermal_soc = "100 kWh" - breach_price = "1000 EUR/kWh" - - commitment_a = StockCommitment( - name="buffer max", - index=index, - quantity=max_thermal_soc, - upwards_deviation_price=breach_price, - downwards_deviation_price=0, - device=pd.Series( - "gas boiler", index=index - ), # per-slot device resolution happens elsewhere - device_group=device_group, - ) - commitment_b = StockCommitment( - name="buffer max", - index=index, - quantity=max_thermal_soc, - upwards_deviation_price=breach_price, - downwards_deviation_price=0, - device=pd.Series( - "heat pump power", index=index - ), # per-slot device resolution happens elsewhere - device_group=device_group, - ) - commitment_c = StockCommitment( - name="buffer max", - index=index, - quantity=max_thermal_soc, - upwards_deviation_price=breach_price, - downwards_deviation_price=0, - device=pd.Series( - "battery power", index=index - ), # per-slot device resolution happens elsewhere - device_group=device_group, - ) - assert commitment_a.device_group[0] == "shared thermal buffer" - assert commitment_b.device_group[0] == "shared thermal buffer" - assert commitment_c.device_group[0] == "battery SoC" - - import pandas as pd import numpy as np @@ -80,9 +19,9 @@ def test_multi_feed_device_scheduler_shared_buffer(): # ---- device grouping device_group = pd.Series( { - "gas boiler": "shared thermal buffer", - "heat pump power": "shared thermal buffer", - "battery power": "battery SoC", + 0: "shared thermal buffer", # gas boiler + 1: "shared thermal buffer", # "heat pump power" + 2: "battery SoC", #"battery power" } ) @@ -116,7 +55,7 @@ def test_multi_feed_device_scheduler_shared_buffer(): breach_price = 1_000.0 commitments = [] - for dev in devices: + for d,dev in enumerate(devices): commitments.append( StockCommitment( name="buffer max", @@ -124,7 +63,7 @@ def test_multi_feed_device_scheduler_shared_buffer(): quantity=max_soc, upwards_deviation_price=breach_price, downwards_deviation_price=0, - device=pd.Series(dev, index=index), + device=pd.Series(d, index=index), device_group=device_group, ) ) @@ -146,11 +85,7 @@ def test_multi_feed_device_scheduler_shared_buffer(): # # i.e. NOT three (which would indicate per-device baselines) commitment_groups = set(commitments[0].device_group.values) - breakpoint() - assert commitment_groups == { - "shared thermal buffer", - "battery SoC", - } + assert commitment_groups == {"shared thermal buffer"} # ---- key behavioural check: # total commitment cost should be <= 1 breach per group per timestep From 8b108ed881a7576760d87c8a24a6bb48b8855eb4 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Mon, 26 Jan 2026 15:52:21 +0100 Subject: [PATCH 13/59] feat: function that group commitment quantities Signed-off-by: Ahmad-Wahid --- .../models/planning/linear_optimization.py | 68 +++++++++++++++++-- flexmeasures/ui/static/openapi-specs.json | 2 +- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 2f1ba06d1d..38caeb127e 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -207,6 +207,22 @@ def convert_commitments_to_subcommitments( commitments, commitment_mapping = convert_commitments_to_subcommitments(commitments) + device_group_lookup = {} + + for c, df in enumerate(commitments): + if "device_group" not in df.columns or "device" not in df.columns: + continue + + device_group_lookup[c] = {} + + # keep only relevant rows + rows = df[["device", "device_group"]].dropna().drop_duplicates() + + for _, row in rows.iterrows(): + g = row["device_group"] + d = row["device"] # NOTE: must match model.d indexing + device_group_lookup[c].setdefault(g, set()).add(d) + # Oversimplified check for a convex cost curve df = pd.concat(commitments)[ ["upwards deviation price", "downwards deviation price"] @@ -243,13 +259,21 @@ def convert_commitments_to_subcommitments( ) model.c = RangeSet(0, len(commitments) - 1, doc="Set of commitments") - # Add 2D indices for commitment datetimes (cj) + def commitment_device_groups_init(m): + return ((c, g) for c, groups in device_group_lookup.items() for g in groups) + + model.cg = Set(dimen=2, initialize=commitment_device_groups_init) def commitments_init(m): return ((c, j) for c in m.c for j in commitments[c]["j"]) model.cj = Set(dimen=2, initialize=commitments_init) + def commitment_time_device_groups_init(m): + return ((c, j, g) for (c, j) in m.cj for (_, g) in m.cg if _ == c) + + model.cjg = Set(dimen=3, initialize=commitment_time_device_groups_init) + # Add parameters def price_down_select(m, c): if "downwards deviation price" not in commitments[c].columns: @@ -362,6 +386,40 @@ def device_derivative_up_efficiency(m, d, j): def device_stock_delta(m, d, j): return device_constraints[d]["stock delta"].iloc[j] + def grouped_commitment_equalities(m, c, j, g): + """ + Enforce a commitment deviation constraint on the aggregate of devices in a group. + + For commitment ``c`` at time index ``j``, this constraint couples the commitment + baseline (plus deviation variables) to the summed flow or stock of all devices + belonging to device group ``g``. StockCommitments aggregate device stocks, while + FlowCommitments aggregate device flows. Constraints are skipped if the commitment + is inactive at ``(c, j)`` or if the group contains no devices. + """ + if m.commitment_quantity[c, j] == -infinity: + return Constraint.Skip + + devices_in_group = device_group_lookup.get(c, {}).get(g, set()) + if not devices_in_group: + return Constraint.Skip + + center = ( + m.commitment_quantity[c, j] + + m.commitment_downwards_deviation[c] + + m.commitment_upwards_deviation[c] + ) + + if commitments[c]["class"].apply(lambda cl: cl == StockCommitment).all(): + center -= sum(_get_stock_change(m, d, j) for d in devices_in_group) + else: + center -= sum(m.ems_power[d, j] for d in devices_in_group) + + return ( + 0 if "upwards deviation price" in commitments[c].columns else None, + center, + 0 if "downwards deviation price" in commitments[c].columns else None, + ) + model.up_price = Param(model.c, initialize=price_up_select) model.down_price = Param(model.c, initialize=price_down_select) model.commitment_quantity = Param( @@ -565,6 +623,10 @@ def device_derivative_equalities(m, d, j): 0, ) + model.grouped_commitment_equalities = Constraint( + model.cjg, rule=grouped_commitment_equalities + ) + model.device_energy_bounds = Constraint(model.d, model.j, rule=device_bounds) model.device_power_bounds = Constraint( model.d, model.j, rule=device_derivative_bounds @@ -592,9 +654,7 @@ def device_derivative_equalities(m, d, j): model.ems_power_commitment_equalities = Constraint( model.cj, rule=ems_flow_commitment_equalities ) - model.device_energy_commitment_equalities = Constraint( - model.cj, model.d, rule=device_stock_commitment_equalities - ) + model.device_power_equalities = Constraint( model.d, model.j, rule=device_derivative_equalities ) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 8c2dee2f83..89629bb1cf 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.30.0" + "version": "0.31.0" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", From 485349e425b6efe40b5abbfed45aaf457bbd2246 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Mon, 26 Jan 2026 18:34:58 +0100 Subject: [PATCH 14/59] add commitments for multi group Signed-off-by: Ahmad-Wahid --- .../models/planning/tests/test_commitments.py | 105 +++++++++++++++--- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 4d856c733c..26349f0ff6 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -1,8 +1,11 @@ import pandas as pd import numpy as np -from flexmeasures.data.models.planning import StockCommitment -from flexmeasures.data.models.planning.utils import initialize_index +from flexmeasures.data.models.planning import StockCommitment, FlowCommitment +from flexmeasures.data.models.planning.utils import ( + initialize_index, + add_tiny_price_slope, +) from flexmeasures.data.models.planning.linear_optimization import device_scheduler @@ -19,23 +22,36 @@ def test_multi_feed_device_scheduler_shared_buffer(): # ---- device grouping device_group = pd.Series( { - 0: "shared thermal buffer", # gas boiler - 1: "shared thermal buffer", # "heat pump power" - 2: "battery SoC", #"battery power" + 0: "shared thermal buffer", # gas boiler + 1: "shared thermal buffer", # "heat pump power" + 2: "battery SoC", # "battery power" } ) - + device_commodity = pd.Series( + { + 0: "gas", # gas boiler + 1: "electricity", # "heat pump power" + 2: "electricity", # "battery power" + } + ) + equals = pd.Series(np.nan, index=index) + equals[-1] = 100 # ---- trivial device constraints (stocks unconstrained individually) device_constraints = [] - for _ in devices: + for d, device_name in enumerate(devices): + # 0 and 1 : derivative min 0 + # 2 : derivative min = - production capacity + df = pd.DataFrame( { - "min": -np.inf, - "max": np.inf, + "min": 0, + "max": 100, "equals": np.nan, - "derivative min": -np.inf, - "derivative max": np.inf, + "derivative min": 0 if d in (0, 1) else -20, + "derivative max": 20, "derivative equals": np.nan, + "derivative down efficiency": 0.9, + "derivative up efficiency": 0.9, }, index=index, ) @@ -44,8 +60,8 @@ def test_multi_feed_device_scheduler_shared_buffer(): # ---- no EMS-level constraints ems_constraints = pd.DataFrame( { - "derivative min": -np.inf, - "derivative max": np.inf, + "derivative min": -40, + "derivative max": 40, }, index=index, ) @@ -53,9 +69,36 @@ def test_multi_feed_device_scheduler_shared_buffer(): # ---- shared buffer max = 100 (soft) max_soc = 100.0 breach_price = 1_000.0 + min_soc = pd.Series(0, index=index) + min_soc[-1] = 100 + + # default commodity: electricity + # choice: electricity or gas + gas_price = pd.Series(300, index=index) + electricity_price = pd.Series(600, index=index, name="event_value") + electricity_price.iloc[12:14] = 200 + prices = {"gas": gas_price, "electricity": electricity_price} + + sloped_prices = ( + add_tiny_price_slope(electricity_price.to_frame()) + - electricity_price.to_frame() + ) commitments = [] - for d,dev in enumerate(devices): + + # todo: fix this commitment for the group[boiler, heat pump] to reach 100 kW together + commitments.append( + StockCommitment( + name="buffer min", + index=index, + quantity=min_soc, + upwards_deviation_price=0, + downwards_deviation_price=-breach_price, + device=None, + device_group=device_group, + ) + ) + for d, dev in enumerate(devices): commitments.append( StockCommitment( name="buffer max", @@ -67,6 +110,29 @@ def test_multi_feed_device_scheduler_shared_buffer(): device_group=device_group, ) ) + commitments.append( + FlowCommitment( + name=device_commodity[d], + index=index, + quantity=0, + upwards_deviation_price=prices[device_commodity[d]], + downwards_deviation_price=prices[device_commodity[d]], + device=pd.Series(d, index=index), + device_group=device_commodity, + ) + ) + + commitments.append( + FlowCommitment( + name="preferred_charge_sooner", + index=index, + quantity=0, + upwards_deviation_price=sloped_prices, + downwards_deviation_price=sloped_prices, + device=pd.Series(d, index=index), + device_group=device_commodity, + ) + ) # ---- run scheduler planned_power, planned_costs, results, model = device_scheduler( @@ -85,6 +151,17 @@ def test_multi_feed_device_scheduler_shared_buffer(): # # i.e. NOT three (which would indicate per-device baselines) commitment_groups = set(commitments[0].device_group.values) + + # commitment_costs = [ + # { + # "name": "commitment_costs", + # "data": { + # c.name: costs + # for c, costs in zip(commitments, model.commitment_costs.values()) + # }, + # }, + # ] + assert commitment_groups == {"shared thermal buffer"} # ---- key behavioural check: From 81bb9ecd47e9d50512b18c04d7cafb0c9fcd6b63 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Tue, 27 Jan 2026 20:19:44 +0100 Subject: [PATCH 15/59] fix: get unique list of devices for a frame column Signed-off-by: Ahmad-Wahid --- .../data/models/planning/linear_optimization.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 38caeb127e..37bada383b 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -213,15 +213,20 @@ def convert_commitments_to_subcommitments( if "device_group" not in df.columns or "device" not in df.columns: continue - device_group_lookup[c] = {} + rows = df[["device", "device_group"]].dropna() - # keep only relevant rows - rows = df[["device", "device_group"]].dropna().drop_duplicates() + device_group_lookup[c] = {} for _, row in rows.iterrows(): g = row["device_group"] - d = row["device"] # NOTE: must match model.d indexing - device_group_lookup[c].setdefault(g, set()).add(d) + d = row["device"] + + if isinstance(d, (list, tuple, set, np.ndarray)): + devices = set(d) + else: + devices = {d} + + device_group_lookup[c].setdefault(g, set()).update(devices) # Oversimplified check for a convex cost curve df = pd.concat(commitments)[ From 334e4d38b289097a1d3cbf5f91f9c446c5013847 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Tue, 27 Jan 2026 20:23:28 +0100 Subject: [PATCH 16/59] fix: create util functions that extract devices for a list of values and map them to the respective group id Signed-off-by: Ahmad-Wahid --- flexmeasures/data/models/planning/__init__.py | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py index df7e6dbd3f..a1e7cfa747 100644 --- a/flexmeasures/data/models/planning/__init__.py +++ b/flexmeasures/data/models/planning/__init__.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta from tabulate import tabulate from typing import Any, Dict, List, Type, Union - +from collections.abc import Iterable import pandas as pd from flask import current_app @@ -351,7 +351,7 @@ def _init_device_group(self): # Extract device universe if isinstance(self.device, pd.Series): - devices = self.device.unique() + devices = extract_devices(self.device) else: devices = [self.device] @@ -414,7 +414,7 @@ def to_frame(self) -> pd.DataFrame: ) # map device → device_group if self.device is not None: - df["device_group"] = self.device.map(self.device_group) + df["device_group"] = map_device_to_group(self.device, self.device_group) else: df["device_group"] = 0 @@ -440,3 +440,48 @@ class StockCommitment(Commitment): Scheduler.compute_schedule = deprecated(Scheduler.compute, "0.14")( Scheduler.compute_schedule ) + + +def extract_devices(device): + """ + Return a flat list of unique device identifiers from: + - scalar device + - Series of scalars + - Series of iterables (e.g. [0, 1]) + """ + if device is None: + return [] + + if isinstance(device, pd.Series): + values = device.dropna().values + else: + values = [device] + + devices = set() + for v in values: + if isinstance(v, Iterable) and not isinstance(v, (str, bytes)): + devices.update(v) + else: + devices.add(v) + + return list(devices) + + +def map_device_to_group(device_series, device_group_map): + """ + Map device identifiers to device_group. + + - scalar device → group label + - iterable of devices → group label (must be identical) + """ + + def resolve(v): + if isinstance(v, (list, tuple, set)): + groups = {device_group_map[d] for d in v} + if len(groups) != 1: + raise ValueError(f"Devices {v} map to multiple device groups: {groups}") + return groups.pop() + else: + return device_group_map[v] + + return device_series.apply(resolve) From f738d9f32a9604f72cf1ad92f8e466f4df9ea062 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Tue, 27 Jan 2026 20:25:13 +0100 Subject: [PATCH 17/59] fix: create a series for a list of grouped devices Signed-off-by: Ahmad-Wahid --- flexmeasures/data/models/planning/tests/test_commitments.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 26349f0ff6..fd67097da6 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -86,7 +86,6 @@ def test_multi_feed_device_scheduler_shared_buffer(): commitments = [] - # todo: fix this commitment for the group[boiler, heat pump] to reach 100 kW together commitments.append( StockCommitment( name="buffer min", @@ -94,7 +93,9 @@ def test_multi_feed_device_scheduler_shared_buffer(): quantity=min_soc, upwards_deviation_price=0, downwards_deviation_price=-breach_price, - device=None, + # instead of device=None, I considered to create a series for the devices that we need for this + # specific commitment. + device=pd.Series([[0, 1]] * len(index), index=index), device_group=device_group, ) ) From 620c6dc350b278053f9c79b5c01559517e1ea819 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Sat, 31 Jan 2026 17:23:27 +0100 Subject: [PATCH 18/59] drop outdated comments --- flexmeasures/data/models/planning/tests/test_commitments.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index fd67097da6..8d4df6e720 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -36,7 +36,6 @@ def test_multi_feed_device_scheduler_shared_buffer(): ) equals = pd.Series(np.nan, index=index) equals[-1] = 100 - # ---- trivial device constraints (stocks unconstrained individually) device_constraints = [] for d, device_name in enumerate(devices): # 0 and 1 : derivative min 0 @@ -57,7 +56,6 @@ def test_multi_feed_device_scheduler_shared_buffer(): ) device_constraints.append(df) - # ---- no EMS-level constraints ems_constraints = pd.DataFrame( { "derivative min": -40, From 40eb747b61321d1d47a9c448fcee2a9ca4ac0ca6 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Sat, 31 Jan 2026 19:59:01 +0100 Subject: [PATCH 19/59] use commitment costs and add asserts for electricity and gas Signed-off-by: Ahmad-Wahid --- .../models/planning/tests/test_commitments.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 8d4df6e720..078ae3d502 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -151,15 +151,15 @@ def test_multi_feed_device_scheduler_shared_buffer(): # i.e. NOT three (which would indicate per-device baselines) commitment_groups = set(commitments[0].device_group.values) - # commitment_costs = [ - # { - # "name": "commitment_costs", - # "data": { - # c.name: costs - # for c, costs in zip(commitments, model.commitment_costs.values()) - # }, - # }, - # ] + commitment_costs = { + "name": "commitment_costs", + "data": { + c.name: costs + for c, costs in zip(commitments, model.commitment_costs.values()) + }, + } + assert commitment_costs["data"]["electricity"] == -11440.0 + assert round(commitment_costs["data"]["gas"], 2) == 21333.33 assert commitment_groups == {"shared thermal buffer"} From 708d86949420d6662c06818c4fe40e6cf4b5d046 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Sat, 31 Jan 2026 21:47:09 +0100 Subject: [PATCH 20/59] and an assert for commodity costs Signed-off-by: Ahmad-Wahid --- .../data/models/planning/tests/test_commitments.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 078ae3d502..56179ff8f2 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -150,6 +150,12 @@ def test_multi_feed_device_scheduler_shared_buffer(): # # i.e. NOT three (which would indicate per-device baselines) commitment_groups = set(commitments[0].device_group.values) + commodity_commitments = { + c.name + for c in commitments + if isinstance(c, FlowCommitment) and c.name in {"gas", "electricity"} + } + assert commodity_commitments == {"gas", "electricity"} commitment_costs = { "name": "commitment_costs", @@ -158,8 +164,10 @@ def test_multi_feed_device_scheduler_shared_buffer(): for c, costs in zip(commitments, model.commitment_costs.values()) }, } - assert commitment_costs["data"]["electricity"] == -11440.0 - assert round(commitment_costs["data"]["gas"], 2) == 21333.33 + commodity_costs = { + k: v for k, v in commitment_costs["data"].items() if k in {"gas", "electricity"} + } + assert set(commodity_costs.keys()) == {"gas", "electricity"} assert commitment_groups == {"shared thermal buffer"} From ce307f82474d690cb115d2664447cc344bc96666 Mon Sep 17 00:00:00 2001 From: Ahmad-Wahid Date: Sat, 31 Jan 2026 22:13:22 +0100 Subject: [PATCH 21/59] add an extra assert on costs Signed-off-by: Ahmad-Wahid --- flexmeasures/data/models/planning/tests/test_commitments.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 56179ff8f2..639813a090 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -177,3 +177,5 @@ def test_multi_feed_device_scheduler_shared_buffer(): # If baselines were duplicated, cost would be ~2x for the shared buffer. expected_max_cost = len(index) * breach_price * 2 assert planned_costs <= expected_max_cost + total_commodity_cost = sum(commodity_costs.values()) + assert total_commodity_cost <= planned_costs From 82f807ec91da001378a9a7a822e5e03f4bb59ccc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Mar 2026 16:21:40 +0100 Subject: [PATCH 22/59] fix: wrong timezone; the test relied on the preference to charge sooner and discharge later, rather than on the EPEX price transition, as the inline test documentation advertised Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_solver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 919936d315..1f0a5baa27 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -2235,11 +2235,14 @@ def test_battery_storage_different_units( battery_name="Test battery", power_sensor_name=power_sensor_name, ) - tz = pytz.timezone("Europe/Amsterdam") + tz = pytz.timezone(epex_da.timezone) # transition from cheap to expensive (90 -> 100) start = tz.localize(datetime(2015, 1, 2, 14, 0, 0)) end = tz.localize(datetime(2015, 1, 2, 16, 0, 0)) + assert len(epex_da.search_beliefs(start, end)) == 2 + assert epex_da.search_beliefs(start, end).values[0][0] == 90 + assert epex_da.search_beliefs(start, end).values[1][0] == 100 resolution = timedelta(minutes=15) flex_model = { From 128550f5b03f76e50bae7ec8f87b31162fb25d9d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Mar 2026 17:17:24 +0100 Subject: [PATCH 23/59] feat: move preference to charge sooner and discharge later into a StockCommitment to prefer being full Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 35 +++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 6fb7aa5e94..dd53db19fc 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -198,17 +198,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 start = pd.Timestamp(start).tz_convert("UTC") end = pd.Timestamp(end).tz_convert("UTC") - # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now. - # We penalise future consumption and reward future production with at most 1 per thousand times the energy price spread. - # todo: move to flow or stock commitment per device - if any(prefer_charging_sooner): - up_deviation_prices = add_tiny_price_slope( - up_deviation_prices, "event_value" - ) - down_deviation_prices = add_tiny_price_slope( - down_deviation_prices, "event_value" - ) - # Create Series with EMS capacities ems_power_capacity_in_mw = get_continuous_series_sensor_or_quantity( variable_quantity=self.flex_context.get("ems_power_capacity_in_mw"), @@ -463,6 +452,28 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) commitments.append(commitment) + # Use a tiny price slope to prefer a fuller SoC sooner rather than later + # This corresponds to a preference for charging now rather than later, and discharging later rather than now. + # We penalise future consumption and reward future production with at most 1 per thousand times the energy price spread. + for d, prefer_charging_sooner_d in enumerate(prefer_charging_sooner): + if prefer_charging_sooner_d: + tiny_price_slope = ( + add_tiny_price_slope(up_deviation_prices, "event_value") + - up_deviation_prices + ) + commitment = StockCommitment( + name=f"prefer charging device {d} sooner", + quantity=soc_max[d] - soc_at_start[d], + # Prefer curtailing consumption later by penalizing later consumption + upwards_deviation_price=0, + # Prefer curtailing production later by penalizing later production + downwards_deviation_price=-0.00000001, + # downwards_deviation_price=-tiny_price_slope / 1000000,#0.00000001, + index=index, + device=d, + ) + commitments.append(commitment) + # Set up device constraints: scheduled flexible devices for this EMS (from index 0 to D-1), plus the forecasted inflexible devices (at indices D to n). device_constraints = [ initialize_df(StorageScheduler.COLUMNS, start, end, resolution) @@ -940,7 +951,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 def convert_to_commitments( self, **timing_kwargs, - ) -> list[FlowCommitment]: + ) -> list[FlowCommitment | StockCommitment]: """Convert list of commitment specifications (dicts) to a list of FlowCommitments.""" commitment_specs = self.flex_context.get("commitments", []) if len(commitment_specs) == 0: From 130b9dd6528291903cb7582d2763e843ca94241b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Mar 2026 17:38:42 +0100 Subject: [PATCH 24/59] fix: test case no longer relies on arbitrage opportunity coming from artificial price slope Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_solver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 1f0a5baa27..d85a45243b 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -816,8 +816,8 @@ def compute_schedule(flex_model): # soc maxima and soc minima soc_maxima = [ - {"datetime": "2015-01-02T15:00:00+01:00", "value": 1.0}, - {"datetime": "2015-01-02T16:00:00+01:00", "value": 1.0}, + {"datetime": "2015-01-02T12:00:00+01:00", "value": 1.0}, + {"datetime": "2015-01-02T13:00:00+01:00", "value": 1.0}, ] soc_minima = [{"datetime": "2015-01-02T08:00:00+01:00", "value": 3.5}] @@ -853,7 +853,7 @@ def compute_schedule(flex_model): # test for soc_maxima # check that the local maximum constraint is respected - assert soc_schedule_2.loc["2015-01-02T15:00:00+01:00"] <= 1.0 + assert soc_schedule_2.loc["2015-01-02T13:00:00+01:00"] <= 1.0 # test for soc_targets # check that the SOC target (at 19 pm, local time) is met From 1eab8282406d85d343ed6b22f73f781f68e45777 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Mar 2026 17:40:42 +0100 Subject: [PATCH 25/59] feat: check for optimal schedule Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_solver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index d85a45243b..6ee32f8b59 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -270,6 +270,7 @@ def run_test_charge_discharge_sign( for soc_at_start_d in soc_at_start ], ) + assert results.solver.termination_condition == "optimal" device_power_sign = pd.Series(model.device_power_sign.extract_values())[0] device_power_up = pd.Series(model.device_power_up.extract_values())[0] From b9bc4a9c0786907c25737a2b5c20492501cf79ec Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Mar 2026 17:58:51 +0100 Subject: [PATCH 26/59] feat: prefer a full storage earlier over later Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 6 ++++-- flexmeasures/data/models/planning/utils.py | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index dd53db19fc..b473ed650a 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -458,7 +458,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 for d, prefer_charging_sooner_d in enumerate(prefer_charging_sooner): if prefer_charging_sooner_d: tiny_price_slope = ( - add_tiny_price_slope(up_deviation_prices, "event_value") + add_tiny_price_slope( + up_deviation_prices, "event_value", order="desc" + ) - up_deviation_prices ) commitment = StockCommitment( @@ -467,7 +469,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # Prefer curtailing consumption later by penalizing later consumption upwards_deviation_price=0, # Prefer curtailing production later by penalizing later production - downwards_deviation_price=-0.00000001, + downwards_deviation_price=-tiny_price_slope, # downwards_deviation_price=-tiny_price_slope / 1000000,#0.00000001, index=index, device=d, diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 549ef6a01a..c7e20860f0 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -2,6 +2,7 @@ from packaging import version from datetime import date, datetime, timedelta +from typing import Literal from flask import current_app import pandas as pd @@ -67,7 +68,10 @@ def initialize_index( def add_tiny_price_slope( - orig_prices: pd.DataFrame, col_name: str = "event_value", d: float = 10**-4 + orig_prices: pd.DataFrame, + col_name: str = "event_value", + d: float = 10**-4, + order: Literal["asc", "desc"] = "asc", ) -> pd.DataFrame: """Add tiny price slope to col_name to represent e.g. inflation as a simple linear price increase. This is meant to break ties, when multiple time slots have equal prices, in favour of acting sooner. @@ -79,9 +83,16 @@ def add_tiny_price_slope( max_penalty = price_spread * d else: max_penalty = d - prices[col_name] = prices[col_name] + np.linspace( - 0, max_penalty, prices[col_name].size - ) + if order == "asc": + prices[col_name] = prices[col_name] + np.linspace( + 0, max_penalty, prices[col_name].size + ) + elif order == "desc": + prices[col_name] = prices[col_name] + np.linspace( + max_penalty, 0, prices[col_name].size + ) + else: + raise ValueError("order must be 'asc' or 'desc'") return prices From 57df5c31d9f6e3e9a868c3e35bff4db7fda2d003 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Mar 2026 18:04:18 +0100 Subject: [PATCH 27/59] docs: update commitment name and inline comments Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index b473ed650a..3c2388d435 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -464,11 +464,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 - up_deviation_prices ) commitment = StockCommitment( - name=f"prefer charging device {d} sooner", + name=f"prefer a full storage {d} sooner", quantity=soc_max[d] - soc_at_start[d], - # Prefer curtailing consumption later by penalizing later consumption upwards_deviation_price=0, - # Prefer curtailing production later by penalizing later production + # Penalize not being full, with lower penalties later downwards_deviation_price=-tiny_price_slope, # downwards_deviation_price=-tiny_price_slope / 1000000,#0.00000001, index=index, From ce71637238b1d4c1eb81a400edb1fce9a7c9a67c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 14 Mar 2026 11:30:46 +0100 Subject: [PATCH 28/59] docs: touch up test explanation Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_solver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 6ee32f8b59..b46ee0f0b1 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1788,7 +1788,7 @@ def test_battery_stock_delta_sensor( - Battery of size 2 MWh. - Consumption capacity of the battery is 2 MW. - The battery cannot discharge. - With these settings, the battery needs to charge at a power or greater than the usage forecast + With these settings, the battery needs to charge at a power equal or greater than the usage forecast to keep the SOC within bounds ([0, 2 MWh]). """ _, battery = get_sensors_from_db(db, add_battery_assets) From f6183df2df24639858907f7ae681636673d8140b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 14 Mar 2026 11:43:55 +0100 Subject: [PATCH 29/59] fix: update test case given preference for a full battery Signed-off-by: F.N. Claessen --- .../data/models/planning/tests/test_solver.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index b46ee0f0b1..e810e8c854 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1791,7 +1791,7 @@ def test_battery_stock_delta_sensor( With these settings, the battery needs to charge at a power equal or greater than the usage forecast to keep the SOC within bounds ([0, 2 MWh]). """ - _, battery = get_sensors_from_db(db, add_battery_assets) + epex_da, battery = get_sensors_from_db(db, add_battery_assets) tz = pytz.timezone("Europe/Amsterdam") start = tz.localize(datetime(2015, 1, 1)) end = tz.localize(datetime(2015, 1, 2)) @@ -1836,9 +1836,21 @@ def test_battery_stock_delta_sensor( with pytest.raises(InfeasibleProblemException): scheduler.compute() elif stock_delta_sensor is None: - # No usage -> the battery does not charge + # No usage -> the battery only charges when energy is free + free_hour = "2015-01-01 17:00:00+00:00" + prices = epex_da.search_beliefs(start, end) + zero_prices = prices[prices.event_value == 0] + assert len(zero_prices) == 1, "this test assumes a single hour of free energy" + assert ( + len(zero_prices[zero_prices.event_starts == free_hour]) == 1 + ), "this test assumes free energy from 5 to 6 PM UTC" schedule = scheduler.compute() - assert all(schedule == 0) + assert all( + schedule[schedule.index != free_hour] == 0 + ), "no charging expected when energy is not free, given no soc-usage" + assert all( + schedule[schedule.index == free_hour] == capacity + ), "max charging expected when energy is free, because of preference to have a full SoC" else: # Some usage -> the battery needs to charge schedule = scheduler.compute() From ed471a8b836c60da2964e2a666c4613db98bcf5d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 14 Mar 2026 11:46:52 +0100 Subject: [PATCH 30/59] delete: clean up comment Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 3c2388d435..4d1e551c95 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -469,7 +469,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 upwards_deviation_price=0, # Penalize not being full, with lower penalties later downwards_deviation_price=-tiny_price_slope, - # downwards_deviation_price=-tiny_price_slope / 1000000,#0.00000001, index=index, device=d, ) From 7611fb9eadbde61efa317f8ace3e7adbb9d971de Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 14 Mar 2026 11:59:56 +0100 Subject: [PATCH 31/59] feat: model the preference to curtail later within the same StockCommitment, using a tiny price slope to prefer a fuller SoC sooner rather than later, by lowering penalties later Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 38 +++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4d1e551c95..44024c0cb8 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -430,32 +430,13 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # Take the contracted capacity as a hard constraint ems_constraints["derivative min"] = ems_production_capacity - # Flow commitments per device + # Commitments per device - # Add tiny price slope to prefer curtailing later rather than now. - # The price slope is half of the slope to prefer charging sooner - for d, prefer_curtailing_later_d in enumerate(prefer_curtailing_later): - if prefer_curtailing_later_d: - tiny_price_slope = ( - add_tiny_price_slope(up_deviation_prices, "event_value") - - up_deviation_prices - ) - tiny_price_slope *= 0.5 - commitment = FlowCommitment( - name=f"prefer curtailing device {d} later", - # Prefer curtailing consumption later by penalizing later consumption - upwards_deviation_price=tiny_price_slope, - # Prefer curtailing production later by penalizing later production - downwards_deviation_price=-tiny_price_slope, - index=index, - device=d, - ) - commitments.append(commitment) - - # Use a tiny price slope to prefer a fuller SoC sooner rather than later + # StockCommitment per device to prefer a full storage by penalizing not being full # This corresponds to a preference for charging now rather than later, and discharging later rather than now. - # We penalise future consumption and reward future production with at most 1 per thousand times the energy price spread. - for d, prefer_charging_sooner_d in enumerate(prefer_charging_sooner): + for d, (prefer_charging_sooner_d, prefer_curtailing_later_d) in enumerate( + zip(prefer_charging_sooner, prefer_curtailing_later) + ): if prefer_charging_sooner_d: tiny_price_slope = ( add_tiny_price_slope( @@ -463,12 +444,17 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) - up_deviation_prices ) + if prefer_curtailing_later: + # Use a tiny price slope to prefer a fuller SoC sooner rather than later, by lowering penalties later + penalty = tiny_price_slope + else: + # Constant penalty + penalty = tiny_price_slope[0] commitment = StockCommitment( name=f"prefer a full storage {d} sooner", quantity=soc_max[d] - soc_at_start[d], upwards_deviation_price=0, - # Penalize not being full, with lower penalties later - downwards_deviation_price=-tiny_price_slope, + downwards_deviation_price=-penalty, index=index, device=d, ) From bf16e63606ed5a2889fe25b45c0fd4f28a40b486 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 14 Mar 2026 12:17:35 +0100 Subject: [PATCH 32/59] fix: reduce tiny price slope Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 44024c0cb8..5ef5517b36 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -440,7 +440,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if prefer_charging_sooner_d: tiny_price_slope = ( add_tiny_price_slope( - up_deviation_prices, "event_value", order="desc" + up_deviation_prices, "event_value", d=10**-7, order="desc" ) - up_deviation_prices ) From 06c30dc297b85f266b47e990523ccd4b90b28e1f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Mar 2026 13:00:41 +0100 Subject: [PATCH 33/59] docs: delete duplicate changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 0890c4d01a..4c768d68bd 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -68,7 +68,6 @@ New features * Improved the UX for creating sensors, clicking on ``Enter`` now validates and creates a sensor [see `PR #1876 `_] * Show zero values in bar charts even though they have 0 area [see `PR #1932 `_ and `PR #1936 `_] * Added ``root`` and ``depth`` fields to the `[GET] /assets` endpoint for listing assets, to allow selecting descendants of a given root asset up to a given depth [see `PR #1874 `_] -* Give ability to edit sensor timezone from the UI [see `PR #1900 `_] * Support creating schedules with only information known prior to some time, now also via the CLI (the API already supported it) [see `PR #1871 `_]. * Added capability to update an asset's parent from the UI [`PR #1957 `_] * Add ``fields`` param to the asset-listing endpoints, to save bandwidth in response data [see `PR #1884 `_] From d99089b2844a90667ec3e789fbb6f9d3d26e8c6d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 18 Mar 2026 15:47:47 +0100 Subject: [PATCH 34/59] docs: fix broken link Signed-off-by: F.N. Claessen --- documentation/dev/setup-and-guidelines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/dev/setup-and-guidelines.rst b/documentation/dev/setup-and-guidelines.rst index 3b7ae6e246..d5d9b7458d 100644 --- a/documentation/dev/setup-and-guidelines.rst +++ b/documentation/dev/setup-and-guidelines.rst @@ -63,7 +63,7 @@ On Linux and Windows, everything will be installed using Python packages. On MacOS, this will install all test dependencies, and locally install the HiGHS solver. For this to work, make sure you have `Homebrew `_ installed. -Besides the HiGHS solver (as the current default), the CBC solver is required for tests as well. See `The install instructions `_ for more information. Configuration ^^^^^^^^^^^^^ From cf01f1d16df71c214733062c4b9b75a6df24a9d3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 18 Mar 2026 16:56:39 +0100 Subject: [PATCH 35/59] Revert "fix: reduce tiny price slope" This reverts commit bf16e63606ed5a2889fe25b45c0fd4f28a40b486. Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 5ef5517b36..44024c0cb8 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -440,7 +440,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if prefer_charging_sooner_d: tiny_price_slope = ( add_tiny_price_slope( - up_deviation_prices, "event_value", d=10**-7, order="desc" + up_deviation_prices, "event_value", order="desc" ) - up_deviation_prices ) From bdbdead8337d44d3442a2cf5e05940874a8592aa Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 18 Mar 2026 17:42:53 +0100 Subject: [PATCH 36/59] fix: soc unit conversion Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 44024c0cb8..5a22572311 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -452,7 +452,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 penalty = tiny_price_slope[0] commitment = StockCommitment( name=f"prefer a full storage {d} sooner", - quantity=soc_max[d] - soc_at_start[d], + quantity=(soc_max[d] - soc_at_start[d]) + * (timedelta(hours=1) / resolution), upwards_deviation_price=0, downwards_deviation_price=-penalty, index=index, From f987706a7e70273c3a817b4494e63af137624322 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 18 Mar 2026 17:49:42 +0100 Subject: [PATCH 37/59] fix: adapt test to check for 1 hour of free energy at 15-min scheduling resolution Signed-off-by: F.N. Claessen --- .../data/models/planning/tests/test_solver.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index e810e8c854..a45a3dd2d9 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1838,18 +1838,17 @@ def test_battery_stock_delta_sensor( elif stock_delta_sensor is None: # No usage -> the battery only charges when energy is free free_hour = "2015-01-01 17:00:00+00:00" - prices = epex_da.search_beliefs(start, end) + prices = epex_da.search_beliefs(start, end, resolution=resolution) zero_prices = prices[prices.event_value == 0] - assert len(zero_prices) == 1, "this test assumes a single hour of free energy" - assert ( - len(zero_prices[zero_prices.event_starts == free_hour]) == 1 - ), "this test assumes free energy from 5 to 6 PM UTC" + assert all( + zero_prices.event_starts.hour == pd.Timestamp(free_hour).hour + ), "this test assumes a single hour of free energy from 5 to 6 PM UTC" schedule = scheduler.compute() assert all( - schedule[schedule.index != free_hour] == 0 + schedule[~schedule.index.isin(zero_prices.event_starts)] == 0 ), "no charging expected when energy is not free, given no soc-usage" assert all( - schedule[schedule.index == free_hour] == capacity + schedule[schedule.index.isin(zero_prices.event_starts)] == capacity ), "max charging expected when energy is free, because of preference to have a full SoC" else: # Some usage -> the battery needs to charge From 534179afc876798df1b0fa98b72cc32f178abe9e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 19 Mar 2026 12:27:57 +0100 Subject: [PATCH 38/59] style: black Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_commitments.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 0a0ba8b408..9c14e7037f 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -1,7 +1,11 @@ import pandas as pd import numpy as np -from flexmeasures.data.models.planning import Commitment, StockCommitment, FlowCommitment +from flexmeasures.data.models.planning import ( + Commitment, + StockCommitment, + FlowCommitment, +) from flexmeasures.data.models.planning.utils import ( initialize_index, add_tiny_price_slope, From 05aed7e2939b93065dc9b31058254c0a8909cfb4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 Mar 2026 12:20:22 +0100 Subject: [PATCH 39/59] fix: check curtailment preference per distinct device Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 5a22572311..dc8335e885 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -444,7 +444,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) - up_deviation_prices ) - if prefer_curtailing_later: + if prefer_curtailing_later_d: # Use a tiny price slope to prefer a fuller SoC sooner rather than later, by lowering penalties later penalty = tiny_price_slope else: From e55f638097a5e9b70f4bc4ac8081d769274891ce Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 Mar 2026 14:20:56 +0100 Subject: [PATCH 40/59] fix: set tight tolerance for HiGHS solver Signed-off-by: F.N. Claessen --- .../data/models/planning/linear_optimization.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 005940456e..0cc5e1ea65 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -626,12 +626,26 @@ def cost_function(m): if cbc_path is not None: solver.set_executable(cbc_path) + # Set tight tolerance for HiGHS solver + profile = {} + if "highs" in solver_name.lower(): + profile = { + "mip_rel_gap": "0", + "mip_abs_gap": "0", + "primal_feasibility_tolerance": "1e-9", + "dual_feasibility_tolerance": "1e-9", + "mip_feasibility_tolerance": "1e-9", + } + # disable logs for the HiGHS solver in case that LOGGING_LEVEL is INFO if current_app.config["LOGGING_LEVEL"] == "INFO" and ( "highs" in solver_name.lower() ): solver.options["output_flag"] = "false" + for option_name, option_value in profile.items(): + solver.options[option_name] = option_value + # load_solutions=False to avoid a RuntimeError exception in appsi solvers when solving an infeasible problem. results = solver.solve(model, load_solutions=False) From 764712f8fc0aeca4128f494978d53d54abe20fb3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 Mar 2026 14:21:44 +0100 Subject: [PATCH 41/59] refactor: merge if-blocks Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/linear_optimization.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 0cc5e1ea65..93b919d6bb 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -636,12 +636,9 @@ def cost_function(m): "dual_feasibility_tolerance": "1e-9", "mip_feasibility_tolerance": "1e-9", } - - # disable logs for the HiGHS solver in case that LOGGING_LEVEL is INFO - if current_app.config["LOGGING_LEVEL"] == "INFO" and ( - "highs" in solver_name.lower() - ): - solver.options["output_flag"] = "false" + # disable logs for the HiGHS solver in case that LOGGING_LEVEL is INFO + if current_app.config["LOGGING_LEVEL"] == "INFO": + profile["output_flag"] = "false" for option_name, option_value in profile.items(): solver.options[option_name] = option_value From 9070eaea798346278d855980f322608855375a1d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 Mar 2026 09:33:57 +0100 Subject: [PATCH 42/59] fix: use iloc Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index dc8335e885..ebf03c21ed 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -449,7 +449,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 penalty = tiny_price_slope else: # Constant penalty - penalty = tiny_price_slope[0] + penalty = tiny_price_slope.iloc[0][0] commitment = StockCommitment( name=f"prefer a full storage {d} sooner", quantity=(soc_max[d] - soc_at_start[d]) From 857e9c18504efa60249c0236eda9c59298d77552 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 Mar 2026 09:45:26 +0100 Subject: [PATCH 43/59] fix: diminish tiny price slope by number of planning steps Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index ebf03c21ed..cfc74c200e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -440,7 +440,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if prefer_charging_sooner_d: tiny_price_slope = ( add_tiny_price_slope( - up_deviation_prices, "event_value", order="desc" + up_deviation_prices, + "event_value", + d=10**-4 / len(index), + order="desc", ) - up_deviation_prices ) From 54c9c36ee9dcfb61ba5842e266116cc54d0eb5df Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 Mar 2026 09:48:58 +0100 Subject: [PATCH 44/59] refactor: always diminish tiny price slope by number of planning steps, such that its relative weight does not grow with the number of steps Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 5 +---- flexmeasures/data/models/planning/utils.py | 7 ++++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index cfc74c200e..ebf03c21ed 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -440,10 +440,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if prefer_charging_sooner_d: tiny_price_slope = ( add_tiny_price_slope( - up_deviation_prices, - "event_value", - d=10**-4 / len(index), - order="desc", + up_deviation_prices, "event_value", order="desc" ) - up_deviation_prices ) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index c7e20860f0..74d01c15b5 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -75,14 +75,15 @@ def add_tiny_price_slope( ) -> pd.DataFrame: """Add tiny price slope to col_name to represent e.g. inflation as a simple linear price increase. This is meant to break ties, when multiple time slots have equal prices, in favour of acting sooner. - We penalise the future with at most d times the price spread (1 per thousand by default). + We penalise the future with at most d times the price spread (1 per thousand by default), + divided over the number of planning steps. """ prices = orig_prices.copy() price_spread = prices[col_name].max() - prices[col_name].min() if price_spread > 0: - max_penalty = price_spread * d + max_penalty = price_spread * d / len(prices) else: - max_penalty = d + max_penalty = d / len(prices) if order == "asc": prices[col_name] = prices[col_name] + np.linspace( 0, max_penalty, prices[col_name].size From 158c7a0ed41b799877a09742c8bffb7ac26b77d0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 Mar 2026 09:50:41 +0100 Subject: [PATCH 45/59] chore: increment StorageScheduler version Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index ebf03c21ed..2a37cd600e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1291,7 +1291,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: class StorageScheduler(MetaStorageScheduler): - __version__ = "7" + __version__ = "8" __author__ = "Seita" fallback_scheduler_class: Type[Scheduler] = StorageFallbackScheduler From 49df3a64ed15eeda3d1136189b7b546d5da31a40 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 17:50:27 +0200 Subject: [PATCH 46/59] feat: rewrite test to not rely on multi-commodity Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_commitments.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 9c14e7037f..e507942bc9 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -185,6 +185,145 @@ def test_multi_feed_device_scheduler_shared_buffer(): assert total_commodity_cost <= planned_costs +def test_device_group_shared_buffer(): + """ + Two devices (heat pumps) jointly charge a shared thermal buffer. + A single StockCommitment targets the *combined* stock of both devices + (via device_group), so the deviation penalty is applied once per group, + not once per device. + + Key behaviour under test: + - The combined stock of devices 0+1 must reach min_soc=100 by the last slot. + - The optimizer is free to split the load between the two devices. + - The total breach cost is bounded by 1 breach per timestep (not 2), + confirming that device_group prevents cost duplication. + """ + # ---- time + start = pd.Timestamp("2026-01-01T00:00+01") + end = pd.Timestamp("2026-01-02T00:00+01") + resolution = pd.Timedelta("PT1H") + index = initialize_index(start=start, end=end, resolution=resolution) + n = len(index) + + # ---- two electricity devices feeding a shared thermal buffer, plus a battery + # device 0: heat pump A (charge only) + # device 1: heat pump B (charge only) + # device 2: battery (charge and discharge) + device_group = pd.Series( + { + 0: "shared thermal buffer", + 1: "shared thermal buffer", + 2: "battery SoC", + } + ) + + # ---- device constraints + device_constraints = [] + for d in range(3): + df = pd.DataFrame( + { + "min": 0, + "max": 100, + "equals": np.nan, + "derivative min": 0 if d in (0, 1) else -20, # HPs: charge only + "derivative max": 20, + "derivative equals": np.nan, + "derivative down efficiency": 0.9, + "derivative up efficiency": 0.9, + }, + index=index, + ) + device_constraints.append(df) + + ems_constraints = pd.DataFrame( + {"derivative min": -40, "derivative max": 40}, + index=index, + ) + + # ---- shared buffer must reach 100 kWh by the last slot (soft lower bound) + breach_price = 1_000.0 + min_soc = pd.Series(0, index=index) + min_soc.iloc[-1] = 100 + + # A uniform energy price gives the optimizer a cost signal without commodity logic. + energy_price = pd.Series(100, index=index) + + # ---- commitments + commitments = [] + + # StockCommitment 1: combined stock of devices 0+1 must reach min_soc + # device=[[0,1], [0,1], ...] selects both HP devices per timestep + commitments.append( + StockCommitment( + name="buffer min", + index=index, + quantity=min_soc, + upwards_deviation_price=0, + downwards_deviation_price=-breach_price, + device=pd.Series([[0, 1]] * n, index=index), + device_group=device_group, + ) + ) + + # StockCommitment 2+3+4: individual upper bounds (each device's stock <= 100) + for d in range(3): + commitments.append( + StockCommitment( + name=f"buffer max {d}", + index=index, + quantity=100.0, + upwards_deviation_price=breach_price, + downwards_deviation_price=0, + device=pd.Series(d, index=index), + device_group=device_group, + ) + ) + + # FlowCommitment: uniform energy price for each device + for d in range(3): + commitments.append( + FlowCommitment( + name="energy", + index=index, + quantity=0, + upwards_deviation_price=energy_price, + downwards_deviation_price=energy_price, + device=pd.Series(d, index=index), + device_group=device_group, + ) + ) + + # ---- run + planned_power, planned_costs, results, model = device_scheduler( + device_constraints=device_constraints, + ems_constraints=ems_constraints, + commitments=commitments, + initial_stock=0, + ) + + # ---- solved + assert results.solver.termination_condition in ("optimal", "locallyOptimal") + + # ---- device_group structure: two groups + assert set(device_group.values) == {"shared thermal buffer", "battery SoC"} + + # ---- KEY: the "buffer min" commitment cost must be zero + # Both HPs together can supply up to 2×20 kW×24 h×0.9 ≈ 864 kWh, + # so reaching 100 kWh is always feasible at zero breach cost. + # If device_group were NOT applied (separate commitment per device), + # the cost would be non-zero because each device alone cannot satisfy + # the full 100 kWh target in the model. + buffer_min_costs = sum( + v + for c, v in zip(commitments, model.commitment_costs.values()) + if c.name == "buffer min" + ) + assert buffer_min_costs == 0, ( + "Buffer min commitment should be satisfied at zero cost " + "(both HPs together can easily reach 100 kWh)" + ) + + def make_index(n: int = 5) -> pd.DatetimeIndex: """ Create a simple hourly DatetimeIndex for testing. From a5184371c2e174a06b572b7e137726af0335d34e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 17:56:32 +0200 Subject: [PATCH 47/59] feat: rewrite test to prove that only when both HPs share a single commitment does the optimizer treat their stocks as a combined resource Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_commitments.py | 151 ++++++++++-------- 1 file changed, 88 insertions(+), 63 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index e507942bc9..14f7436afb 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -185,30 +185,23 @@ def test_multi_feed_device_scheduler_shared_buffer(): assert total_commodity_cost <= planned_costs -def test_device_group_shared_buffer(): +def _run_hp_buffer_scenario(index, target_soc, shared: bool): """ - Two devices (heat pumps) jointly charge a shared thermal buffer. - A single StockCommitment targets the *combined* stock of both devices - (via device_group), so the deviation penalty is applied once per group, - not once per device. - - Key behaviour under test: - - The combined stock of devices 0+1 must reach min_soc=100 by the last slot. - - The optimizer is free to split the load between the two devices. - - The total breach cost is bounded by 1 breach per timestep (not 2), - confirming that device_group prevents cost duplication. + Helper: run the two-heat-pump scheduler with either a shared or per-device buffer + commitment and return the total "buffer min" breach cost. + + Each heat pump (device 0 and 1) can supply at most + derivative_max × hours × efficiency = 20 kW × 24 h × 0.9 = 432 kWh. + + shared=True → one StockCommitment on the *combined* stock of devices 0+1 + (maximum reachable: 864 kWh). + shared=False → two separate StockCommitments, one per HP, each requiring + target_soc on its *own* stock (maximum per device: 432 kWh). """ - # ---- time - start = pd.Timestamp("2026-01-01T00:00+01") - end = pd.Timestamp("2026-01-02T00:00+01") - resolution = pd.Timedelta("PT1H") - index = initialize_index(start=start, end=end, resolution=resolution) n = len(index) + breach_price = 1_000.0 + energy_price = pd.Series(100, index=index) - # ---- two electricity devices feeding a shared thermal buffer, plus a battery - # device 0: heat pump A (charge only) - # device 1: heat pump B (charge only) - # device 2: battery (charge and discharge) device_group = pd.Series( { 0: "shared thermal buffer", @@ -218,14 +211,17 @@ def test_device_group_shared_buffer(): ) # ---- device constraints + # device 0: heat pump A (charge only) + # device 1: heat pump B (charge only) + # device 2: battery (charge and discharge) device_constraints = [] for d in range(3): df = pd.DataFrame( { "min": 0, - "max": 100, + "max": 500, "equals": np.nan, - "derivative min": 0 if d in (0, 1) else -20, # HPs: charge only + "derivative min": 0 if d in (0, 1) else -20, "derivative max": 20, "derivative equals": np.nan, "derivative down efficiency": 0.9, @@ -240,47 +236,52 @@ def test_device_group_shared_buffer(): index=index, ) - # ---- shared buffer must reach 100 kWh by the last slot (soft lower bound) - breach_price = 1_000.0 - min_soc = pd.Series(0, index=index) - min_soc.iloc[-1] = 100 - - # A uniform energy price gives the optimizer a cost signal without commodity logic. - energy_price = pd.Series(100, index=index) + min_soc = pd.Series(0.0, index=index) + min_soc.iloc[-1] = target_soc - # ---- commitments commitments = [] - # StockCommitment 1: combined stock of devices 0+1 must reach min_soc - # device=[[0,1], [0,1], ...] selects both HP devices per timestep - commitments.append( - StockCommitment( - name="buffer min", - index=index, - quantity=min_soc, - upwards_deviation_price=0, - downwards_deviation_price=-breach_price, - device=pd.Series([[0, 1]] * n, index=index), - device_group=device_group, + if shared: + # One commitment covering the *combined* stock of both HPs. + commitments.append( + StockCommitment( + name="buffer min", + index=index, + quantity=min_soc, + upwards_deviation_price=0, + downwards_deviation_price=-breach_price, + device=pd.Series([[0, 1]] * n, index=index), + device_group=device_group, + ) ) - ) + else: + # Two separate commitments, one per HP — each must reach target_soc alone. + for d in range(2): + commitments.append( + StockCommitment( + name="buffer min", + index=index, + quantity=min_soc, + upwards_deviation_price=0, + downwards_deviation_price=-breach_price, + device=pd.Series(d, index=index), + device_group=device_group, + ) + ) - # StockCommitment 2+3+4: individual upper bounds (each device's stock <= 100) + # Individual upper bounds (soft) and energy price for all three devices. for d in range(3): commitments.append( StockCommitment( name=f"buffer max {d}", index=index, - quantity=100.0, + quantity=500.0, upwards_deviation_price=breach_price, downwards_deviation_price=0, device=pd.Series(d, index=index), device_group=device_group, ) ) - - # FlowCommitment: uniform energy price for each device - for d in range(3): commitments.append( FlowCommitment( name="energy", @@ -293,34 +294,58 @@ def test_device_group_shared_buffer(): ) ) - # ---- run - planned_power, planned_costs, results, model = device_scheduler( + _, _, results, model = device_scheduler( device_constraints=device_constraints, ems_constraints=ems_constraints, commitments=commitments, initial_stock=0, ) - # ---- solved assert results.solver.termination_condition in ("optimal", "locallyOptimal") - # ---- device_group structure: two groups - assert set(device_group.values) == {"shared thermal buffer", "battery SoC"} - - # ---- KEY: the "buffer min" commitment cost must be zero - # Both HPs together can supply up to 2×20 kW×24 h×0.9 ≈ 864 kWh, - # so reaching 100 kWh is always feasible at zero breach cost. - # If device_group were NOT applied (separate commitment per device), - # the cost would be non-zero because each device alone cannot satisfy - # the full 100 kWh target in the model. - buffer_min_costs = sum( + return sum( v for c, v in zip(commitments, model.commitment_costs.values()) if c.name == "buffer min" ) - assert buffer_min_costs == 0, ( - "Buffer min commitment should be satisfied at zero cost " - "(both HPs together can easily reach 100 kWh)" + + +def test_device_group_shared_buffer(): + """ + Two heat pumps (devices 0 and 1) charge a shared thermal buffer with a target of + 800 kWh by the last time slot. + + Each HP can supply at most 20 kW × 24 h × 0.9 = 432 kWh on its own, so neither + can reach 800 kWh individually. Together they can supply up to 864 kWh, so the + target is feasible when the commitment tracks their *combined* stock. + + This test verifies two contrasting scenarios: + + 1. Shared buffer (device_group): one StockCommitment on the combined stock of both + HPs. The optimizer fills 800 kWh across the two devices with zero breach cost. + + 2. Separate buffers (no device_group): one StockCommitment per HP, each requiring + 800 kWh on its own stock. Each HP falls short by ~368 kWh, so both commitments + incur a breach, and the total breach cost is positive. + """ + start = pd.Timestamp("2026-01-01T00:00+01") + end = pd.Timestamp("2026-01-02T00:00+01") + resolution = pd.Timedelta("PT1H") + index = initialize_index(start=start, end=end, resolution=resolution) + + # 800 kWh: above what one HP can reach (432 kWh), below what two can reach (864 kWh). + target_soc = 800.0 + + shared_cost = _run_hp_buffer_scenario(index, target_soc, shared=True) + separate_cost = _run_hp_buffer_scenario(index, target_soc, shared=False) + + assert shared_cost == 0, ( + f"Shared buffer: both HPs together can reach {target_soc} kWh, " + f"so breach cost must be zero (got {shared_cost})" + ) + assert separate_cost > 0, ( + f"Separate buffers: each HP alone cannot reach {target_soc} kWh, " + f"so breach cost must be positive (got {separate_cost})" ) From 42586360f9d6f96a13b2fab3523883930ecad609 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 19:27:15 +0200 Subject: [PATCH 48/59] fix: do not coerce device_group into a time series Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py index 13af1e15a4..c95882d507 100644 --- a/flexmeasures/data/models/planning/__init__.py +++ b/flexmeasures/data/models/planning/__init__.py @@ -290,10 +290,12 @@ class Commitment: downwards_deviation_price: pd.Series = 0 def __post_init__(self): + # device_group is a device→label lookup table, not a time series; + # exclude it from automatic time-series index coercion. series_attributes = [ attr for attr, _type in self.__annotations__.items() - if _type == "pd.Series" and hasattr(self, attr) + if _type == "pd.Series" and hasattr(self, attr) and attr != "device_group" ] for series_attr in series_attributes: val = getattr(self, series_attr) From b4f8b2b824b037c5fb6f1c3941671144493d7485 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 19:29:42 +0200 Subject: [PATCH 49/59] docs: changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 06f9ea03e5..10a61b9bd2 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -16,6 +16,7 @@ Infrastructure / Support ---------------------- * Upgraded dependencies [see `PR #2114 `_] * Run ``flexmeasures jobs run-worker`` with RQ's embedded scheduler on by default so jobs created with ``enqueue_in`` are promoted from the scheduled registry when due; pass ``--without-scheduler`` to disable [see `PR #2112 `_] +* Prepare the ``device_scheduler`` to deal with commitments per device group [see `PR #1934 `_] Bugfixes ----------- From 6e3a5a42e37ece975ea3a8afa68858f2feef5eb5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 21:01:20 +0200 Subject: [PATCH 50/59] docs: pretty_print is not specifically for FlowCommitments Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py index c95882d507..d29efff60c 100644 --- a/flexmeasures/data/models/planning/__init__.py +++ b/flexmeasures/data/models/planning/__init__.py @@ -399,9 +399,9 @@ def _init_device_group(self): def pretty_print(self): """ - Pretty-print a list of FlowCommitment objects as tabulated pandas DataFrames. + Pretty-print a list of Commitment objects as tabulated pandas DataFrames. - For each FlowCommitment, a DataFrame indexed by time is created containing + For each Commitment, a DataFrame indexed by time is created containing the commitment name, device values, group index, quantity, and any available upward or downward deviation prices. Each commitment is printed separately in a readable table format, making this function suitable for debugging, From 7eb1319b3416967204b8818305f1a6da13f6bf99 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 21:04:38 +0200 Subject: [PATCH 51/59] docs: add (back) inline dev notes Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/linear_optimization.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 69d308158a..56c67e2936 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -264,16 +264,19 @@ def convert_commitments_to_subcommitments( ) model.c = RangeSet(0, len(commitments) - 1, doc="Set of commitments") + # Add 2D indices for commitment device groups (cg) def commitment_device_groups_init(m): return ((c, g) for c, groups in device_group_lookup.items() for g in groups) model.cg = Set(dimen=2, initialize=commitment_device_groups_init) + # Add 2D indices for commitment datetimes (cj) def commitments_init(m): return ((c, j) for c in m.c for j in commitments[c]["j"]) model.cj = Set(dimen=2, initialize=commitments_init) + # Add 3D indices for commitment datetime device groups (cjg) def commitment_time_device_groups_init(m): return ((c, j, g) for (c, j) in m.cj for (_, g) in m.cg if _ == c) From a298dc739c12e7897e3d4e29775d8fd128f23867 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 21:06:43 +0200 Subject: [PATCH 52/59] feat: strengthen asserts Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_commitments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 14f7436afb..b13900cd63 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -145,8 +145,8 @@ def test_multi_feed_device_scheduler_shared_buffer(): initial_stock=0, ) - # ---- sanity: model solved - assert results.solver.termination_condition in ("optimal", "locallyOptimal") + # ---- sanity: model solved optimally + assert results.solver.termination_condition == "optimal" # ---- key assertion: exactly TWO commitment groups # - one for "shared thermal buffer" @@ -301,7 +301,7 @@ def _run_hp_buffer_scenario(index, target_soc, shared: bool): initial_stock=0, ) - assert results.solver.termination_condition in ("optimal", "locallyOptimal") + assert results.solver.termination_condition == "optimal" return sum( v From 7f15c1306bff95af926de2990fc12a500df057fd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 21:15:48 +0200 Subject: [PATCH 53/59] feat: add check for exact electricity costs expected Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_commitments.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index b13900cd63..655dfa7bad 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -184,6 +184,12 @@ def test_multi_feed_device_scheduler_shared_buffer(): total_commodity_cost = sum(commodity_costs.values()) assert total_commodity_cost <= planned_costs + # Expect the total electricity costs to be: + # 2 * 20 for the heat pump + # 2 * 20 for the battery charging + # minus 2 * 20 * 0.9 * 0.9 for the battery discharging after roundtrip efficiency + assert commodity_costs["electricity"] == 200 * (40 + 40) - 600 * (40 * 0.9 * 0.9) + def _run_hp_buffer_scenario(index, target_soc, shared: bool): """ From ffd0d86b125a163e81f7675ad42749021431974d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 22:17:58 +0200 Subject: [PATCH 54/59] fix: sum over electricity costs; and move to StockCommitment to model preference for full soc sooner Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_commitments.py | 92 +++++++++++++------ 1 file changed, 65 insertions(+), 27 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 655dfa7bad..c33024f94e 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -1,3 +1,4 @@ +import pytest import pandas as pd import numpy as np @@ -8,7 +9,6 @@ ) from flexmeasures.data.models.planning.utils import ( initialize_index, - add_tiny_price_slope, ) from flexmeasures.data.models.planning.linear_optimization import device_scheduler @@ -81,10 +81,9 @@ def test_multi_feed_device_scheduler_shared_buffer(): electricity_price.iloc[12:14] = 200 prices = {"gas": gas_price, "electricity": electricity_price} - sloped_prices = ( - add_tiny_price_slope(electricity_price.to_frame()) - - electricity_price.to_frame() - ) + # Tie-breaking: prefer filling each device's storage as early as possible. + soc_max = 100.0 + penalty = 0.001 commitments = [] @@ -126,14 +125,13 @@ def test_multi_feed_device_scheduler_shared_buffer(): ) commitments.append( - FlowCommitment( - name="preferred_charge_sooner", + StockCommitment( + name=f"prefer a full storage {d} sooner", index=index, - quantity=0, - upwards_deviation_price=sloped_prices, - downwards_deviation_price=sloped_prices, + quantity=soc_max, + upwards_deviation_price=0, + downwards_deviation_price=-penalty, device=pd.Series(d, index=index), - device_group=device_commodity, ) ) @@ -161,16 +159,20 @@ def test_multi_feed_device_scheduler_shared_buffer(): } assert commodity_commitments == {"gas", "electricity"} - commitment_costs = { - "name": "commitment_costs", - "data": { - c.name: costs - for c, costs in zip(commitments, model.commitment_costs.values()) - }, - } - commodity_costs = { - k: v for k, v in commitment_costs["data"].items() if k in {"gas", "electricity"} - } + # Sum per-commitment costs grouped by name so that duplicate names + # (e.g. "electricity" for both heat pump and battery) are accumulated. + electricity_cost = sum( + costs + for c, costs in zip(commitments, model.commitment_costs.values()) + if c.name == "electricity" + ) + gas_cost = sum( + costs + for c, costs in zip(commitments, model.commitment_costs.values()) + if c.name == "gas" + ) + commodity_costs = {"gas": gas_cost, "electricity": electricity_cost} + assert set(commodity_costs.keys()) == {"gas", "electricity"} assert commitment_groups == {"shared thermal buffer"} @@ -188,13 +190,15 @@ def test_multi_feed_device_scheduler_shared_buffer(): # 2 * 20 for the heat pump # 2 * 20 for the battery charging # minus 2 * 20 * 0.9 * 0.9 for the battery discharging after roundtrip efficiency - assert commodity_costs["electricity"] == 200 * (40 + 40) - 600 * (40 * 0.9 * 0.9) + assert electricity_cost == pytest.approx( + 200 * (40 + 40) - 600 * (40 * 0.9 * 0.9), rel=1e-6 + ) def _run_hp_buffer_scenario(index, target_soc, shared: bool): """ Helper: run the two-heat-pump scheduler with either a shared or per-device buffer - commitment and return the total "buffer min" breach cost. + commitment and return costs for assertions. Each heat pump (device 0 and 1) can supply at most derivative_max × hours × efficiency = 20 kW × 24 h × 0.9 = 432 kWh. @@ -245,6 +249,10 @@ def _run_hp_buffer_scenario(index, target_soc, shared: bool): min_soc = pd.Series(0.0, index=index) min_soc.iloc[-1] = target_soc + # Tie-breaking: prefer filling each device's storage as early as possible. + soc_max = 500.0 # matches device_constraints["max"] + penalty = 0.001 + commitments = [] if shared: @@ -299,8 +307,18 @@ def _run_hp_buffer_scenario(index, target_soc, shared: bool): device_group=device_group, ) ) + commitments.append( + StockCommitment( + name=f"prefer a full storage {d} sooner", + index=index, + quantity=soc_max, + upwards_deviation_price=0, + downwards_deviation_price=-penalty, + device=d, + ) + ) - _, _, results, model = device_scheduler( + planned_power, planned_costs, results, model = device_scheduler( device_constraints=device_constraints, ems_constraints=ems_constraints, commitments=commitments, @@ -309,11 +327,17 @@ def _run_hp_buffer_scenario(index, target_soc, shared: bool): assert results.solver.termination_condition == "optimal" - return sum( + buffer_min_cost = sum( v for c, v in zip(commitments, model.commitment_costs.values()) if c.name == "buffer min" ) + energy_cost = sum( + v + for c, v in zip(commitments, model.commitment_costs.values()) + if c.name == "energy" + ) + return {"buffer_min_cost": buffer_min_cost, "energy_cost": energy_cost} def test_device_group_shared_buffer(): @@ -342,8 +366,13 @@ def test_device_group_shared_buffer(): # 800 kWh: above what one HP can reach (432 kWh), below what two can reach (864 kWh). target_soc = 800.0 - shared_cost = _run_hp_buffer_scenario(index, target_soc, shared=True) - separate_cost = _run_hp_buffer_scenario(index, target_soc, shared=False) + shared_result = _run_hp_buffer_scenario(index, target_soc, shared=True) + separate_result = _run_hp_buffer_scenario(index, target_soc, shared=False) + + shared_cost = shared_result["buffer_min_cost"] + separate_cost = separate_result["buffer_min_cost"] + shared_energy = shared_result["energy_cost"] + separate_energy = separate_result["energy_cost"] assert shared_cost == 0, ( f"Shared buffer: both HPs together can reach {target_soc} kWh, " @@ -354,6 +383,15 @@ def test_device_group_shared_buffer(): f"so breach cost must be positive (got {separate_cost})" ) + # With shared buffer, the optimizer charges exactly 800 kWh combined. + # Total energy flow = 800 / 0.9 (accounting for charge efficiency). + assert shared_energy == pytest.approx(target_soc / 0.9 * 100, rel=1e-6) + + # With separate buffers, each HP charges at maximum power for all 24 hours + # since the 800 kWh individual target is unreachable. + # Total energy = 2 HPs × 20 kW × 24 h × 100 price. + assert separate_energy == pytest.approx(2 * 20 * 24 * 100, rel=1e-6) + def make_index(n: int = 5) -> pd.DatetimeIndex: """ From a33790a6cac290f01b11aaeb82b02593f5d5cf29 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 23:59:11 +0200 Subject: [PATCH 55/59] fix: replace vague assert with explicit asserts Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_commitments.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index c33024f94e..6602020ad1 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -177,14 +177,25 @@ def test_multi_feed_device_scheduler_shared_buffer(): assert commitment_groups == {"shared thermal buffer"} - # ---- key behavioural check: - # total commitment cost should be <= 1 breach per group per timestep - # - # If baselines were duplicated, cost would be ~2x for the shared buffer. - expected_max_cost = len(index) * breach_price * 2 - assert planned_costs <= expected_max_cost - total_commodity_cost = sum(commodity_costs.values()) - assert total_commodity_cost <= planned_costs + # The shared buffer minimum (SoC ≥ 100 at the final step) must be met without + # any breach. If baseline costs were duplicated the optimiser would be driven + # to over-invest in commodities to avoid inflated penalties, which would also + # show up here as a non-zero breach cost. + buffer_min_cost = sum( + costs + for c, costs in zip(commitments, model.commitment_costs.values()) + if c.name == "buffer min" + ) + assert buffer_min_cost == 0, ( + f"Shared buffer target was breached (breach cost = {buffer_min_cost}). " + "This may indicate that baseline costs were incorrectly duplicated." + ) + + # At hours 12–13 electricity (200) is cheaper than gas (300), so the heat pump + # runs at full power: 2 h × 20 kW = 40 kWh flow → 40 × 0.9 = 36 kWh SoC + # contribution to the shared buffer. The gas boiler covers the remainder: + # (100 − 36) kWh SoC / 0.9 efficiency × 300 price. + assert gas_cost == pytest.approx((100 - 40 * 0.9) / 0.9 * 300, rel=1e-6) # Expect the total electricity costs to be: # 2 * 20 for the heat pump From 6b6a474cabd18abbede1b2cae887a51faf3acb44 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Apr 2026 23:59:31 +0200 Subject: [PATCH 56/59] docs: lose confusing comment Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_commitments.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_commitments.py b/flexmeasures/data/models/planning/tests/test_commitments.py index 6602020ad1..195409afc5 100644 --- a/flexmeasures/data/models/planning/tests/test_commitments.py +++ b/flexmeasures/data/models/planning/tests/test_commitments.py @@ -177,10 +177,7 @@ def test_multi_feed_device_scheduler_shared_buffer(): assert commitment_groups == {"shared thermal buffer"} - # The shared buffer minimum (SoC ≥ 100 at the final step) must be met without - # any breach. If baseline costs were duplicated the optimiser would be driven - # to over-invest in commodities to avoid inflated penalties, which would also - # show up here as a non-zero breach cost. + # The shared buffer minimum (SoC ≥ 100 at the final step) must be met without any breach buffer_min_cost = sum( costs for c, costs in zip(commitments, model.commitment_costs.values()) From 70e1af6dbff73ce9c79c7b64197526998c1a61af Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 25 Apr 2026 00:11:23 +0200 Subject: [PATCH 57/59] fix: backwards compatibility in case no device is specified; The two changes together are the backwards-compatible default that was missing: when no device is specified (device=None), to_frame() must emit NaN in the device_group column (not crash), so the optimizer routes the commitment through ems_flow_commitment_equalities exactly as it did before device_group was introduced. Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/__init__.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py index d29efff60c..997df93e4d 100644 --- a/flexmeasures/data/models/planning/__init__.py +++ b/flexmeasures/data/models/planning/__init__.py @@ -6,6 +6,7 @@ from tabulate import tabulate from typing import Any, Type +import numpy as np import pandas as pd from flask import current_app @@ -369,12 +370,7 @@ def __post_init__(self): self._init_device_group() def _init_device_group(self): - # EMS-level commitment - if self.device is None: - self.device_group = pd.Series({"EMS": 0}, name="device_group") - return - - # Extract device universe + # Extract device universe (empty for EMS-level commitments where device was None) if isinstance(self.device, pd.Series): devices = extract_devices(self.device) else: @@ -382,6 +378,11 @@ def _init_device_group(self): devices = list(devices) + # EMS-level commitment: no device assignments, leave device_group empty. + if not devices: + self.device_group = pd.Series(dtype=object, name="device_group") + return + # Default: one group per device (backwards compatible) if self.device_group is None: self.device_group = pd.Series( @@ -437,11 +438,17 @@ def to_frame(self) -> pd.DataFrame: ], axis=1, ) - # map device → device_group - if self.device is not None: + # Map device → device_group. + # For EMS-level commitments (device was None, now a NaN series) there are + # no device assignments, so device_group is an empty Series and we must not + # call map_device_to_group (it would KeyError on the NaN values). + devices = extract_devices(self.device) + if devices: df["device_group"] = map_device_to_group(self.device, self.device_group) else: - df["device_group"] = 0 + df["device_group"] = ( + np.nan + ) # EMS-level: handled by ems_flow_commitment_equalities return df From 792d2df11c1185b39b70310db5bab4ba7a29a436 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 25 Apr 2026 10:56:46 +0200 Subject: [PATCH 58/59] =?UTF-8?q?fix:=20when=20device=20is=20present=20but?= =?UTF-8?q?=20device=5Fgroup=20is=20absent,=20fall=20back=20to=20the=20pre?= =?UTF-8?q?-existing=20behaviour=20=E2=80=94=20each=20device=20is=20its=20?= =?UTF-8?q?own=20group?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: F.N. Claessen --- .../data/models/planning/linear_optimization.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 56c67e2936..1b85452f9d 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -210,16 +210,27 @@ def convert_commitments_to_subcommitments( device_group_lookup = {} for c, df in enumerate(commitments): - if "device_group" not in df.columns or "device" not in df.columns: + if "device" not in df.columns: + # EMS-level commitment: no device grouping needed here; + # handled by ems_flow_commitment_equalities. continue - rows = df[["device", "device_group"]].dropna() + has_device_group = "device_group" in df.columns + if has_device_group: + rows = df[["device", "device_group"]].dropna() + else: + # Backwards-compatible default: each device is its own group. + # This preserves the behaviour of old-style DataFrame commitments that + # pre-date the device_group feature (e.g. from initialize_device_commitment). + rows = df[["device"]].dropna() device_group_lookup[c] = {} for _, row in rows.iterrows(): - g = row["device_group"] d = row["device"] + # When no device_group column is present, use the device id itself as + # the group label so that each device forms an independent group. + g = row["device_group"] if has_device_group else d if isinstance(d, (list, tuple, set, np.ndarray)): devices = set(d) From a9174a5861f27ea86ecd98b045be3600c95531b7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 26 Apr 2026 15:46:13 +0200 Subject: [PATCH 59/59] dev: update expectations of how costs are shared between EV and battery Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_simultaneous.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py index c9a1fd8836..d58d6ab4c8 100644 --- a/flexmeasures/data/tests/test_scheduling_simultaneous.py +++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py @@ -107,9 +107,9 @@ def test_create_simultaneous_jobs( total_cost = ev_costs + battery_costs # Define expected costs based on resolution - expected_ev_costs = 2.2375 - expected_battery_costs = -5.515 expected_total_cost = -3.2775 + expected_ev_costs = 2.3125 + expected_battery_costs = expected_total_cost - expected_ev_costs # Check costs assert (