From 8edbf8af9eb70cf8f965fd4acb6977cbea040307 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:41:08 +0100 Subject: [PATCH 01/52] feat: start test cases for deserializing time series data to BeliefsDataFrames Signed-off-by: F.N. Claessen --- .../schemas/tests/test_sensor_data_schema.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index b1590937a8..6294e749ae 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -5,6 +5,7 @@ from marshmallow import ValidationError import pandas as pd +from timely_beliefs import BeliefsDataFrame from unittest import mock from flexmeasures.api.common.schemas.sensor_data import ( @@ -481,3 +482,70 @@ def test_build_asset_jobs_data(db, app, add_battery_assets): app.queues["forecasting"].empty() assert app.queues["scheduling"].count == 0 assert app.queues["forecasting"].count == 0 + + +@pytest.mark.parametrize( + "deserialization_input, exp_length, exp_deserialization_output", + [ + # A single 2-hour event (spanning 3 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T14:40+01", + "duration": "PT2H", + "values": [20.5], + "unit": "EUR/kWh", + }, + 2, + {}, + ), + # Six 2-minute events (spanning 2 wall-clock hours) are deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T10:50:40+01", + "duration": "PT12M", + "values": [1, 2, 3, 4, 5, 6], + "unit": "EUR/kWh", + }, + 2, + {}, + ), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_time_series_deserialization( + setup_markets, + setup_roles_users, + deserialization_input, + exp_length, + exp_deserialization_output, + requesting_user, +): + """Test loading of posted sensor data.""" + schema = PostSensorDataSchema() + + # Look up sensor by name and replace it with the sensor ID + sensor = setup_markets[deserialization_input["sensor"]] + deserialization_input["sensor"] = sensor.id + + deser = schema.load(deserialization_input) + bdf = deser["bdf"] + + assert isinstance(bdf, BeliefsDataFrame) + assert bdf.event_resolution == sensor.event_resolution + assert len(bdf) == exp_length + + resolution = pd.Timedelta(deserialization_input["duration"]) / len( + deserialization_input["values"] + ) + if resolution < sensor.event_resolution: + assert bdf.event_starts[0] == pd.Timestamp( + deserialization_input["start"] + ).floor(sensor.event_resolution) + else: + assert bdf.event_starts[0] == pd.Timestamp(deserialization_input["start"]) + assert len(bdf.lineage.belief_times) == 1, "expected unique belief time" + # assert deser == exp_deserialization_output From 3b2770f94d87164d1bfd92f142ead694971b3493 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:41:49 +0100 Subject: [PATCH 02/52] fix: update type annotation Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7f6b0f337a..7d70299daa 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -303,7 +303,7 @@ def check_multiple_instantenous_values(self, data, **kwargs): ) @post_load() - def post_load_sequence(self, data: dict, **kwargs) -> BeliefsDataFrame: + def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame]: """ If needed, upsample and convert units, then deserialize to a BeliefsDataFrame. Returns a dict with the BDF in it, as that is expected by webargs when used with as_kwargs=True. From 34a2dc50fdeca42e2088c6d035373efd362f0409 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:42:25 +0100 Subject: [PATCH 03/52] feat: handle resampling after converting to a BeliefsDataFrame Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7d70299daa..66b8eaf94a 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -267,28 +267,6 @@ def check_schema_unit_against_type(self, data, **kwargs): f"The unit required for this message type should be convertible to an energy price unit, got incompatible unit: {posted_unit}" ) - @validates_schema - def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): - """Ensure event frequency is compatible with the sensor's event resolution. - - For a sensor recording instantaneous values, any event frequency is compatible. - For a sensor recording non-instantaneous values, the event frequency must fit the sensor's event resolution. - Currently, only upsampling is supported (e.g. converting hourly events to 15-minute events). - """ - required_resolution = data["sensor"].event_resolution - - if required_resolution == timedelta(hours=0): - # For instantaneous sensors, any event frequency is compatible - return - - # The event frequency is inferred by assuming sequential, equidistant values within a time interval. - # The event resolution is assumed to be equal to the event frequency. - inferred_resolution = data["duration"] / len(data["values"]) - if inferred_resolution % required_resolution != timedelta(hours=0): - raise ValidationError( - f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}." - ) - @validates_schema def check_multiple_instantenous_values(self, data, **kwargs): """Ensure that we are not getting multiple instantaneous values that overlap. @@ -308,7 +286,6 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame If needed, upsample and convert units, then deserialize to a BeliefsDataFrame. Returns a dict with the BDF in it, as that is expected by webargs when used with as_kwargs=True. """ - data = self.possibly_upsample_values(data) data = self.possibly_convert_units(data) bdf = self.load_bdf(data) @@ -371,6 +348,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: event_resolution = sensor_data["duration"] / num_values start = sensor_data["start"] sensor = sensor_data["sensor"] + inferred_resolution = sensor_data["duration"] / len(sensor_data["values"]) if frequency := sensor.get_attribute("frequency"): start = pd.Timestamp(start).round(frequency) @@ -396,12 +374,15 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: belief_timing["belief_horizon"] = sensor_data["horizon"] else: belief_timing["belief_time"] = server_now() - return BeliefsDataFrame( + bdf = BeliefsDataFrame( s, source=source, sensor=sensor_data["sensor"], + event_resolution=inferred_resolution, **belief_timing, ) + resampled_bdf = bdf.resample_events(sensor.event_resolution) + return resampled_bdf class GetSensorDataSchemaEntityAddress(GetSensorDataSchema): From 8d2d2d3bad0afa96e88653551ab9409937d40164 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:48:57 +0100 Subject: [PATCH 04/52] fix: catch NotImplementedError to fix failing test Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 66b8eaf94a..754f219d1d 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -381,7 +381,12 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: event_resolution=inferred_resolution, **belief_timing, ) - resampled_bdf = bdf.resample_events(sensor.event_resolution) + try: + resampled_bdf = bdf.resample_events(sensor.event_resolution) + except NotImplementedError: + raise ValidationError( + f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {sensor.event_resolution}." + ) return resampled_bdf From 86ad7745107624e5a6e5e63ccd63e301b438d22c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 16:21:41 +0100 Subject: [PATCH 05/52] feat: two more test cases Signed-off-by: F.N. Claessen --- .../schemas/tests/test_sensor_data_schema.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 6294e749ae..f31e9c9072 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -487,7 +487,7 @@ def test_build_asset_jobs_data(db, app, add_battery_assets): @pytest.mark.parametrize( "deserialization_input, exp_length, exp_deserialization_output", [ - # A single 2-hour event (spanning 3 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + # # A single 2-hour event (spanning 3 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor ( { "sensor": "epex_da", # name is used to look up the corresponding sensor ID @@ -499,6 +499,30 @@ def test_build_asset_jobs_data(db, app, add_battery_assets): 2, {}, ), + # One 5-minute event (spanning 2 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T10:58:40+01", + "duration": "PT5M", + "values": [1], + "unit": "EUR/kWh", + }, + 2, + {}, + ), + # One 2-minute event (spanning 1 wall-clock hour) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T10:58:00+01", + "duration": "PT2M", + "values": [1], + "unit": "EUR/kWh", + }, + 1, + {}, + ), # Six 2-minute events (spanning 2 wall-clock hours) are deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor ( { From 30fb2ad412da6edc4ea7687cfac837a3666635f5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 17:01:35 +0100 Subject: [PATCH 06/52] fix: update test expectations Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 6 ------ flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index c9c1822c7c..49bc4ef574 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -145,12 +145,6 @@ def test_post_sensor_data_bad_auth( "request_field, new_value, error_field, error_text", [ ("start", "2021-06-07T00:00:00", "start", "Not a valid aware datetime"), - ( - "duration", - "PT30M", - "_schema", - "Resolution of 0:05:00 is incompatible", - ), # downsampling not supported ("unit", "m", "_schema", "Required unit"), ("type", "GetSensorDataRequest", "type", "Must be one of"), ], diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index 3912bc0cca..aafa84b94d 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -31,12 +31,12 @@ ), # upsample from single value for 1-hour interval, sent as float rather than list of floats ( 4, - 0, + 6, "m³/h", False, - None, + -11.28, False, - 422, + 200, ), # failed to resample from 15-min intervals to 10-min intervals ( 10, From 6c08284715af42b8edd63aa15108c9daf3fd64de Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 2 May 2026 13:29:35 +0100 Subject: [PATCH 07/52] fix: floor direct POST sensor data starts Signed-off-by: Mohamed Belhsan Hmida --- .../api/common/schemas/sensor_data.py | 6 ++- .../api/v3_0/tests/test_sensor_data.py | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 644d51412f..889a542e2d 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -446,7 +446,11 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: sensor = sensor_data["sensor"] inferred_resolution = sensor_data["duration"] / len(sensor_data["values"]) - if frequency := sensor.get_attribute("frequency"): + if sensor.event_resolution != timedelta(0) and sensor.get_attribute( + "round_datetimes_on_ingestion", True + ): + start = pd.Timestamp(start).floor(sensor.event_resolution) + elif frequency := sensor.get_attribute("frequency"): start = pd.Timestamp(start).round(frequency) if event_resolution == timedelta(hours=0): diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index ae0a698294..a9e705b538 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -2,6 +2,7 @@ from datetime import timedelta from flask import url_for +import pandas as pd import pytest from sqlalchemy import event from sqlalchemy.engine import Engine @@ -233,6 +234,43 @@ def test_post_invalid_sensor_data( ) +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_post_non_instantaneous_sensor_data_floor( + client, setup_api_test_data, requesting_user +): + post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") + post_data["start"] = "2021-06-08T00:00:40+02:00" + sensor = setup_api_test_data["some gas sensor"] + + rows = len(sensor.search_beliefs()) + + response = client.post( + url_for("SensorAPI:post_data", id=sensor.id), + json=post_data, + ) + + assert response.status_code == 200 + + data = sensor.search_beliefs().reset_index() + new_data = data[ + data["event_start"].between( + pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), + ) + ] + assert len(sensor.search_beliefs()) - rows == 6 + assert list(new_data["event_start"]) == [ + pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:10:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:20:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:30:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:40:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), + ] + + @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) From f590e583b53778a38e263d863068ea049fde23d1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 2 May 2026 13:30:20 +0100 Subject: [PATCH 08/52] fix: floor uploaded sensor data datetimes Signed-off-by: Mohamed Belhsan Hmida --- .../v3_0/tests/test_sensors_api_freshdb.py | 79 +++++++++++++++++++ flexmeasures/data/schemas/sensors.py | 35 ++++++++ 2 files changed, 114 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index f0c7e12e0c..749ca8d503 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -1,6 +1,7 @@ import io import pytest from datetime import timedelta +import pandas as pd from flask import url_for from sqlalchemy import select @@ -232,6 +233,84 @@ def test_upload_sensor_data_with_unit_conversion_success( assert [b.event_value for b in beliefs] == expected_event_values +@pytest.mark.parametrize( + "requesting_user, sensor_index, start_date, data_resolution, data_values, expected_event_starts, expected_event_values", + [ + ( + "test_prosumer_user_2@seita.nl", + 1, + "2025-01-01T10:00:40+00:00", + timedelta(minutes=15), + [2, 3, 4], + [ + pd.Timestamp("2025-01-01T10:00:00+00:00"), + pd.Timestamp("2025-01-01T10:15:00+00:00"), + pd.Timestamp("2025-01-01T10:30:00+00:00"), + ], + [2, 3, 4], + ), + ( + "test_prosumer_user_2@seita.nl", + 2, + "2025-01-01T10:00:40+00:00", + timedelta(minutes=30), + [10, 20, 20, 40], + [ + pd.Timestamp("2025-01-01T10:00:00+00:00"), + pd.Timestamp("2025-01-01T11:00:00+00:00"), + ], + [30, 60], + ), + ], + indirect=["requesting_user"], +) +def test_upload_sensor_data_floors_offclock_datetimes( + fresh_db, + client, + add_battery_assets_fresh_db, + requesting_user, + sensor_index, + start_date, + data_resolution, + data_values, + expected_event_starts, + expected_event_values, +): + test_battery = add_battery_assets_fresh_db["Test battery"] + sensor = test_battery.sensors[sensor_index] + + csv_content = generate_csv_content( + start_time_str=start_date, + interval=data_resolution, + values=data_values, + ) + file_obj = io.BytesIO(csv_content.encode("utf-8")) + + response = client.post( + url_for("SensorAPI:upload_data", id=sensor.id), + data={"uploaded-files": (file_obj, "data.csv"), "unit": sensor.unit}, + content_type="multipart/form-data", + ) + assert response.status_code == 200 + + beliefs = ( + fresh_db.session.execute( + select(TimedBelief) + .filter(TimedBelief.sensor_id == sensor.id) + .filter(TimedBelief.event_start >= expected_event_starts[0]) + .filter( + TimedBelief.event_start + < expected_event_starts[-1] + sensor.event_resolution + ) + .order_by(TimedBelief.event_start) + ) + .scalars() + .all() + ) + assert [b.event_start for b in beliefs] == expected_event_starts + assert [b.event_value for b in beliefs] == expected_event_values + + @pytest.mark.parametrize( "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_err_msg, expected_status", [ diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index af73d93ae1..8775531371 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -67,6 +67,7 @@ class TimedEventSchema(Schema): def __init__( self, timezone: str | None = None, + event_resolution: timedelta | None = None, value_validator: Validator | None = None, to_unit: str | None = None, default_src_unit: str | None = None, @@ -79,6 +80,7 @@ def __init__( :param timezone: Optionally, set a timezone to be able to interpret nominal durations. """ self.timezone = timezone + self.event_resolution = event_resolution self.value_validator = value_validator super().__init__(*args, **kwargs) if to_unit is not None: @@ -97,12 +99,23 @@ def validate_value(self, _value, **kwargs): if self.value_validator is not None: self.value_validator(_value) + def floor_timing_fields(self, data: dict) -> None: + if self.event_resolution in (None, timedelta(0)): + return + + for key in ("datetime", "start", "end"): + if data.get(key) is not None: + data[key] = ( + pd.Timestamp(data[key]).floor(self.event_resolution).to_pydatetime() + ) + @validates_schema def check_time_window(self, data, **kwargs): """Checks whether a complete time interval can be derived from the timing fields. The data is updated in-place, guaranteeing that the 'start' and 'end' fields are filled out. """ + self.floor_timing_fields(data) dt = data.get("datetime") start = data.get("start") end = data.get("end") @@ -336,6 +349,7 @@ def __init__( default_src_unit: str | None = None, return_magnitude: bool = False, timezone: str | None = None, + event_resolution: timedelta | None = None, value_validator: Validator | None = None, additional_sensor_units: list[str] | None = None, **kwargs, @@ -380,6 +394,7 @@ def __init__( value_validator = RepurposeValidatorToIgnoreSensorsAndLists(value_validator) self.validators.insert(0, value_validator) self.timezone = timezone + self.event_resolution = event_resolution self.value_validator = value_validator if to_unit.startswith("/") and len(to_unit) < 2: raise ValueError( @@ -443,6 +458,7 @@ def _deserialize_list(self, value: list[dict]) -> list[dict]: fields.Nested( TimedEventSchema( timezone=self.timezone, + event_resolution=self.event_resolution, value_validator=self.value_validator, to_unit=self.to_unit, default_src_unit=self.default_src_unit, @@ -709,6 +725,11 @@ def post_load(self, fields, **kwargs): # Reraise the error if an event frequency could not be inferred pd.infer_freq(bdf.index.unique("event_start")) + if sensor.event_resolution != timedelta(0) and sensor.get_attribute( + "round_datetimes_on_ingestion", True + ): + bdf = floor_bdf_event_starts(bdf, bdf.event_resolution) + bdf["event_value"] = convert_units( bdf["event_value"], from_unit, @@ -747,6 +768,20 @@ def post_load(self, fields, **kwargs): return fields +def floor_bdf_event_starts( + bdf: tb.BeliefsDataFrame, event_resolution: timedelta +) -> tb.BeliefsDataFrame: + floored_bdf = pd.DataFrame(bdf.reset_index()) + floored_bdf["event_start"] = pd.DatetimeIndex(bdf.event_starts).floor( + event_resolution + ) + return tb.BeliefsDataFrame( + floored_bdf, + sensor=bdf.sensor, + event_resolution=bdf.event_resolution, + ) + + class QuantitySchema(Schema): """Represents a quantity string like '1 EUR/MWh'.""" From 0566ceb4d3107c56b00a3db56f8b5e9be13e606a Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 2 May 2026 13:35:51 +0100 Subject: [PATCH 09/52] fix: floor flex-model scheduling datetimes Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 26 +++++++++++++ .../api/v3_0/tests/test_sensor_schedules.py | 38 +++++++++++++++++++ .../data/schemas/scheduling/storage.py | 9 +++++ 3 files changed, 73 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 3d0e407453..23ab7e2e79 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,7 +1,9 @@ from __future__ import annotations import isodate +from copy import deepcopy from datetime import datetime, timedelta +import pandas as pd from flexmeasures.data.services.sensors import ( serialize_sensor_status_data, @@ -221,6 +223,27 @@ class TriggerScheduleKwargsSchema(Schema): ) +def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: + if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( + "round_datetimes_on_ingestion", True + ): + return flex_model + + floored_flex_model = deepcopy(flex_model) + for value in floored_flex_model.values(): + if not isinstance(value, list): + continue + for timed_event in value: + if not isinstance(timed_event, dict): + continue + for key in ("datetime", "start", "end"): + if key in timed_event: + timed_event[key] = isodate.datetime_isoformat( + pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) + ) + return floored_flex_model + + class SensorAPI(FlaskView): route_base = "/sensors" trailing_slash = False @@ -886,6 +909,9 @@ def trigger_schedule( f"Resolution of {resolution} is incompatible with the sensor's required resolution of {sensor.event_resolution}." ) + if flex_model is not None: + flex_model = floor_timed_event_datetimes(flex_model, sensor) + end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( asset_or_sensor=sensor, diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 4490d5d996..a77a2959ec 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -98,6 +98,44 @@ def test_trigger_schedule_with_invalid_flexmodel( ) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_floors_flex_model_datetimes( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + requesting_user, +): + message = message_for_trigger_schedule(with_targets=True) + offclock_target = "2015-01-02T23:00:40+01:00" + expected_target = "2015-01-02T23:00:00+01:00" + for field in ("soc-targets", "soc-minima", "soc-maxima"): + message["flex-model"][field][0]["datetime"] = offclock_target + + sensor = add_charging_station_assets["Test charging station"].sensors[0] + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + + assert trigger_schedule_response.status_code == 200 + assert len(app.queues["scheduling"]) == 1 + + job = app.queues["scheduling"].jobs[0] + for field in ("soc-targets", "soc-minima", "soc-maxima"): + target = job.kwargs["flex_model"][field][0] + assert target["datetime"] == expected_target + if "start" in target: + assert parse_datetime(target["start"]) == parse_datetime(expected_target) + if "end" in target: + assert parse_datetime(target["end"]) == parse_datetime(expected_target) + + @pytest.mark.parametrize("message", [message_for_trigger_schedule(unknown_prices=True)]) @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 807774fe5a..e5b2bf9ad1 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -236,6 +236,12 @@ def __init__( self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None + self.rounding_resolution = ( + sensor.event_resolution + if sensor is not None + and sensor.get_attribute("round_datetimes_on_ingestion", True) + else None + ) # guess default soc-unit if default_soc_unit is None: @@ -250,6 +256,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, + event_resolution=self.rounding_resolution, data_key="soc-maxima", ) @@ -257,6 +264,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, + event_resolution=self.rounding_resolution, data_key="soc-minima", value_validator=validate.Range(min=0), ) @@ -264,6 +272,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, + event_resolution=self.rounding_resolution, data_key="soc-targets", ) From a857806055d66ca94c8d28c3aa3581db7aee0600 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Sat, 9 May 2026 14:04:15 +0100 Subject: [PATCH 10/52] fix: preserve 422 validation for invalid flex-model datetimes Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/sensors.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index dc37fffb35..64c4831f83 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -233,17 +233,30 @@ def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: return flex_model floored_flex_model = deepcopy(flex_model) - for value in floored_flex_model.values(): + for value_name, value in floored_flex_model.items(): if not isinstance(value, list): continue - for timed_event in value: + for index, timed_event in enumerate(value): if not isinstance(timed_event, dict): continue for key in ("datetime", "start", "end"): if key in timed_event: - timed_event[key] = isodate.datetime_isoformat( - pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) - ) + try: + timed_event[key] = isodate.datetime_isoformat( + pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) + ) + except (TypeError, ValueError) as exc: + raise ValidationError( + { + value_name: { + index: { + key: [ + f"Not a valid datetime: {timed_event[key]!r}." + ] + } + } + } + ) from exc return floored_flex_model From f30429cc560ad2e6de59c142335924c57dfb0b4e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 9 May 2026 14:13:00 +0100 Subject: [PATCH 11/52] style: apply pre-commit Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 64c4831f83..c678c14e66 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -243,7 +243,9 @@ def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: if key in timed_event: try: timed_event[key] = isodate.datetime_isoformat( - pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) + pd.Timestamp(timed_event[key]).floor( + sensor.event_resolution + ) ) except (TypeError, ValueError) as exc: raise ValidationError( From 3a56e0c40d03a6c1cf8715f8fd88cf36f7516ea8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:25:02 +0000 Subject: [PATCH 12/52] fix: return 422 for invalid flex-model timed-event datetimes Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/sensors.py | 5 +++- .../api/v3_0/tests/test_sensor_schedules.py | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index c678c14e66..dc258ac201 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -940,7 +940,10 @@ def trigger_schedule( ) if flex_model is not None: - flex_model = floor_timed_event_datetimes(flex_model, sensor) + try: + flex_model = floor_timed_event_datetimes(flex_model, sensor) + except ValidationError as err: + return unprocessable_entity(err.messages) end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index de3deb7ef7..7497b24971 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -152,6 +152,35 @@ def test_trigger_schedule_floors_flex_model_datetimes( assert parse_datetime(target["end"]) == parse_datetime(expected_target) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_invalid_flex_model_datetime_returns_422( + app, + add_battery_assets, + keep_scheduling_queue_empty, + requesting_user, +): + message = message_for_trigger_schedule(with_targets=True) + message["flex-model"]["soc-minima"][0]["datetime"] = "not-a-datetime" + + sensor = add_battery_assets["Test battery"].sensors[0] + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + + assert trigger_schedule_response.status_code == 422 + assert "soc-minima" in trigger_schedule_response.json["message"]["json"] + assert ( + "Not a valid datetime" + in trigger_schedule_response.json["message"]["json"]["soc-minima"]["0"][ + "datetime" + ][0] + ) + + @pytest.mark.parametrize( "flex_config, field", [ From dd2fcc4ec813863a3986693fe5e784198fb451ad Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:26:42 +0000 Subject: [PATCH 13/52] test: refine invalid flex-model datetime regression assertion Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 7497b24971..92e53b3ac1 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -155,7 +155,7 @@ def test_trigger_schedule_floors_flex_model_datetimes( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_trigger_schedule_invalid_flex_model_datetime_returns_422( +def test_trigger_schedule_with_invalid_flex_model_datetime( app, add_battery_assets, keep_scheduling_queue_empty, @@ -173,12 +173,10 @@ def test_trigger_schedule_invalid_flex_model_datetime_returns_422( assert trigger_schedule_response.status_code == 422 assert "soc-minima" in trigger_schedule_response.json["message"]["json"] - assert ( - "Not a valid datetime" - in trigger_schedule_response.json["message"]["json"]["soc-minima"]["0"][ - "datetime" - ][0] - ) + datetime_error = trigger_schedule_response.json["message"]["json"]["soc-minima"][ + "0" + ]["datetime"][0] + assert "Not a valid datetime" in datetime_error @pytest.mark.parametrize( From aa93fe11f4b65ea95843d3e1609fe62a4056e1b9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:28:05 +0000 Subject: [PATCH 14/52] test: make flex-model datetime error assertion less brittle Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 92e53b3ac1..0dfb656b9c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -172,11 +172,12 @@ def test_trigger_schedule_with_invalid_flex_model_datetime( ) assert trigger_schedule_response.status_code == 422 - assert "soc-minima" in trigger_schedule_response.json["message"]["json"] - datetime_error = trigger_schedule_response.json["message"]["json"]["soc-minima"][ - "0" - ]["datetime"][0] - assert "Not a valid datetime" in datetime_error + errors = trigger_schedule_response.json["message"]["json"] + minima_errors = errors.get("soc-minima", {}) + timed_event_errors = minima_errors.get("0", {}) + datetime_errors = timed_event_errors.get("datetime", []) + assert datetime_errors + assert "Not a valid datetime" in datetime_errors[0] @pytest.mark.parametrize( From e0f88a81045b53eb1ac1a7077e4f876c6879cf20 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:29:27 +0000 Subject: [PATCH 15/52] test: assert error payload shape for invalid flex-model datetime Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 0dfb656b9c..6aad9bd4cf 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -172,9 +172,14 @@ def test_trigger_schedule_with_invalid_flex_model_datetime( ) assert trigger_schedule_response.status_code == 422 + assert "message" in trigger_schedule_response.json + assert "json" in trigger_schedule_response.json["message"] errors = trigger_schedule_response.json["message"]["json"] + assert "soc-minima" in errors minima_errors = errors.get("soc-minima", {}) + assert "0" in minima_errors timed_event_errors = minima_errors.get("0", {}) + assert "datetime" in timed_event_errors datetime_errors = timed_event_errors.get("datetime", []) assert datetime_errors assert "Not a valid datetime" in datetime_errors[0] From aaebfaa1bac6f4e3807381705779e335adabd240 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 08:58:29 +0200 Subject: [PATCH 16/52] refactor: simplify test, incl. by not mixing timezone offsets Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensor_data.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 92b23fdde9..4ec1e9834f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -246,11 +246,16 @@ def test_post_invalid_sensor_data( def test_post_non_instantaneous_sensor_data_floor( client, setup_api_test_data, requesting_user ): + imprecise_start = "2021-06-08T00:00:40+02:00" + precise_start = "2021-06-08T00:00:00+02:00" + precise_end = "2021-06-08T01:00:00+02:00" post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") - post_data["start"] = "2021-06-08T00:00:40+02:00" + post_data["start"] = imprecise_start sensor = setup_api_test_data["some gas sensor"] - rows = len(sensor.search_beliefs()) + assert ( + len(sensor.search_beliefs(precise_start, precise_end)) == 0 + ), "No beliefs were expected before we post our test data." response = client.post( url_for("SensorAPI:post_data", id=sensor.id), @@ -259,21 +264,15 @@ def test_post_non_instantaneous_sensor_data_floor( assert response.status_code == 200 - data = sensor.search_beliefs().reset_index() - new_data = data[ - data["event_start"].between( - pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), - ) - ] - assert len(sensor.search_beliefs()) - rows == 6 + new_data = sensor.search_beliefs(precise_start, precise_end).reset_index() + assert len(new_data) == 6 assert list(new_data["event_start"]) == [ - pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:10:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:20:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:30:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:40:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), + pd.Timestamp("2021-06-08 00:00:00+02"), + pd.Timestamp("2021-06-08 00:10:00+02"), + pd.Timestamp("2021-06-08 00:20:00+02"), + pd.Timestamp("2021-06-08 00:30:00+02"), + pd.Timestamp("2021-06-08 00:40:00+02"), + pd.Timestamp("2021-06-08 00:50:00+02"), ] From 43fe0b5b95ea1bb1785b72bc2f55c063cf2e922c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 09:06:12 +0200 Subject: [PATCH 17/52] refactor: rename variable to match variable name in test_trigger_schedule_floors_flex_model_datetimes Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 4ec1e9834f..880103de27 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -246,11 +246,11 @@ def test_post_invalid_sensor_data( def test_post_non_instantaneous_sensor_data_floor( client, setup_api_test_data, requesting_user ): - imprecise_start = "2021-06-08T00:00:40+02:00" + offclock_start = "2021-06-08T00:00:40+02:00" precise_start = "2021-06-08T00:00:00+02:00" precise_end = "2021-06-08T01:00:00+02:00" post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") - post_data["start"] = imprecise_start + post_data["start"] = offclock_start sensor = setup_api_test_data["some gas sensor"] assert ( From 9624f09f6bc69353ca2ad49bd98b68d34de2f58d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 09:24:29 +0200 Subject: [PATCH 18/52] refactor: use search method over writing custom query Signed-off-by: F.N. Claessen --- .../v3_0/tests/test_sensors_api_freshdb.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 749ca8d503..0d85011457 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -293,22 +293,13 @@ def test_upload_sensor_data_floors_offclock_datetimes( ) assert response.status_code == 200 - beliefs = ( - fresh_db.session.execute( - select(TimedBelief) - .filter(TimedBelief.sensor_id == sensor.id) - .filter(TimedBelief.event_start >= expected_event_starts[0]) - .filter( - TimedBelief.event_start - < expected_event_starts[-1] + sensor.event_resolution - ) - .order_by(TimedBelief.event_start) - ) - .scalars() - .all() + bdf = sensor.search_beliefs( + expected_event_starts[0], expected_event_starts[-1] + sensor.event_resolution + ) + pd.testing.assert_index_equal( + bdf.event_starts, pd.DatetimeIndex(expected_event_starts, name="event_start") ) - assert [b.event_start for b in beliefs] == expected_event_starts - assert [b.event_value for b in beliefs] == expected_event_values + assert bdf["event_value"].to_list() == expected_event_values @pytest.mark.parametrize( From 860c700391075af1e564b287f4fa7d243e9dd66a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 09:33:18 +0200 Subject: [PATCH 19/52] feat: test resampling from 5 to 15 minutes Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensors_api_freshdb.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 0d85011457..c5370c4d3d 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -261,6 +261,18 @@ def test_upload_sensor_data_with_unit_conversion_success( ], [30, 60], ), + ( + "test_prosumer_user_2@seita.nl", + 1, + "2025-01-01T10:00:40+00:00", + timedelta(minutes=5), + [10, 20, 30, 40], + [ + pd.Timestamp("2025-01-01T10:00:00+00:00"), + pd.Timestamp("2025-01-01T10:15:00+00:00"), + ], + [20, 40], + ), ], indirect=["requesting_user"], ) From 222841089bf0e5e440f1d1b643d7d7d3d95dc1b0 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 15 May 2026 16:49:57 +0100 Subject: [PATCH 20/52] docs: document flex-model datetime flooring helper Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index dc258ac201..df5aea3059 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -227,6 +227,11 @@ class TriggerScheduleKwargsSchema(Schema): def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: + """Floor timed-event datetimes in list-valued flex-model fields. + + This only touches list entries that look like timed-event dictionaries, + such as ``soc-minima``, ``soc-maxima`` and ``soc-targets``. + """ if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( "round_datetimes_on_ingestion", True ): From 0d92e52e2d3ab941fb3aab8b541b6d1177447437 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 15 May 2026 16:50:17 +0100 Subject: [PATCH 21/52] test: clarify sensor_index assumptions in upload tests Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensors_api_freshdb.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index c5370c4d3d..01cb7e95c6 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -11,6 +11,20 @@ from flexmeasures.api.v3_0.tests.utils import generate_csv_content +TEST_BATTERY_SENSOR_NAMES = ( + "power", + "power (kW)", + "energy (kWh)", + "state of charge", + "consumption sensor", + "cost sensor", +) + + +def assert_test_battery_sensor(sensor, sensor_index: int) -> None: + assert sensor.name == TEST_BATTERY_SENSOR_NAMES[sensor_index] + + @pytest.mark.parametrize( "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_event_values, expected_status", [ @@ -187,6 +201,7 @@ def test_upload_sensor_data_with_unit_conversion_success( ) test_battery = add_battery_assets_fresh_db["Test battery"] sensor = test_battery.sensors[sensor_index] + assert_test_battery_sensor(sensor, sensor_index) num_test_intervals = len(data_values) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." @@ -290,6 +305,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( ): test_battery = add_battery_assets_fresh_db["Test battery"] sensor = test_battery.sensors[sensor_index] + assert_test_battery_sensor(sensor, sensor_index) csv_content = generate_csv_content( start_time_str=start_date, @@ -371,6 +387,7 @@ def test_upload_sensor_data_with_unit_conversion_failure( ) test_battery = add_battery_assets_fresh_db["Test battery"] sensor = test_battery.sensors[sensor_index] + assert_test_battery_sensor(sensor, sensor_index) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." ) From b7d3cf0251851202eef73baa01a309f7c157cf3f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:00:23 +0100 Subject: [PATCH 22/52] feat: handle floored downsampling posts Signed-off-by: Mohamed Belhsan Hmida --- .../api/common/schemas/sensor_data.py | 27 +++++++-- .../api/v3_0/tests/test_sensor_data.py | 55 ++++++++++++++----- 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 3701b0b198..b78e895682 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -369,7 +369,8 @@ def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): For a sensor recording instantaneous values, any event frequency is compatible. For a sensor recording non-instantaneous values, the event frequency must fit the sensor's event resolution. - Currently, only upsampling is supported (e.g. converting hourly events to 15-minute events). + Upsampling and downsampling are supported when the inferred resolution and + the sensor resolution are multiples of each other. """ required_resolution = data["sensor"].event_resolution @@ -380,7 +381,9 @@ def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): # The event frequency is inferred by assuming sequential, equidistant values within a time interval. # The event resolution is assumed to be equal to the event frequency. inferred_resolution = data["duration"] / len(data["values"]) - if inferred_resolution % required_resolution != timedelta(hours=0): + if inferred_resolution % required_resolution != timedelta( + hours=0 + ) and required_resolution % inferred_resolution != timedelta(hours=0): raise ValidationError( f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}." ) @@ -407,6 +410,7 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame data = self.possibly_upsample_values(data) data = self.possibly_convert_units(data) bdf = self.load_bdf(data) + bdf = self.possibly_downsample_bdf(bdf, data["sensor"].event_resolution) # Post-load validation against message type _type = data.get("type", None) @@ -429,7 +433,7 @@ def possibly_convert_units(data): data["values"], from_unit=data["unit"], to_unit=data["sensor"].unit, - event_resolution=data["sensor"].event_resolution, + event_resolution=data["duration"] / len(data["values"]), ) return data @@ -457,6 +461,21 @@ def possibly_upsample_values(data): ) return data + @staticmethod + def possibly_downsample_bdf( + bdf: BeliefsDataFrame, required_resolution: timedelta + ) -> BeliefsDataFrame: + """ + Downsample the data if needed, to fit the sensor's resolution. + Marshmallow runs this after validation. + """ + if required_resolution == timedelta(hours=0): + return bdf + + if bdf.event_resolution < required_resolution: + bdf = bdf.resample_events(required_resolution) + return bdf + @staticmethod def load_bdf(sensor_data: dict) -> BeliefsDataFrame: """ @@ -469,7 +488,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: sensor = sensor_data["sensor"] if sensor.event_resolution != timedelta(0) and sensor.get_attribute( - "round_datetimes_on_ingestion", True + "floor_datetimes_to_resolution", True ): start = pd.Timestamp(start).floor(sensor.event_resolution) elif frequency := sensor.get_attribute("frequency"): diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 880103de27..271ecfb830 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -200,10 +200,10 @@ def test_post_sensor_data_bad_auth( ("start", "2021-06-07T00:00:00", "start", "Not a valid aware datetime"), ( "duration", - "PT30M", + "PT25M", "_schema", - "Resolution of 0:05:00 is incompatible", - ), # downsampling not supported + "Resolution of 0:04:10 is incompatible", + ), ("unit", "m", "_schema", "Required unit"), ("type", "GetSensorDataRequest", "type", "Must be one of"), ], @@ -243,14 +243,40 @@ def test_post_invalid_sensor_data( @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) +@pytest.mark.parametrize( + "offclock_start, precise_start, precise_end, values, expected_values", + [ + ( + "2021-06-08T00:00:40+02:00", + "2021-06-08T00:00:00+02:00", + "2021-06-08T01:00:00+02:00", + [-11.28] * 6, + [-11.28] * 6, + ), + ( + "2021-06-09T00:00:40+02:00", + "2021-06-09T00:00:00+02:00", + "2021-06-09T01:00:00+02:00", + [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200], + [150, 350, 550, 750, 950, 1150], + ), + ], +) def test_post_non_instantaneous_sensor_data_floor( - client, setup_api_test_data, requesting_user + client, + setup_api_test_data, + requesting_user, + offclock_start, + precise_start, + precise_end, + values, + expected_values, ): - offclock_start = "2021-06-08T00:00:40+02:00" - precise_start = "2021-06-08T00:00:00+02:00" - precise_end = "2021-06-08T01:00:00+02:00" - post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") + post_data = make_sensor_data_request_for_gas_sensor( + num_values=len(values), unit="m³/h" + ) post_data["start"] = offclock_start + post_data["values"] = values sensor = setup_api_test_data["some gas sensor"] assert ( @@ -267,13 +293,14 @@ def test_post_non_instantaneous_sensor_data_floor( new_data = sensor.search_beliefs(precise_start, precise_end).reset_index() assert len(new_data) == 6 assert list(new_data["event_start"]) == [ - pd.Timestamp("2021-06-08 00:00:00+02"), - pd.Timestamp("2021-06-08 00:10:00+02"), - pd.Timestamp("2021-06-08 00:20:00+02"), - pd.Timestamp("2021-06-08 00:30:00+02"), - pd.Timestamp("2021-06-08 00:40:00+02"), - pd.Timestamp("2021-06-08 00:50:00+02"), + pd.Timestamp(precise_start), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=10), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=20), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=30), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=40), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=50), ] + assert new_data["event_value"].to_list() == expected_values @pytest.mark.parametrize( From 531d2dbabe1518335083bcddf552b07af94d4f6f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:00:28 +0100 Subject: [PATCH 23/52] test: cover datetime flooring opt-out Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 6 +- .../api/v3_0/tests/test_sensor_schedules.py | 57 ++++++++++++++++++- .../data/schemas/scheduling/storage.py | 10 ++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index df5aea3059..de040e0842 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -233,7 +233,7 @@ def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: such as ``soc-minima``, ``soc-maxima`` and ``soc-targets``. """ if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( - "round_datetimes_on_ingestion", True + "floor_datetimes_to_resolution", True ): return flex_model @@ -1251,6 +1251,8 @@ def fetch_one(self, id, sensor: Sensor): timezone: Europe/Amsterdam event_resolution: PT15M entity_address: ea1.2021-01.io.flexmeasures:fm1.14 + attributes: + floor_datetimes_to_resolution: true generic_asset_id: 1 power_sensor: summary: A power sensor recording average consumption every 5 minutes. @@ -1314,6 +1316,7 @@ def post(self, sensor_data: dict): "event_resolution": "PT1H" "unit": "kWh" "generic_asset_id": 1 + "attributes": '{"floor_datetimes_to_resolution": false}' responses: 201: description: New Sensor @@ -1330,6 +1333,7 @@ def post(self, sensor_data: dict): "entity_address": "ea1.2023-08.localhost:fm1.1" "event_resolution": "PT1H" "generic_asset_id": 1 + "attributes": '{"floor_datetimes_to_resolution": false}' "timezone": "UTC" "id": 2 400: diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 6aad9bd4cf..1e71006fff 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -32,6 +32,10 @@ def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets) return sensor +def get_sensor_by_name(asset, name: str) -> Sensor: + return next(sensor for sensor in asset.sensors if sensor.name == name) + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) @@ -132,7 +136,9 @@ def test_trigger_schedule_floors_flex_model_datetimes( for field in ("soc-targets", "soc-minima", "soc-maxima"): message["flex-model"][field][0]["datetime"] = offclock_target - sensor = add_charging_station_assets["Test charging station"].sensors[0] + sensor = get_sensor_by_name( + add_charging_station_assets["Test charging station"], "power" + ) with app.test_client() as client: trigger_schedule_response = client.post( url_for("SensorAPI:trigger_schedule", id=sensor.id), @@ -152,6 +158,47 @@ def test_trigger_schedule_floors_flex_model_datetimes( assert parse_datetime(target["end"]) == parse_datetime(expected_target) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_keeps_flex_model_datetimes_when_flooring_disabled( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + requesting_user, +): + message = message_for_trigger_schedule(with_targets=True) + offclock_target = "2015-01-02T23:00:40+01:00" + for field in ("soc-targets", "soc-minima", "soc-maxima"): + message["flex-model"][field][0]["datetime"] = offclock_target + + sensor = get_sensor_by_name( + add_charging_station_assets["Test charging station"], "power" + ) + original_attributes = dict(sensor.attributes or {}) + sensor.attributes = { + **original_attributes, + "floor_datetimes_to_resolution": False, + } + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + sensor.attributes = original_attributes + + assert trigger_schedule_response.status_code == 200 + assert len(app.queues["scheduling"]) == 1 + + job = app.queues["scheduling"].jobs[0] + for field in ("soc-targets", "soc-minima", "soc-maxima"): + target = job.kwargs["flex_model"][field][0] + assert target["datetime"] == offclock_target + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) @@ -321,7 +368,9 @@ def test_get_schedule_fallback( start = "2015-01-02T00:00:00+01:00" epex_da = get_test_sensor(db) - charging_station = add_charging_station_assets[charging_station_name].sensors[0] + charging_station = get_sensor_by_name( + add_charging_station_assets[charging_station_name], "power" + ) capacity = charging_station.get_attribute( "capacity_in_mw", @@ -479,7 +528,9 @@ def test_get_schedule_fallback_not_redirect( start = "2015-01-02T00:00:00+01:00" epex_da = get_test_sensor(db) - charging_station = add_charging_station_assets[charging_station_name].sensors[0] + charging_station = get_sensor_by_name( + add_charging_station_assets[charging_station_name], "power" + ) capacity = charging_station.get_attribute( "capacity_in_mw", diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index e5b2bf9ad1..a41e609e18 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -236,10 +236,10 @@ def __init__( self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None - self.rounding_resolution = ( + self.flooring_resolution = ( sensor.event_resolution if sensor is not None - and sensor.get_attribute("round_datetimes_on_ingestion", True) + and sensor.get_attribute("floor_datetimes_to_resolution", True) else None ) @@ -256,7 +256,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.rounding_resolution, + event_resolution=self.flooring_resolution, data_key="soc-maxima", ) @@ -264,7 +264,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.rounding_resolution, + event_resolution=self.flooring_resolution, data_key="soc-minima", value_validator=validate.Range(min=0), ) @@ -272,7 +272,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.rounding_resolution, + event_resolution=self.flooring_resolution, data_key="soc-targets", ) From 067bd92145e637fe0ef20bb99af2f4a2b79c7c21 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:00:35 +0100 Subject: [PATCH 24/52] fix: reject flooring event collisions Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 44 ++++++++++--- .../data/schemas/tests/test_sensor.py | 61 +++++++++++++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 8775531371..6437b539e3 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -235,8 +235,18 @@ def timezone_validator(value: str): attributes = JSON( required=False, metadata=dict( - description="JSON serializable attributes to store arbitrary information on the sensor. A few attributes lead to special behaviour, such as `consumption_is_positive`, which informs the platform whether consumption values should be saved (and shown in charts) as positive or negative values.", - example="{consumption_is_positive: True}", + description=( + "JSON serializable attributes to store arbitrary information on " + "the sensor. A few attributes lead to special behaviour, such as " + "`consumption_is_positive`, which informs the platform whether " + "consumption values should be saved (and shown in charts) as " + "positive or negative values, `floor_datetimes_to_resolution`, " + "which controls whether off-clock datetimes are floored to a " + "non-instantaneous sensor's resolution, and `frequency`, which " + "rounds incoming instantaneous measurements to a configured " + "Pandas frequency." + ), + example='{"consumption_is_positive": true, "floor_datetimes_to_resolution": true}', ), ) @@ -726,7 +736,7 @@ def post_load(self, fields, **kwargs): pd.infer_freq(bdf.index.unique("event_start")) if sensor.event_resolution != timedelta(0) and sensor.get_attribute( - "round_datetimes_on_ingestion", True + "floor_datetimes_to_resolution", True ): bdf = floor_bdf_event_starts(bdf, bdf.event_resolution) @@ -771,15 +781,31 @@ def post_load(self, fields, **kwargs): def floor_bdf_event_starts( bdf: tb.BeliefsDataFrame, event_resolution: timedelta ) -> tb.BeliefsDataFrame: - floored_bdf = pd.DataFrame(bdf.reset_index()) - floored_bdf["event_start"] = pd.DatetimeIndex(bdf.event_starts).floor( + floored_event_starts = bdf.index.get_level_values("event_start").floor( event_resolution ) - return tb.BeliefsDataFrame( - floored_bdf, - sensor=bdf.sensor, - event_resolution=bdf.event_resolution, + + new_index = pd.MultiIndex.from_arrays( + [ + ( + floored_event_starts + if name == "event_start" + else bdf.index.get_level_values(name) + ) + for name in bdf.index.names + ], + names=bdf.index.names, ) + if new_index.duplicated().any(): + raise ValidationError( + "Flooring event_start would merge multiple beliefs with the same " + "source, belief_time and event_start. Please provide data already " + "aligned to the event resolution or use distinct belief/source metadata." + ) + + floored_bdf = bdf.copy() + floored_bdf.index = new_index + return floored_bdf class QuantitySchema(Schema): diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a547eaa66a..50606a2bc4 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -1,8 +1,11 @@ import pytest +import pandas as pd +import timely_beliefs as tb from flexmeasures import Sensor from flexmeasures.data.schemas.sensors import ( QuantityOrSensor, VariableQuantityField, + floor_bdf_event_starts, ) from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -175,3 +178,61 @@ def test_time_series_field(input_param, dst_unit, fails, db): assert not fails except Exception as e: assert fails, e + + +def test_floor_bdf_event_starts(setup_dummy_sensors, setup_sources): + sensor1, _, _, _ = setup_dummy_sensors + belief_time = pd.Timestamp("2025-01-01T09:00:00+00:00") + bdf = tb.BeliefsDataFrame( + pd.DataFrame( + { + "event_start": pd.to_datetime( + [ + "2025-01-01T10:00:40+00:00", + "2025-01-01T10:15:40+00:00", + ] + ), + "belief_time": [belief_time, belief_time], + "source": [setup_sources["Seita"], setup_sources["Seita"]], + "event_value": [1.0, 2.0], + } + ), + sensor=sensor1, + event_resolution=pd.Timedelta(minutes=15), + ) + + floored_bdf = floor_bdf_event_starts(bdf, pd.Timedelta(minutes=15)) + + pd.testing.assert_index_equal( + floored_bdf.event_starts, + pd.DatetimeIndex( + ["2025-01-01T10:00:00+00:00", "2025-01-01T10:15:00+00:00"], + name="event_start", + ), + ) + assert floored_bdf["event_value"].to_list() == [1.0, 2.0] + + +def test_floor_bdf_event_starts_rejects_collisions(setup_dummy_sensors, setup_sources): + sensor1, _, _, _ = setup_dummy_sensors + belief_time = pd.Timestamp("2025-01-01T09:00:00+00:00") + bdf = tb.BeliefsDataFrame( + pd.DataFrame( + { + "event_start": pd.to_datetime( + [ + "2025-01-01T10:00:40+00:00", + "2025-01-01T10:08:10+00:00", + ] + ), + "belief_time": [belief_time, belief_time], + "source": [setup_sources["Seita"], setup_sources["Seita"]], + "event_value": [1.0, 2.0], + } + ), + sensor=sensor1, + event_resolution=pd.Timedelta(minutes=7, seconds=30), + ) + + with pytest.raises(ValidationError, match="would merge multiple beliefs"): + floor_bdf_event_starts(bdf, pd.Timedelta(minutes=15)) From fa31e0e3302b14cf9baa65e0620bdd9fd088f955 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:06:24 +0100 Subject: [PATCH 25/52] docs: document datetime flooring attribute Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/notation.rst | 11 ++++++++++- flexmeasures/ui/static/openapi-specs.json | 11 ++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index dd956c8c23..ddc6aaa2e7 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -120,6 +120,12 @@ In all current versions of the FlexMeasures API, only equidistant timeseries dat - "start" should be a timestamp on the hour or a multiple of the sensor resolution thereafter (e.g. "16:10" works if the resolution is 5 minutes), and - "duration" should also be a multiple of the sensor resolution. +For non-instantaneous sensors, FlexMeasures floors off-clock datetimes to the +sensor's resolution by default when ingesting sensor data. For example, data +posted with ``"start": "2026-05-12T08:29:58+02:00"`` to a 15-minute sensor is +saved from ``2026-05-12T08:15:00+02:00``. Set the sensor attribute +``"floor_datetimes_to_resolution": false`` to disable this behaviour. + .. _beliefs: @@ -249,7 +255,10 @@ FlexMeasures handles two types of time series, which can be distinguished by def Specifying a frequency and resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the frequency by dividing the duration by the number of values, and, for sensors that record non-instantaneous events, assumes the resolution of the data is equal to the frequency. When POSTing data, FlexMeasures checks this inferred resolution against the required resolution of the sensors that are posted to. -If these can't be matched (through upsampling), an error will occur. +If these can't be matched through upsampling or downsampling, an error will occur. +Off-clock event starts for non-instantaneous sensors are floored to the sensor's resolution by default. +The sensor attribute ``floor_datetimes_to_resolution`` can be set to ``false`` to keep incoming datetimes unchanged. +This flooring behaviour is distinct from the existing ``frequency`` sensor attribute, which rounds incoming instantaneous measurements to a configured Pandas frequency. GET requests (such as */sensors/data*) return data with a frequency either equal to the resolution that the sensor is configured for (for non-instantaneous sensors), or a default frequency befitting (in our opinion) the requested time interval. A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index a11c8b84e7..a8e6579eae 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -177,6 +177,9 @@ "timezone": "Europe/Amsterdam", "event_resolution": "PT15M", "entity_address": "ea1.2021-01.io.flexmeasures:fm1.14", + "attributes": { + "floor_datetimes_to_resolution": true + }, "generic_asset_id": 1 } }, @@ -1189,7 +1192,8 @@ "name": "power", "event_resolution": "PT1H", "unit": "kWh", - "generic_asset_id": 1 + "generic_asset_id": 1, + "attributes": "{\"floor_datetimes_to_resolution\": false}" } } } @@ -1214,6 +1218,7 @@ "entity_address": "ea1.2023-08.localhost:fm1.1", "event_resolution": "PT1H", "generic_asset_id": 1, + "attributes": "{\"floor_datetimes_to_resolution\": false}", "timezone": "UTC", "id": 2 } @@ -5699,8 +5704,8 @@ "description": "Obsolete identifier from [USEF](https://www.usef.energy/)." }, "attributes": { - "description": "JSON serializable attributes to store arbitrary information on the sensor. A few attributes lead to special behaviour, such as `consumption_is_positive`, which informs the platform whether consumption values should be saved (and shown in charts) as positive or negative values.", - "example": "{consumption_is_positive: True}" + "description": "JSON serializable attributes to store arbitrary information on the sensor. A few attributes lead to special behaviour, such as `consumption_is_positive`, which informs the platform whether consumption values should be saved (and shown in charts) as positive or negative values, `floor_datetimes_to_resolution`, which controls whether off-clock datetimes are floored to a non-instantaneous sensor's resolution, and `frequency`, which rounds incoming instantaneous measurements to a configured Pandas frequency.", + "example": "{\"consumption_is_positive\": true, \"floor_datetimes_to_resolution\": true}" }, "generic_asset_id": { "type": "integer", From cd4cfd2dd810bf50b65368526ea63e4cdfc48651 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:06:56 +0100 Subject: [PATCH 26/52] docs: add changelog entry Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 81de9d392a..30b4aebe20 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -13,6 +13,7 @@ New features * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_] * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] +* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] Infrastructure / Support ---------------------- From d4ee9454ac2f6ec1532f1729026e149b29ee28b8 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 21 May 2026 09:48:19 +0100 Subject: [PATCH 27/52] docs: align sensor data resampling docs Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 4 ++-- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 85febd182f..98ac717f74 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -582,7 +582,7 @@ def upload_data( The unit has to be convertible to the sensor's unit. The resolution of the data has to match the sensor's required resolution, but - FlexMeasures will attempt to upsample lower resolutions. + FlexMeasures will attempt to resample compatible resolutions. The list of values may include null values. The request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES (3 MiB by default). @@ -718,7 +718,7 @@ def post_data(self, id: int, sensor: Sensor, sensor_data: dict): The sensor is the one with ID=1. The unit has to be convertible to the sensor's unit. The resolution of the data has to match the sensor's required resolution, but - FlexMeasures will attempt to upsample lower resolutions. + FlexMeasures will attempt to resample compatible resolutions. The list of values may include null values. The request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES (3 MiB by default). diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b2c6ca38f7..3f24818074 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -487,7 +487,7 @@ }, "post": { "summary": "Post sensor data", - "description": "Send data values via JSON, where the duration and number of values determine the resolution.\n\nThe example request posts four values for a duration of one hour, where the first\nevent start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.\n\nThe sensor is the one with ID=1.\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", + "description": "Send data values via JSON, where the duration and number of values determine the resolution.\n\nThe example request posts four values for a duration of one hour, where the first\nevent start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.\n\nThe sensor is the one with ID=1.\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to resample compatible resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", "security": [ { "ApiAuthKey": [] @@ -1532,7 +1532,7 @@ "/api/v3_0/sensors/{id}/data/upload": { "post": { "summary": "Upload sensor data by file", - "description": "The file should have columns for a timestamp (event_start) and a value (event_value).\nThe timestamp should be in ISO 8601 format.\nThe value should be a numeric value.\n\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", + "description": "The file should have columns for a timestamp (event_start) and a value (event_value).\nThe timestamp should be in ISO 8601 format.\nThe value should be a numeric value.\n\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to resample compatible resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", "security": [ { "ApiKeyAuth": [] From 61583e328161bef3b107af67dd226dfe4a90ad79 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 05:23:31 +0100 Subject: [PATCH 28/52] fix: preserve submitted flex-model in schedule jobs Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 49 ------------------- .../api/v3_0/tests/test_sensor_schedules.py | 11 ++--- 2 files changed, 3 insertions(+), 57 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 98ac717f74..eab474f488 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,9 +1,7 @@ from __future__ import annotations import isodate -from copy import deepcopy from datetime import datetime, timedelta -import pandas as pd from flexmeasures.data.services.sensors import ( serialize_sensor_status_data, @@ -289,47 +287,6 @@ def support_legacy_field_name(self, data, **kwargs): return data -def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: - """Floor timed-event datetimes in list-valued flex-model fields. - - This only touches list entries that look like timed-event dictionaries, - such as ``soc-minima``, ``soc-maxima`` and ``soc-targets``. - """ - if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( - "floor_datetimes_to_resolution", True - ): - return flex_model - - floored_flex_model = deepcopy(flex_model) - for value_name, value in floored_flex_model.items(): - if not isinstance(value, list): - continue - for index, timed_event in enumerate(value): - if not isinstance(timed_event, dict): - continue - for key in ("datetime", "start", "end"): - if key in timed_event: - try: - timed_event[key] = isodate.datetime_isoformat( - pd.Timestamp(timed_event[key]).floor( - sensor.event_resolution - ) - ) - except (TypeError, ValueError) as exc: - raise ValidationError( - { - value_name: { - index: { - key: [ - f"Not a valid datetime: {timed_event[key]!r}." - ] - } - } - } - ) from exc - return floored_flex_model - - class SensorAPI(FlaskView): route_base = "/sensors" trailing_slash = False @@ -1045,12 +1002,6 @@ def trigger_schedule( f"Resolution of {resolution} is incompatible with the sensor's required resolution of {sensor.event_resolution}." ) - if flex_model is not None: - try: - flex_model = floor_timed_event_datetimes(flex_model, sensor) - except ValidationError as err: - return unprocessable_entity(err.messages) - end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( asset_or_sensor=sensor, diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 1e71006fff..2c5081a49c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -121,7 +121,7 @@ def test_trigger_schedule_with_invalid_flexmodel( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_trigger_schedule_floors_flex_model_datetimes( +def test_trigger_schedule_preserves_flex_model_datetimes_in_job_kwargs( app, add_market_prices, add_battery_assets, @@ -132,7 +132,6 @@ def test_trigger_schedule_floors_flex_model_datetimes( ): message = message_for_trigger_schedule(with_targets=True) offclock_target = "2015-01-02T23:00:40+01:00" - expected_target = "2015-01-02T23:00:00+01:00" for field in ("soc-targets", "soc-minima", "soc-maxima"): message["flex-model"][field][0]["datetime"] = offclock_target @@ -151,17 +150,13 @@ def test_trigger_schedule_floors_flex_model_datetimes( job = app.queues["scheduling"].jobs[0] for field in ("soc-targets", "soc-minima", "soc-maxima"): target = job.kwargs["flex_model"][field][0] - assert target["datetime"] == expected_target - if "start" in target: - assert parse_datetime(target["start"]) == parse_datetime(expected_target) - if "end" in target: - assert parse_datetime(target["end"]) == parse_datetime(expected_target) + assert target["datetime"] == offclock_target @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_trigger_schedule_keeps_flex_model_datetimes_when_flooring_disabled( +def test_trigger_schedule_preserves_flex_model_datetimes_when_flooring_disabled( app, add_market_prices, add_battery_assets, From 2eceadf7d1ba18cf0a0dd126c68b8d130294fa0a Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:00:59 +0100 Subject: [PATCH 29/52] feat: keep storage soc event times in schema Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/storage.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 9dd32c2ed9..e1c60c7143 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -237,12 +237,6 @@ def __init__( self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None - self.flooring_resolution = ( - sensor.event_resolution - if sensor is not None - and sensor.get_attribute("floor_datetimes_to_resolution", True) - else None - ) # guess default soc-unit if default_soc_unit is None: @@ -257,7 +251,6 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.flooring_resolution, data_key="soc-maxima", ) @@ -265,7 +258,6 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.flooring_resolution, data_key="soc-minima", value_validator=validate.Range(min=0), ) @@ -273,7 +265,6 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.flooring_resolution, data_key="soc-targets", ) From 1d37e4f5a8e933cf0a2b95b37d0f15e4f70f7e2e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:04 +0100 Subject: [PATCH 30/52] feat: project off-tick soc constraints to grid Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 289 ++++++++++++++++++- 1 file changed, 273 insertions(+), 16 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 93a67a3e96..3c1ddb1477 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -648,22 +648,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint soc_maxima[d] = None - if soc_at_start[d] is not None: - device_constraints[d] = add_storage_constraints( - start, - end, - resolution, - soc_at_start[d], - soc_targets[d], - soc_maxima[d], - soc_minima[d], - soc_max[d], - soc_min[d], - ) - else: - # No need to validate non-existing storage constraints - skip_validation = True - power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity( variable_quantity=power_capacity_in_mw[d], unit="MW", @@ -679,6 +663,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if sensor_d is not None and sensor_d.get_attribute( "is_strictly_non_positive" ): + production_capacity_d = pd.Series( + 0, index=power_capacity_in_mw[d].index + ) device_constraints[d]["derivative min"] = 0 else: production_capacity_d = get_continuous_series_sensor_or_quantity( @@ -747,6 +734,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if sensor_d is not None and sensor_d.get_attribute( "is_strictly_non_negative" ): + consumption_capacity_d = pd.Series( + 0, index=power_capacity_in_mw[d].index + ) device_constraints[d]["derivative max"] = 0 else: consumption_capacity_d = get_continuous_series_sensor_or_quantity( @@ -811,6 +801,44 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # consumption-capacity will become a hard constraint device_constraints[d]["derivative max"] = consumption_capacity_d + if soc_at_start[d] is not None: + if ( + sensor_d is not None + and sensor_d.event_resolution != timedelta(0) + and sensor_d.get_attribute("floor_datetimes_to_resolution", True) + ): + ( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + ) = normalize_off_tick_soc_constraints( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + consumption_capacity_d, + production_capacity_d, + resolution, + soc_min[d], + soc_max[d], + ) + + storage_constraints = add_storage_constraints( + start, + end, + resolution, + soc_at_start[d], + soc_targets[d], + soc_maxima[d], + soc_minima[d], + soc_max[d], + soc_min[d], + ) + for column in ("equals", "min", "max"): + device_constraints[d][column] = storage_constraints[column] + else: + # No need to validate non-existing storage constraints + skip_validation = True + all_stock_delta = [] for is_usage, soc_delta in zip([False, True], [soc_gain[d], soc_usage[d]]): @@ -1716,6 +1744,235 @@ def create_constraint_violations_message(constraint_violations: list) -> str: return message +def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: + timestamp = pd.Timestamp(dt) + return timestamp == timestamp.floor(resolution) + + +def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: + if isinstance(value, ur.Quantity): + return value.to("MWh").magnitude + return float(value) + + +def _soc_event_at( + soc_event: dict[str, datetime | float], + dt: pd.Timestamp, + value: float, +) -> dict[str, datetime | float]: + shifted_event = copy.copy(soc_event) + shifted_event["value"] = value + shifted_event["start"] = dt.to_pydatetime() + shifted_event["end"] = dt.to_pydatetime() + if "datetime" in shifted_event: + shifted_event["datetime"] = dt.to_pydatetime() + shifted_event.pop("duration", None) + return shifted_event + + +def _energy_capacity_between( + capacity: pd.Series, + start: pd.Timestamp, + end: pd.Timestamp, + resolution: timedelta, +) -> float: + if end <= start: + return 0 + + if capacity.index.tz is not None: + start = start.tz_convert(capacity.index.tz) + end = end.tz_convert(capacity.index.tz) + + capacity = capacity.astype(float).fillna(0) + tick = start.floor(resolution) + energy = 0.0 + while tick < end: + next_tick = tick + resolution + overlap_start = max(start, tick) + overlap_end = min(end, next_tick) + if overlap_end > overlap_start and tick in capacity.index: + energy += float(capacity.loc[tick]) * ( + (overlap_end - overlap_start) / pd.Timedelta(hours=1) + ) + tick = next_tick + return energy + + +def _add_soc_bound( + soc_events: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + bound_type: str, +) -> None: + for existing_event in soc_events: + if existing_event.get("start") == soc_event.get("start") and existing_event.get( + "end" + ) == soc_event.get("end"): + existing_value = _soc_value_in_mwh(existing_event["value"]) + soc_value = _soc_value_in_mwh(soc_event["value"]) + existing_event["value"] = ( + max(existing_value, soc_value) + if bound_type == "min" + else min(existing_value, soc_value) + ) + return + soc_events.append(soc_event) + + +def normalize_off_tick_soc_constraints( + soc_targets: list[dict[str, datetime | float]] | Sensor | None, + soc_maxima: list[dict[str, datetime | float]] | Sensor | None, + soc_minima: list[dict[str, datetime | float]] | Sensor | None, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: ur.Quantity | float, + soc_max: ur.Quantity | float, +) -> tuple[ + list[dict[str, datetime | float]] | Sensor | None, + list[dict[str, datetime | float]] | Sensor | None, + list[dict[str, datetime | float]] | Sensor | None, +]: + """Project off-tick point-like SoC constraints onto the scheduling grid.""" + + if isinstance(soc_minima, Sensor) or isinstance(soc_maxima, Sensor): + return soc_targets, soc_maxima, soc_minima + + normalized_minima = copy.deepcopy(soc_minima or []) + normalized_maxima = copy.deepcopy(soc_maxima or []) + + soc_min_value = _soc_value_in_mwh(soc_min) + soc_max_value = _soc_value_in_mwh(soc_max) + + if not isinstance(soc_targets, Sensor) and soc_targets is not None: + normalized_targets = [] + for soc_target in soc_targets: + if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( + soc_target["end"], resolution + ): + normalized_targets.append(copy.copy(soc_target)) + continue + + target_time = pd.Timestamp(soc_target["end"]) + previous_tick = target_time.floor(resolution) + next_tick = target_time.ceil(resolution) + target_value = _soc_value_in_mwh(soc_target["value"]) + + charge_before_target = _energy_capacity_between( + consumption_capacity, previous_tick, target_time, resolution + ) + discharge_before_target = _energy_capacity_between( + production_capacity, previous_tick, target_time, resolution + ) + + normalized_targets.append( + _soc_event_at(soc_target, next_tick, target_value) + ) + _add_soc_bound( + normalized_minima, + _soc_event_at( + soc_target, + previous_tick, + max(soc_min_value, target_value - charge_before_target), + ), + bound_type="min", + ) + _add_soc_bound( + normalized_maxima, + _soc_event_at( + soc_target, + previous_tick, + min(soc_max_value, target_value + discharge_before_target), + ), + bound_type="max", + ) + else: + normalized_targets = soc_targets + + for soc_minimum in copy.deepcopy(soc_minima or []): + if soc_minimum["start"] != soc_minimum["end"] or _is_on_schedule_tick( + soc_minimum["end"], resolution + ): + continue + + minimum_time = pd.Timestamp(soc_minimum["end"]) + previous_tick = minimum_time.floor(resolution) + next_tick = minimum_time.ceil(resolution) + minimum_value = _soc_value_in_mwh(soc_minimum["value"]) + _add_soc_bound( + normalized_minima, + _soc_event_at( + soc_minimum, + previous_tick, + max( + soc_min_value, + minimum_value + - _energy_capacity_between( + consumption_capacity, previous_tick, minimum_time, resolution + ), + ), + ), + bound_type="min", + ) + _add_soc_bound( + normalized_minima, + _soc_event_at( + soc_minimum, + next_tick, + max( + soc_min_value, + minimum_value + - _energy_capacity_between( + production_capacity, minimum_time, next_tick, resolution + ), + ), + ), + bound_type="min", + ) + + for soc_maximum in copy.deepcopy(soc_maxima or []): + if soc_maximum["start"] != soc_maximum["end"] or _is_on_schedule_tick( + soc_maximum["end"], resolution + ): + continue + + maximum_time = pd.Timestamp(soc_maximum["end"]) + previous_tick = maximum_time.floor(resolution) + next_tick = maximum_time.ceil(resolution) + maximum_value = _soc_value_in_mwh(soc_maximum["value"]) + _add_soc_bound( + normalized_maxima, + _soc_event_at( + soc_maximum, + previous_tick, + min( + soc_max_value, + maximum_value + + _energy_capacity_between( + production_capacity, previous_tick, maximum_time, resolution + ), + ), + ), + bound_type="max", + ) + _add_soc_bound( + normalized_maxima, + _soc_event_at( + soc_maximum, + next_tick, + min( + soc_max_value, + maximum_value + + _energy_capacity_between( + consumption_capacity, maximum_time, next_tick, resolution + ), + ), + ), + bound_type="max", + ) + + return normalized_targets, normalized_maxima or None, normalized_minima or None + + def build_device_soc_values( soc_values: ur.Quantity | list[dict[str, datetime | float]] | pd.Series | None, soc_at_start: float, From a7b01a6a711b4436499c5e33e53c0de4a6c061bc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:08 +0100 Subject: [PATCH 31/52] test: cover off-tick soc target projection Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index dbb5d792f7..4804593fbb 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -252,6 +252,51 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours +def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db): + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + }, + ) + + _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) + storage_constraints = device_constraints[0].tz_convert(tz) + + assert pd.isna(storage_constraints.loc[start, "equals"]) + assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From 7dea2a7a1cd26293376a5ad82090067a63a7db7d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:12 +0100 Subject: [PATCH 32/52] fix: relax off-tick soc constraints by default Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 3c1ddb1477..c4a30d3f57 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -44,6 +44,7 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] +SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") class MetaStorageScheduler(Scheduler): @@ -1043,6 +1044,7 @@ def deserialize_flex_config(self): self.flex_model = {} self.collect_flex_config() + self.enable_relax_soc_constraints_for_off_tick_soc_constraints() self.flex_context = FlexContextSchema().load(self.flex_context) if isinstance(self.flex_model, dict): @@ -1096,6 +1098,33 @@ def deserialize_flex_config(self): return self.flex_model + def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: + """Relax SoC constraints when off-tick SoC events require grid projection.""" + if self.flex_context is None: + self.flex_context = {} + if not self.flex_model_has_off_tick_soc_constraints(): + return + self.flex_context["relax-soc-constraints"] = True + + def flex_model_has_off_tick_soc_constraints(self) -> bool: + if isinstance(self.flex_model, dict): + resolution = self.resolution + if self.sensor is not None: + if not self.sensor.get_attribute("floor_datetimes_to_resolution", True): + return False + resolution = resolution or self.sensor.event_resolution + return flex_model_has_off_tick_soc_constraints( + self.flex_model, resolution=resolution + ) + if isinstance(self.flex_model, list): + return any( + flex_model_has_off_tick_soc_constraints( + flex_model, resolution=self.resolution + ) + for flex_model in self.flex_model + ) + return False + def has_soc_at_start(self) -> bool: return ( "soc-at-start" in self.flex_model @@ -1749,6 +1778,34 @@ def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: return timestamp == timestamp.floor(resolution) +def flex_model_has_off_tick_soc_constraints( + flex_model: dict, + resolution: timedelta | None, +) -> bool: + if resolution in (None, timedelta(0)): + return False + + for field_name in SOC_TIMED_EVENT_FIELDS: + field_value = flex_model.get(field_name) + if not isinstance(field_value, list): + continue + for soc_event in field_value: + if not isinstance(soc_event, dict): + continue + for timing_field in ("datetime", "start", "end"): + if soc_event.get(timing_field) is None: + continue + try: + is_on_tick = _is_on_schedule_tick( + soc_event[timing_field], resolution + ) + except (TypeError, ValueError): + continue + if not is_on_tick: + return True + return False + + def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: if isinstance(value, ur.Quantity): return value.to("MWh").magnitude From e7b9ee1a8112d2c1d035864a6a7b804b261ae418 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:16 +0100 Subject: [PATCH 33/52] test: cover off-tick soc relaxation Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 4804593fbb..eaac9de055 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -297,6 +297,46 @@ def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) +def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "relax-soc-constraints": False, + }, + ) + + scheduler.deserialize_config() + + assert scheduler.flex_context["relax_soc_constraints"] is True + assert scheduler.flex_context["soc_minima_breach_price"] is not None + assert scheduler.flex_context["soc_maxima_breach_price"] is not None + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From 03cf1e20fd518bf412e38c9ab0c4d7e3c3cdfcc5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:05:50 +0100 Subject: [PATCH 34/52] fix: preserve non-event soc constraints Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 64 ++++++++++++++++---- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index c4a30d3f57..05880fc8cf 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1875,32 +1875,58 @@ def _add_soc_bound( soc_events.append(soc_event) +def _normalized_soc_events_or_original( + original_soc_events: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + normalized_soc_events: list[dict[str, datetime | float]], +) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: + if isinstance(original_soc_events, list): + return normalized_soc_events + if original_soc_events is None and normalized_soc_events: + return normalized_soc_events + return original_soc_events + + def normalize_off_tick_soc_constraints( - soc_targets: list[dict[str, datetime | float]] | Sensor | None, - soc_maxima: list[dict[str, datetime | float]] | Sensor | None, - soc_minima: list[dict[str, datetime | float]] | Sensor | None, + soc_targets: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_maxima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_minima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), consumption_capacity: pd.Series, production_capacity: pd.Series, resolution: timedelta, soc_min: ur.Quantity | float, soc_max: ur.Quantity | float, ) -> tuple[ - list[dict[str, datetime | float]] | Sensor | None, - list[dict[str, datetime | float]] | Sensor | None, - list[dict[str, datetime | float]] | Sensor | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, ]: """Project off-tick point-like SoC constraints onto the scheduling grid.""" - if isinstance(soc_minima, Sensor) or isinstance(soc_maxima, Sensor): + if not any( + isinstance(soc_events, list) and soc_events + for soc_events in (soc_targets, soc_maxima, soc_minima) + ): return soc_targets, soc_maxima, soc_minima - normalized_minima = copy.deepcopy(soc_minima or []) - normalized_maxima = copy.deepcopy(soc_maxima or []) + normalized_minima = ( + copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + ) + normalized_maxima = ( + copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] + ) soc_min_value = _soc_value_in_mwh(soc_min) soc_max_value = _soc_value_in_mwh(soc_max) - if not isinstance(soc_targets, Sensor) and soc_targets is not None: + if isinstance(soc_targets, list): normalized_targets = [] for soc_target in soc_targets: if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( @@ -1945,7 +1971,11 @@ def normalize_off_tick_soc_constraints( else: normalized_targets = soc_targets - for soc_minimum in copy.deepcopy(soc_minima or []): + if isinstance(soc_minima, list): + soc_minimum_events = copy.deepcopy(soc_minima) + else: + soc_minimum_events = [] + for soc_minimum in soc_minimum_events: if soc_minimum["start"] != soc_minimum["end"] or _is_on_schedule_tick( soc_minimum["end"], resolution ): @@ -1986,7 +2016,11 @@ def normalize_off_tick_soc_constraints( bound_type="min", ) - for soc_maximum in copy.deepcopy(soc_maxima or []): + if isinstance(soc_maxima, list): + soc_maximum_events = copy.deepcopy(soc_maxima) + else: + soc_maximum_events = [] + for soc_maximum in soc_maximum_events: if soc_maximum["start"] != soc_maximum["end"] or _is_on_schedule_tick( soc_maximum["end"], resolution ): @@ -2027,7 +2061,11 @@ def normalize_off_tick_soc_constraints( bound_type="max", ) - return normalized_targets, normalized_maxima or None, normalized_minima or None + return ( + normalized_targets, + _normalized_soc_events_or_original(soc_maxima, normalized_maxima), + _normalized_soc_events_or_original(soc_minima, normalized_minima), + ) def build_device_soc_values( From 9e26e034495341ffb34be954e111d5e1f6992424 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:51:40 +0100 Subject: [PATCH 35/52] fix: normalize off-tick soc on schedule grid Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 36 ++++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 05880fc8cf..5d4b4777a0 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -803,11 +803,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["derivative max"] = consumption_capacity_d if soc_at_start[d] is not None: - if ( - sensor_d is not None - and sensor_d.event_resolution != timedelta(0) - and sensor_d.get_attribute("floor_datetimes_to_resolution", True) - ): + if _should_normalize_off_tick_soc_constraints(sensor_d): ( soc_targets[d], soc_maxima[d], @@ -1108,18 +1104,20 @@ def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: def flex_model_has_off_tick_soc_constraints(self) -> bool: if isinstance(self.flex_model, dict): - resolution = self.resolution - if self.sensor is not None: - if not self.sensor.get_attribute("floor_datetimes_to_resolution", True): - return False - resolution = resolution or self.sensor.event_resolution + if not _should_normalize_off_tick_soc_constraints(self.sensor): + return False + resolution = _get_soc_constraint_resolution( + self.resolution, self.sensor, self.default_resolution + ) return flex_model_has_off_tick_soc_constraints( self.flex_model, resolution=resolution ) if isinstance(self.flex_model, list): + resolution = self.resolution or self.default_resolution return any( flex_model_has_off_tick_soc_constraints( - flex_model, resolution=self.resolution + flex_model.get("sensor-flex-model", flex_model), + resolution=resolution, ) for flex_model in self.flex_model ) @@ -1778,6 +1776,22 @@ def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: return timestamp == timestamp.floor(resolution) +def _get_soc_constraint_resolution( + schedule_resolution: timedelta | None, + sensor: Sensor | None, + default_resolution: timedelta, +) -> timedelta | None: + if schedule_resolution not in (None, timedelta(0)): + return schedule_resolution + if sensor is not None and sensor.event_resolution != timedelta(0): + return sensor.event_resolution + return default_resolution + + +def _should_normalize_off_tick_soc_constraints(sensor: Sensor | None) -> bool: + return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) + + def flex_model_has_off_tick_soc_constraints( flex_model: dict, resolution: timedelta | None, From 9e4e7093f2e12814c3f20430fbab40e55addf454 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:52:05 +0100 Subject: [PATCH 36/52] test: cover off-tick soc bounds Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 187 +++++++++++++++++- 1 file changed, 186 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index eaac9de055..5927f0655c 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -7,7 +7,10 @@ import pandas as pd from flexmeasures.data.models.planning import Scheduler -from flexmeasures.data.models.planning.storage import StorageScheduler +from flexmeasures.data.models.planning.storage import ( + StorageScheduler, + normalize_off_tick_soc_constraints, +) from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.planning.tests.utils import ( @@ -297,6 +300,188 @@ def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) +def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( + add_battery_assets, db +): + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + instantaneous_power_sensor = Sensor( + name="instantaneous-power", + generic_asset=battery.generic_asset, + event_resolution=timedelta(0), + unit="MW", + ) + db.session.add(instantaneous_power_sensor) + db.session.flush() + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + instantaneous_power_sensor, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + }, + ) + + _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) + storage_constraints = device_constraints[0].tz_convert(tz) + + assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + + +@pytest.mark.parametrize( + "soc_minima, soc_maxima, expected_previous_value, expected_next_value", + [ + ( + [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 1, + } + ], + None, + 0.992, + 1, + ), + ( + None, + [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 0.5, + } + ], + 0.5, + 0.502, + ), + ], +) +def test_off_tick_soc_bounds_are_projected_to_storage_grid( + soc_minima, + soc_maxima, + expected_previous_value, + expected_next_value, +): + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=soc_maxima, + soc_minima=soc_minima, + consumption_capacity=capacity, + production_capacity=pd.Series(0, index=capacity.index), + resolution=resolution, + soc_min=0, + soc_max=1, + ) + + normalized_events = normalized_minima or normalized_maxima + + assert _soc_event_value_at(normalized_events, previous_tick) == pytest.approx( + expected_previous_value + ) + assert _soc_event_value_at(normalized_events, next_tick) == pytest.approx( + expected_next_value + ) + + +def _soc_event_value_at(events, dt): + matches = [ + event + for event in events + if pd.Timestamp(event["start"]) == dt and pd.Timestamp(event["end"]) == dt + ] + assert len(matches) == 1 + return matches[0]["value"] + + +def test_off_tick_soc_bounds_are_merged_on_the_same_storage_grid_tick(): + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 4)), + "start": tz.localize(datetime(2015, 1, 1, 17, 4)), + "end": tz.localize(datetime(2015, 1, 1, 17, 4)), + "value": 0.8, + }, + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 8)), + "start": tz.localize(datetime(2015, 1, 1, 17, 8)), + "end": tz.localize(datetime(2015, 1, 1, 17, 8)), + "value": 0.6, + }, + ], + soc_minima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 4)), + "start": tz.localize(datetime(2015, 1, 1, 17, 4)), + "end": tz.localize(datetime(2015, 1, 1, 17, 4)), + "value": 0.4, + }, + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 8)), + "start": tz.localize(datetime(2015, 1, 1, 17, 8)), + "end": tz.localize(datetime(2015, 1, 1, 17, 8)), + "value": 0.7, + }, + ], + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=0, + soc_max=1, + ) + + assert _soc_event_value_at(normalized_minima, previous_tick) == pytest.approx(0.7) + assert _soc_event_value_at(normalized_minima, next_tick) == pytest.approx(0.7) + assert _soc_event_value_at(normalized_maxima, previous_tick) == pytest.approx(0.6) + assert _soc_event_value_at(normalized_maxima, next_tick) == pytest.approx(0.6) + + def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" From 32a3d2b5e93460981beedd752dd57f6d1d29c217 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:55:51 +0100 Subject: [PATCH 37/52] test: tolerate upload conversion precision Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 01cb7e95c6..43cbb06b5f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -245,7 +245,7 @@ def test_upload_sensor_data_with_unit_conversion_success( len(beliefs) == expected_num_beliefs ), f"Fetched {len(beliefs)} beliefs from the database, expecting {expected_num_beliefs}." - assert [b.event_value for b in beliefs] == expected_event_values + assert [b.event_value for b in beliefs] == pytest.approx(expected_event_values) @pytest.mark.parametrize( From 10fc632192785b28caf225590bfbb43b90501fbc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 14:42:53 +0100 Subject: [PATCH 38/52] test: compare quantities by value Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/tests/test_sensor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 50606a2bc4..a712922227 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -11,6 +11,11 @@ from marshmallow import ValidationError +def assert_quantity_matches(actual_quantity, expected_quantity) -> None: + assert str(actual_quantity.units) == str(expected_quantity.units) + assert actual_quantity.magnitude == pytest.approx(expected_quantity.magnitude) + + @pytest.mark.parametrize( "src_quantity, dst_unit, fails, exp_dst_quantity", [ @@ -84,10 +89,12 @@ def test_quantity_or_sensor_deserialize( try: dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): - assert dst_quantity == ur.Quantity(exp_dst_quantity) + assert_quantity_matches(dst_quantity, ur.Quantity(exp_dst_quantity)) assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): - assert dst_quantity[0]["value"] == ur.Quantity(exp_dst_quantity) + assert_quantity_matches( + dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) + ) assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: From 56ad4ebe71586392fbce31cae1397217ca550287 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 18:30:08 +0100 Subject: [PATCH 39/52] test: avoid exact quantity string comparison Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/tests/test_sensor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a712922227..d07a6160d3 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -95,7 +95,6 @@ def test_quantity_or_sensor_deserialize( assert_quantity_matches( dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) ) - assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: assert fails, e From 9268de88b25733c6b3e5608bbe00bfe44d50094e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 26 May 2026 08:30:46 +0100 Subject: [PATCH 40/52] fix: skip flooring for instantaneous sensors Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/scheduling/storage.py | 1 + .../data/schemas/tests/test_scheduling.py | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 9dd32c2ed9..c6aaaca2db 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -240,6 +240,7 @@ def __init__( self.flooring_resolution = ( sensor.event_resolution if sensor is not None + and sensor.event_resolution != timedelta(0) and sensor.get_attribute("floor_datetimes_to_resolution", True) else None ) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 4fb1bf67af..cf2c7c52a7 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta import pytz import pytest @@ -14,6 +14,7 @@ StorageFlexModelSchema, DBStorageFlexModelSchema, ) +from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.schemas.sensors import TimedEventSchema, VariableQuantityField @@ -162,6 +163,26 @@ def test_process_scheduler_flex_model_process_type(db, app, setup_dummy_sensors) assert process_scheduler_flex_model["process_type"] == ProcessType.SHIFTABLE +def test_storage_flex_model_schema_does_not_floor_instantaneous_sensor( + db, app, dummy_asset +): + sensor = Sensor( + "instantaneous power sensor", + generic_asset=dummy_asset, + event_resolution=timedelta(0), + unit="MW", + ) + db.session.add(sensor) + db.session.flush() + + schema = StorageFlexModelSchema( + sensor=sensor, + start=datetime(2023, 1, 1, tzinfo=pytz.UTC), + ) + + assert schema.flooring_resolution is None + + @pytest.mark.parametrize( "fields, fails", [ From 7c0c9b42f6a0e99c330bbd8ffe1fb220104f1db0 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 26 May 2026 08:30:52 +0100 Subject: [PATCH 41/52] test: fetch upload sensors by name Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensor_schedules.py | 9 +-- .../v3_0/tests/test_sensors_api_freshdb.py | 75 +++++++------------ flexmeasures/api/v3_0/tests/utils.py | 5 ++ 3 files changed, 38 insertions(+), 51 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 2c5081a49c..6928c452ab 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -10,7 +10,10 @@ from flexmeasures.api.common.responses import unknown_schedule, unrecognized_event from flexmeasures.api.tests.utils import check_deprecation -from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule +from flexmeasures.api.v3_0.tests.utils import ( + get_sensor_by_name, + message_for_trigger_schedule, +) from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor from flexmeasures.utils.job_utils import work_on_rq @@ -32,10 +35,6 @@ def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets) return sensor -def get_sensor_by_name(asset, name: str) -> Sensor: - return next(sensor for sensor in asset.sensors if sensor.name == name) - - @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 01cb7e95c6..8c695a17e4 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -8,29 +8,15 @@ from timely_beliefs import BeliefsDataFrame from flexmeasures.data.models.time_series import TimedBelief -from flexmeasures.api.v3_0.tests.utils import generate_csv_content - - -TEST_BATTERY_SENSOR_NAMES = ( - "power", - "power (kW)", - "energy (kWh)", - "state of charge", - "consumption sensor", - "cost sensor", -) - - -def assert_test_battery_sensor(sensor, sensor_index: int) -> None: - assert sensor.name == TEST_BATTERY_SENSOR_NAMES[sensor_index] +from flexmeasures.api.v3_0.tests.utils import generate_csv_content, get_sensor_by_name @pytest.mark.parametrize( - "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_event_values, expected_status", + "requesting_user, sensor_name, data_unit, data_resolution, data_values, expected_event_values, expected_status", [ ( "test_prosumer_user_2@seita.nl", - 2, # this sensor has unit=kWh, res=01:00 + "energy (kWh)", # this sensor has unit=kWh, res=01:00 "kWh", # No conversion needed - kWh to kWh timedelta(hours=1), # No resampling - 1 hour to 1 hour [45.3] * 4, @@ -39,7 +25,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 0, # this sensor has unit=MW, res=00:15 + "power", # this sensor has unit=MW, res=00:15 "kWh", # Conversion needed - kWh to MW timedelta(hours=1), # Upsampling - 1 hour to 15 minutes [45.3] * 4, @@ -50,7 +36,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "MW", # Conversion needed - MW to kW timedelta(hours=1), # Upsampling - 1 hour to 15 minutes [2] * 6, @@ -61,7 +47,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "kWh", # Conversion needed - kWh to kW # Upsampling - 30 minutes to 15 minutes timedelta(minutes=30), @@ -73,7 +59,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 2, # this sensor has unit=kWh, res=01:00 + "energy (kWh)", # this sensor has unit=kWh, res=01:00 "kWh", # No conversion needed - kWh to kWh timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [10, 20, 20, 40], @@ -85,7 +71,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "kWh", # Conversion needed - kWh to kW # Downsampling - 7.5 minutes to 15 minutes timedelta(minutes=7, seconds=30), @@ -98,7 +84,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "MW", # Conversion needed - MW to kW # Downsampling - 7.5 minutes to 15 minutes timedelta(minutes=7, seconds=30), @@ -112,7 +98,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 3, # this sensor has unit=kWh, res=00:00 + "state of charge", # this sensor has unit=kWh, res=00:00 "MWh", # Conversion needed - MWh to kWh # No resampling - 7.5 minutes to instantaneous timedelta(minutes=7, seconds=30), @@ -122,7 +108,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 4, # this sensor has unit=EUR/kWh, res=01:00 + "consumption sensor", # this sensor has unit=EUR/kWh, res=01:00 "EUR/MWh", # Conversion needed - EUR/MWh to EUR/kWh timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [200, 300, 400, 500], @@ -131,7 +117,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 4, # this sensor has unit=EUR/kWh, res=01:00 + "consumption sensor", # this sensor has unit=EUR/kWh, res=01:00 "EUR/kWh", # Conversion needed - EUR/kWh to EUR/kWh timedelta(hours=2), # Upsampling - 2 hours to 1 hour [200, 300, 400], @@ -140,7 +126,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 5, # this sensor has unit=EUR, res=01:00 + "cost sensor", # this sensor has unit=EUR, res=01:00 "kEUR", # Conversion needed - kEUR to EUR timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [2, 3, 4, 2], @@ -150,7 +136,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 5, # this sensor has unit=EUR, res=01:00 + "cost sensor", # this sensor has unit=EUR, res=01:00 "kEUR", # Conversion needed - kEUR to EUR timedelta(hours=2), # Upsampling - 2 hours to 1 hour [5, 6], @@ -163,7 +149,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 5, # this sensor has unit=EUR, res=01:00 + "cost sensor", # this sensor has unit=EUR, res=01:00 "kEUR", # Conversion needed - kEUR to EUR timedelta(hours=1), # No resampling - 1 hour (!) to 1 hour # Note that this test case could also define 2 hours between rows, but since there is only 1 row of data, @@ -181,7 +167,7 @@ def test_upload_sensor_data_with_unit_conversion_success( client, add_battery_assets_fresh_db, requesting_user, - sensor_index, + sensor_name, data_unit, data_resolution, data_values, @@ -200,8 +186,7 @@ def test_upload_sensor_data_with_unit_conversion_success( "2025-01-01T10:00:00+00:00" # This date would be used to generate CSV content ) test_battery = add_battery_assets_fresh_db["Test battery"] - sensor = test_battery.sensors[sensor_index] - assert_test_battery_sensor(sensor, sensor_index) + sensor = get_sensor_by_name(test_battery, sensor_name) num_test_intervals = len(data_values) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." @@ -249,11 +234,11 @@ def test_upload_sensor_data_with_unit_conversion_success( @pytest.mark.parametrize( - "requesting_user, sensor_index, start_date, data_resolution, data_values, expected_event_starts, expected_event_values", + "requesting_user, sensor_name, start_date, data_resolution, data_values, expected_event_starts, expected_event_values", [ ( "test_prosumer_user_2@seita.nl", - 1, + "power (kW)", "2025-01-01T10:00:40+00:00", timedelta(minutes=15), [2, 3, 4], @@ -266,7 +251,7 @@ def test_upload_sensor_data_with_unit_conversion_success( ), ( "test_prosumer_user_2@seita.nl", - 2, + "energy (kWh)", "2025-01-01T10:00:40+00:00", timedelta(minutes=30), [10, 20, 20, 40], @@ -278,7 +263,7 @@ def test_upload_sensor_data_with_unit_conversion_success( ), ( "test_prosumer_user_2@seita.nl", - 1, + "power (kW)", "2025-01-01T10:00:40+00:00", timedelta(minutes=5), [10, 20, 30, 40], @@ -296,7 +281,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( client, add_battery_assets_fresh_db, requesting_user, - sensor_index, + sensor_name, start_date, data_resolution, data_values, @@ -304,8 +289,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( expected_event_values, ): test_battery = add_battery_assets_fresh_db["Test battery"] - sensor = test_battery.sensors[sensor_index] - assert_test_battery_sensor(sensor, sensor_index) + sensor = get_sensor_by_name(test_battery, sensor_name) csv_content = generate_csv_content( start_time_str=start_date, @@ -331,11 +315,11 @@ def test_upload_sensor_data_floors_offclock_datetimes( @pytest.mark.parametrize( - "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_err_msg, expected_status", + "requesting_user, sensor_name, data_unit, data_resolution, data_values, expected_err_msg, expected_status", [ ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "m/s", # Invalid conversion - m/s to kW timedelta(hours=1), # Upsampling - 1 hour to 15 minutes [45.3, 45.3], @@ -344,7 +328,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( ), ( "test_prosumer_user_2@seita.nl", - 2, # this sensor has unit=kWh, res=01:00 + "energy (kWh)", # this sensor has unit=kWh, res=01:00 "kW", # Conversion needed - kW to kWh timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [20, 40, 40, 80], @@ -353,7 +337,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( ), ( "test_prosumer_user_2@seita.nl", - 3, # this sensor has unit=kWh, res=00:00 + "state of charge", # this sensor has unit=kWh, res=00:00 "kW", # Conversion needed - kW to kWh # No resampling - 7.5 minutes to instantaneous timedelta(minutes=7, seconds=30), @@ -369,7 +353,7 @@ def test_upload_sensor_data_with_unit_conversion_failure( client, add_battery_assets_fresh_db, requesting_user, - sensor_index, + sensor_name, data_unit, data_resolution, data_values, @@ -386,8 +370,7 @@ def test_upload_sensor_data_with_unit_conversion_failure( "2025-01-01T10:00:00+00:00" # This date would be used to generate CSV content ) test_battery = add_battery_assets_fresh_db["Test battery"] - sensor = test_battery.sensors[sensor_index] - assert_test_battery_sensor(sensor, sensor_index) + sensor = get_sensor_by_name(test_battery, sensor_name) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." ) diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py index 5409174393..79683e868c 100644 --- a/flexmeasures/api/v3_0/tests/utils.py +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -6,6 +6,11 @@ from flexmeasures import Asset, User from flexmeasures.data.models.audit_log import AssetAuditLog +from flexmeasures.data.models.time_series import Sensor + + +def get_sensor_by_name(asset: Asset, name: str) -> Sensor: + return next(sensor for sensor in asset.sensors if sensor.name == name) def make_sensor_data_request_for_gas_sensor( From baf55433bd4c95f6b05ea7518c450a98392822f0 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Tue, 26 May 2026 08:36:37 +0100 Subject: [PATCH 42/52] Update flexmeasures/api/v3_0/tests/test_sensor_data.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 797eb9f1e6..40e8455acc 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -442,7 +442,7 @@ def test_post_sensor_data_rejects_unknown_sensor_before_queueing( [-11.28] * 6, ), ( - "2021-06-09T00:00:40+02:00", + "2021-06-09T00:04:40+02:00", # floor rather than ceil to 5-min "2021-06-09T00:00:00+02:00", "2021-06-09T01:00:00+02:00", [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200], From 8b2bb7eb853e803e373e2760d0dc7b3f1bc17425 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Wed, 27 May 2026 02:44:25 +0100 Subject: [PATCH 43/52] Update flexmeasures/api/v3_0/tests/test_sensor_data.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 40e8455acc..78b895523e 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -360,6 +360,7 @@ def test_post_sensor_data_rejects_large_json( "duration", "PT25M", "_schema", + # PT25M / 6 values = 4m10s "Resolution of 0:04:10 is incompatible", ), ("unit", "m", "_schema", "Required unit"), From 26a8e22bb0e1c3223531cb702e22732d76f474a6 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:05:57 +0100 Subject: [PATCH 44/52] docs: update changelog for soc projection Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index f4fdd79045..c014c30490 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,7 +9,7 @@ v1.0.0 | July XX, 2026 New features ------------- -* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] +* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_ and `PR #2194 `_] * Sensor references in flex-model and flex-context support various ways of filtering by source [see `PR #2209 `_] @@ -48,7 +48,6 @@ New features * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * Add ``flexmeasures jobs inspect-job`` CLI command to show job status and metadata information (similar to the job status endpoint in the API) [see `PR #2202 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] -* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] * Making monitoring alerts more flexible: allow ``flexmeasures monitor`` alerts to target one or more user IDs or email addresses with ``--recipient``; ``flexmeasures monitor last-seen`` can now narrow monitored users to one or more accounts with ``--account`` or to client accounts with ``--consultancy`` [see `PR #2158 `_] * Improve LightGBM daily seasonal lag handling for sub-hourly forecasting sensors [see `PR #2157 `_] From 95e592258d8c2f73881e9cb17002f454fcfba5bb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:07:50 +0100 Subject: [PATCH 45/52] docs: clarify soc projection terminology Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 4 ++-- flexmeasures/data/models/planning/tests/test_storage.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 0a81697d4e..38cf4abd29 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1128,7 +1128,7 @@ def deserialize_flex_config(self): return self.flex_model def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: - """Relax SoC constraints when off-tick SoC events require grid projection.""" + """Relax SoC constraints when off-tick SoC events require scheduling-tick projection.""" if self.flex_context is None: self.flex_context = {} if not self.flex_model_has_off_tick_soc_constraints(): @@ -2125,7 +2125,7 @@ def normalize_off_tick_soc_constraints( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, ]: - """Project off-tick point-like SoC constraints onto the scheduling grid.""" + """Project off-tick point-like SoC constraints onto the scheduling ticks.""" if not any( isinstance(soc_events, list) and soc_events diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 3a3609e4d0..4a8c190396 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -293,7 +293,7 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours -def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db): +def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets, db): _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -423,7 +423,7 @@ def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( ), ], ) -def test_off_tick_soc_bounds_are_projected_to_storage_grid( +def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( soc_minima, soc_maxima, expected_previous_value, @@ -468,7 +468,7 @@ def _soc_event_value_at(events, dt): return matches[0]["value"] -def test_off_tick_soc_bounds_are_merged_on_the_same_storage_grid_tick(): +def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): tz = pytz.timezone("Europe/Amsterdam") resolution = timedelta(minutes=15) previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) From 8b07cfa2e96846ff0d1177de507587825ba53b16 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:15:57 +0100 Subject: [PATCH 46/52] refactor: rename soc normalization to projection Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 56 +++++++++---------- .../models/planning/tests/test_storage.py | 20 +++---- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 38cf4abd29..fcf0f24191 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -806,12 +806,12 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["derivative max"] = consumption_capacity_d if soc_at_start[d] is not None: - if _should_normalize_off_tick_soc_constraints(sensor_d): + if _should_project_off_tick_soc_constraints(sensor_d): ( soc_targets[d], soc_maxima[d], soc_minima[d], - ) = normalize_off_tick_soc_constraints( + ) = project_off_tick_soc_constraints( soc_targets[d], soc_maxima[d], soc_minima[d], @@ -1137,7 +1137,7 @@ def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: def flex_model_has_off_tick_soc_constraints(self) -> bool: if isinstance(self.flex_model, dict): - if not _should_normalize_off_tick_soc_constraints(self.sensor): + if not _should_project_off_tick_soc_constraints(self.sensor): return False resolution = _get_soc_constraint_resolution( self.resolution, self.sensor, self.default_resolution @@ -1991,7 +1991,7 @@ def _get_soc_constraint_resolution( return default_resolution -def _should_normalize_off_tick_soc_constraints(sensor: Sensor | None) -> bool: +def _should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) @@ -2092,20 +2092,20 @@ def _add_soc_bound( soc_events.append(soc_event) -def _normalized_soc_events_or_original( +def _projected_soc_events_or_original( original_soc_events: ( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None ), - normalized_soc_events: list[dict[str, datetime | float]], + projected_soc_events: list[dict[str, datetime | float]], ) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: if isinstance(original_soc_events, list): - return normalized_soc_events - if original_soc_events is None and normalized_soc_events: - return normalized_soc_events + return projected_soc_events + if original_soc_events is None and projected_soc_events: + return projected_soc_events return original_soc_events -def normalize_off_tick_soc_constraints( +def project_off_tick_soc_constraints( soc_targets: ( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None ), @@ -2133,23 +2133,19 @@ def normalize_off_tick_soc_constraints( ): return soc_targets, soc_maxima, soc_minima - normalized_minima = ( - copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] - ) - normalized_maxima = ( - copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - ) + projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] soc_min_value = _soc_value_in_mwh(soc_min) soc_max_value = _soc_value_in_mwh(soc_max) if isinstance(soc_targets, list): - normalized_targets = [] + projected_targets = [] for soc_target in soc_targets: if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( soc_target["end"], resolution ): - normalized_targets.append(copy.copy(soc_target)) + projected_targets.append(copy.copy(soc_target)) continue target_time = pd.Timestamp(soc_target["end"]) @@ -2164,11 +2160,9 @@ def normalize_off_tick_soc_constraints( production_capacity, previous_tick, target_time, resolution ) - normalized_targets.append( - _soc_event_at(soc_target, next_tick, target_value) - ) + projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) _add_soc_bound( - normalized_minima, + projected_minima, _soc_event_at( soc_target, previous_tick, @@ -2177,7 +2171,7 @@ def normalize_off_tick_soc_constraints( bound_type="min", ) _add_soc_bound( - normalized_maxima, + projected_maxima, _soc_event_at( soc_target, previous_tick, @@ -2186,7 +2180,7 @@ def normalize_off_tick_soc_constraints( bound_type="max", ) else: - normalized_targets = soc_targets + projected_targets = soc_targets if isinstance(soc_minima, list): soc_minimum_events = copy.deepcopy(soc_minima) @@ -2203,7 +2197,7 @@ def normalize_off_tick_soc_constraints( next_tick = minimum_time.ceil(resolution) minimum_value = _soc_value_in_mwh(soc_minimum["value"]) _add_soc_bound( - normalized_minima, + projected_minima, _soc_event_at( soc_minimum, previous_tick, @@ -2218,7 +2212,7 @@ def normalize_off_tick_soc_constraints( bound_type="min", ) _add_soc_bound( - normalized_minima, + projected_minima, _soc_event_at( soc_minimum, next_tick, @@ -2248,7 +2242,7 @@ def normalize_off_tick_soc_constraints( next_tick = maximum_time.ceil(resolution) maximum_value = _soc_value_in_mwh(soc_maximum["value"]) _add_soc_bound( - normalized_maxima, + projected_maxima, _soc_event_at( soc_maximum, previous_tick, @@ -2263,7 +2257,7 @@ def normalize_off_tick_soc_constraints( bound_type="max", ) _add_soc_bound( - normalized_maxima, + projected_maxima, _soc_event_at( soc_maximum, next_tick, @@ -2279,9 +2273,9 @@ def normalize_off_tick_soc_constraints( ) return ( - normalized_targets, - _normalized_soc_events_or_original(soc_maxima, normalized_maxima), - _normalized_soc_events_or_original(soc_minima, normalized_minima), + projected_targets, + _projected_soc_events_or_original(soc_maxima, projected_maxima), + _projected_soc_events_or_original(soc_minima, projected_minima), ) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 4a8c190396..f2ec05eedd 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -9,7 +9,7 @@ from flexmeasures.data.models.planning import Scheduler from flexmeasures.data.models.planning.storage import ( StorageScheduler, - normalize_off_tick_soc_constraints, + project_off_tick_soc_constraints, ) from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -437,7 +437,7 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) ) - _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( soc_targets=None, soc_maxima=soc_maxima, soc_minima=soc_minima, @@ -448,12 +448,12 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( soc_max=1, ) - normalized_events = normalized_minima or normalized_maxima + projected_events = projected_minima or projected_maxima - assert _soc_event_value_at(normalized_events, previous_tick) == pytest.approx( + assert _soc_event_value_at(projected_events, previous_tick) == pytest.approx( expected_previous_value ) - assert _soc_event_value_at(normalized_events, next_tick) == pytest.approx( + assert _soc_event_value_at(projected_events, next_tick) == pytest.approx( expected_next_value ) @@ -477,7 +477,7 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): 0, index=pd.date_range(previous_tick, next_tick, freq=resolution) ) - _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( soc_targets=None, soc_maxima=[ { @@ -514,10 +514,10 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): soc_max=1, ) - assert _soc_event_value_at(normalized_minima, previous_tick) == pytest.approx(0.7) - assert _soc_event_value_at(normalized_minima, next_tick) == pytest.approx(0.7) - assert _soc_event_value_at(normalized_maxima, previous_tick) == pytest.approx(0.6) - assert _soc_event_value_at(normalized_maxima, next_tick) == pytest.approx(0.6) + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx(0.7) + assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx(0.7) + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx(0.6) + assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx(0.6) def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): From 6568284f233c6752774d480a723add65c92ee11c Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:23:05 +0100 Subject: [PATCH 47/52] test: explain soc projection assertions Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index f2ec05eedd..82799c3b19 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -333,9 +333,15 @@ def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) storage_constraints = device_constraints[0].tz_convert(tz) - assert pd.isna(storage_constraints.loc[start, "equals"]) - assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) - assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + assert pd.isna( + storage_constraints.loc[start, "equals"] + ), "off-tick targets should not become exact constraints on the previous tick" + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0.992 * 4 + ), "previous tick should allow charging the missing 0.008 MWh before the target time" + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "next tick should carry the projected exact target" def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( @@ -388,8 +394,12 @@ def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) storage_constraints = device_constraints[0].tz_convert(tz) - assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) - assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0.992 * 4 + ), "instantaneous sensors should still project the previous-tick minimum" + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "instantaneous sensors should still project the exact target to the next tick" @pytest.mark.parametrize( @@ -452,10 +462,10 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( assert _soc_event_value_at(projected_events, previous_tick) == pytest.approx( expected_previous_value - ) + ), "previous tick should use the capacity-adjusted projected SoC bound" assert _soc_event_value_at(projected_events, next_tick) == pytest.approx( expected_next_value - ) + ), "next tick should use the projected SoC bound implied by reachability" def _soc_event_value_at(events, dt): @@ -464,7 +474,7 @@ def _soc_event_value_at(events, dt): for event in events if pd.Timestamp(event["start"]) == dt and pd.Timestamp(event["end"]) == dt ] - assert len(matches) == 1 + assert len(matches) == 1, "projection should create exactly one event per tick" return matches[0]["value"] @@ -514,10 +524,18 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): soc_max=1, ) - assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx(0.7) - assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx(0.7) - assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx(0.6) - assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx(0.6) + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 0.7 + ), "merged minima should keep the stricter lower bound on the previous tick" + assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx( + 0.7 + ), "merged minima should keep the stricter lower bound on the next tick" + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx( + 0.6 + ), "merged maxima should keep the stricter upper bound on the previous tick" + assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx( + 0.6 + ), "merged maxima should keep the stricter upper bound on the next tick" def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): @@ -555,9 +573,15 @@ def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_asset scheduler.deserialize_config() - assert scheduler.flex_context["relax_soc_constraints"] is True - assert scheduler.flex_context["soc_minima_breach_price"] is not None - assert scheduler.flex_context["soc_maxima_breach_price"] is not None + assert ( + scheduler.flex_context["relax_soc_constraints"] is True + ), "off-tick SoC constraints should automatically enable SoC relaxation" + assert ( + scheduler.flex_context["soc_minima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a minima breach price" + assert ( + scheduler.flex_context["soc_maxima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a maxima breach price" def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( From 2a3309be89c07f1a7e9222cd5e22aa0193999860 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Fri, 5 Jun 2026 04:24:55 +0100 Subject: [PATCH 48/52] Update flexmeasures/data/models/planning/storage.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index fcf0f24191..bef345322e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1983,7 +1983,7 @@ def _get_soc_constraint_resolution( schedule_resolution: timedelta | None, sensor: Sensor | None, default_resolution: timedelta, -) -> timedelta | None: +) -> timedelta: if schedule_resolution not in (None, timedelta(0)): return schedule_resolution if sensor is not None and sensor.event_resolution != timedelta(0): From 0ab3ccdfe24459532343151281e120da12ebd8aa Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:32:39 +0100 Subject: [PATCH 49/52] docs: explain soc constraint projection policy Signed-off-by: Mohamed Belhsan Hmida --- documentation/concepts/data-model.rst | 20 +++++++++++++++++++ documentation/features/scheduling.rst | 2 ++ flexmeasures/data/models/planning/storage.py | 20 ++++++++++++++++++- .../models/planning/tests/test_storage.py | 5 +++++ .../data/schemas/scheduling/metadata.py | 6 +++--- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 9b2549cc78..cc91497e78 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -100,6 +100,26 @@ Each belief links to a sensor and a data source. Here are two examples: See also :ref:`one_or_multiple_sensors` for guidance on when such beliefs are best recorded on one shared sensor and when separate sensors are preferable. +.. _projecting_scheduling_constraints: + +Projecting scheduling constraints to a fixed resolution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Schedulers operate on a fixed scheduling resolution, such as 15 minutes. +This means the optimization problem can enforce state-of-charge constraints only on the scheduling ticks implied by that resolution. +When a storage flex-model contains point-like ``soc-targets``, ``soc-minima`` or ``soc-maxima`` between two scheduling ticks, FlexMeasures projects those constraints onto the surrounding ticks instead of simply flooring them. + +For an off-tick ``soc-target``, the target value is moved to the next scheduling tick as an exact target. +The previous tick receives lower and upper bounds that reflect how much the asset could still charge or discharge between the previous tick and the original target time. +For off-tick ``soc-minima``, both surrounding ticks receive lower bounds that preserve whether the requested minimum can still be reached. +For off-tick ``soc-maxima``, both surrounding ticks receive upper bounds with the same reachability logic. + +The projection uses the ``consumption-capacity`` and ``production-capacity`` active at the relevant ticks. +If multiple projected lower bounds land on the same tick, the highest lower bound is kept. +If multiple projected upper bounds land on the same tick, the lowest upper bound is kept. +Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted. + + .. _signs_of_power_beliefs: About signs of power & energy values diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 1643330a6d..22e5190b53 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -253,6 +253,8 @@ For more details on the possible formats for field values, see :ref:`variable_qu .. [#minimum_overlap] In case this field defines partially overlapping time periods, the minimum value is selected. See :ref:`variable_quantities`. +.. [#projecting_scheduling_constraints] Off-tick ``soc-targets``, ``soc-minima`` and ``soc-maxima`` are projected to the surrounding scheduling ticks. See :ref:`projecting_scheduling_constraints`. + For more details on the possible formats for field values, see :ref:`variable_quantities`. Usually, not the whole flexibility model is needed. diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index bef345322e..7da3e7a194 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2125,7 +2125,25 @@ def project_off_tick_soc_constraints( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, ]: - """Project off-tick point-like SoC constraints onto the scheduling ticks.""" + """Project off-tick point-like SoC constraints onto scheduling ticks. + + The scheduler can only enforce constraints at its fixed scheduling resolution. + Point-like ``soc-targets``, ``soc-minima`` and ``soc-maxima`` that fall between + two scheduling ticks are therefore replaced by constraints on the previous and + next tick that preserve reachability using the available charge and discharge + capacity between the original event time and those ticks. + + ``soc-targets`` are projected to an exact target on the next tick, plus + capacity-adjusted lower and upper bounds on the previous tick. ``soc-minima`` + become lower bounds on both surrounding ticks, and ``soc-maxima`` become upper + bounds on both surrounding ticks. If multiple projected bounds land on the same + tick, the stricter lower or upper bound is kept. + + Returns ``(soc_targets, soc_maxima, soc_minima)`` with projected list-based + timed events. Non-list specifications such as sensors, series, fixed + quantities, or ``None`` are returned unchanged unless projected bounds need to + be added to a missing list. + """ if not any( isinstance(soc_events, list) and soc_events diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 82799c3b19..d81e1d967f 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -294,6 +294,7 @@ def test_battery_relaxation(add_battery_assets, db): def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets, db): + """Off-tick targets become a next-tick target and a reachable previous-tick bound.""" _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -347,6 +348,7 @@ def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( add_battery_assets, db ): + """Off-tick projection also applies when the scheduled sensor is instantaneous.""" _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -439,6 +441,7 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( expected_previous_value, expected_next_value, ): + """Off-tick minima and maxima are projected as reachable bounds on surrounding ticks.""" tz = pytz.timezone("Europe/Amsterdam") resolution = timedelta(minutes=15) previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) @@ -479,6 +482,7 @@ def _soc_event_value_at(events, dt): def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): + """Projected bounds sharing a tick keep the stricter minimum or maximum.""" tz = pytz.timezone("Europe/Amsterdam") resolution = timedelta(minutes=15) previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) @@ -539,6 +543,7 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): + """Off-tick SoC constraints enable relaxation because projection can add bounds.""" _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 2aaa23b9a1..57ef666dee 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -245,7 +245,7 @@ def to_dict(self): SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. If a ``soc-minima-breach-price`` is defined, the ``soc-minima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example. [#projecting_scheduling_constraints]_""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { @@ -258,7 +258,7 @@ def to_dict(self): SOC_MAXIMA = MetaData( description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. If a ``soc-maxima-breach-price`` is defined, the ``soc-maxima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#minimum_overlap]_""", +Otherwise, they become hard constraints. [#minimum_overlap]_ [#projecting_scheduling_constraints]_""", example=[ { "value": "51 kWh", @@ -270,7 +270,7 @@ def to_dict(self): SOC_TARGETS = MetaData( description=""" Exact set point(s) of the storage's state of charge that the scheduler needs to realize. -These are hard constraints, which means that any infeasible state-of-charge targets would prevent a complete schedule from being computed. +These are hard constraints, which means that any infeasible state-of-charge targets would prevent a complete schedule from being computed. [#projecting_scheduling_constraints]_ """, example=[{"datetime": "2024-02-05T08:00:00+01:00", "value": "3.2 kWh"}], ) From 26c85a6cd53499e95c327b9e3cb6805281621dc6 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 12:42:04 +0100 Subject: [PATCH 50/52] fix: support missing soc bounds in projection Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 44 +++++++++++++------ .../models/planning/tests/test_storage.py | 36 +++++++++++++++ 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 7da3e7a194..4b4cf94442 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2029,6 +2029,20 @@ def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: return float(value) +def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float | None: + if value is None: + return None + return _soc_value_in_mwh(value) + + +def _clamp_soc_min(value: float, soc_min: float | None) -> float: + return value if soc_min is None else max(soc_min, value) + + +def _clamp_soc_max(value: float, soc_max: float | None) -> float: + return value if soc_max is None else min(soc_max, value) + + def _soc_event_at( soc_event: dict[str, datetime | float], dt: pd.Timestamp, @@ -2118,8 +2132,8 @@ def project_off_tick_soc_constraints( consumption_capacity: pd.Series, production_capacity: pd.Series, resolution: timedelta, - soc_min: ur.Quantity | float, - soc_max: ur.Quantity | float, + soc_min: ur.Quantity | float | None, + soc_max: ur.Quantity | float | None, ) -> tuple[ list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, @@ -2154,8 +2168,8 @@ def project_off_tick_soc_constraints( projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - soc_min_value = _soc_value_in_mwh(soc_min) - soc_max_value = _soc_value_in_mwh(soc_max) + soc_min_value = _optional_soc_value_in_mwh(soc_min) + soc_max_value = _optional_soc_value_in_mwh(soc_max) if isinstance(soc_targets, list): projected_targets = [] @@ -2184,7 +2198,7 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_target, previous_tick, - max(soc_min_value, target_value - charge_before_target), + _clamp_soc_min(target_value - charge_before_target, soc_min_value), ), bound_type="min", ) @@ -2193,7 +2207,9 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_target, previous_tick, - min(soc_max_value, target_value + discharge_before_target), + _clamp_soc_max( + target_value + discharge_before_target, soc_max_value + ), ), bound_type="max", ) @@ -2219,12 +2235,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_minimum, previous_tick, - max( - soc_min_value, + _clamp_soc_min( minimum_value - _energy_capacity_between( consumption_capacity, previous_tick, minimum_time, resolution ), + soc_min_value, ), ), bound_type="min", @@ -2234,12 +2250,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_minimum, next_tick, - max( - soc_min_value, + _clamp_soc_min( minimum_value - _energy_capacity_between( production_capacity, minimum_time, next_tick, resolution ), + soc_min_value, ), ), bound_type="min", @@ -2264,12 +2280,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_maximum, previous_tick, - min( - soc_max_value, + _clamp_soc_max( maximum_value + _energy_capacity_between( production_capacity, previous_tick, maximum_time, resolution ), + soc_max_value, ), ), bound_type="max", @@ -2279,12 +2295,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_maximum, next_tick, - min( - soc_max_value, + _clamp_soc_max( maximum_value + _energy_capacity_between( consumption_capacity, maximum_time, next_tick, resolution ), + soc_max_value, ), ), bound_type="max", diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index d81e1d967f..a68a35f6d8 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -471,6 +471,42 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( ), "next tick should use the projected SoC bound implied by reachability" +def test_off_tick_soc_projection_accepts_missing_global_bounds(): + """Missing global SoC bounds leave projected off-tick bounds unclamped.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( + soc_targets=[ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 0.5, + } + ], + soc_maxima=None, + soc_minima=None, + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=None, + soc_max=None, + ) + + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 0.492 + ), "missing global soc-min should not clamp the projected previous-tick minimum" + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx( + 0.508 + ), "missing global soc-max should not clamp the projected previous-tick maximum" + + def _soc_event_value_at(events, dt): matches = [ event From dc2bf9a81d483b0efe60ea7f6b993c2a570ae0fe Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 12:56:50 +0100 Subject: [PATCH 51/52] refactor: move off-tick soc detection to scheduling schema Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 113 +++++------------- .../data/schemas/scheduling/storage.py | 23 ++++ flexmeasures/data/schemas/scheduling/utils.py | 61 ++++++++++ 3 files changed, 111 insertions(+), 86 deletions(-) create mode 100644 flexmeasures/data/schemas/scheduling/utils.py diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4b4cf94442..66fe9ea3b2 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -35,6 +35,10 @@ FlexContextSchema, MultiSensorFlexModelSchema, ) +from flexmeasures.data.schemas.scheduling.utils import ( + is_on_schedule_tick, + should_project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField from flexmeasures.utils.calculations import ( integrate_time_series, @@ -45,7 +49,6 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] -SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") class MetaStorageScheduler(Scheduler): @@ -806,7 +809,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["derivative max"] = consumption_capacity_d if soc_at_start[d] is not None: - if _should_project_off_tick_soc_constraints(sensor_d): + if should_project_off_tick_soc_constraints(sensor_d): ( soc_targets[d], soc_maxima[d], @@ -1073,8 +1076,8 @@ def deserialize_flex_config(self): self.flex_model = {} self.collect_flex_config() - self.enable_relax_soc_constraints_for_off_tick_soc_constraints() - self.flex_context = FlexContextSchema().load(self.flex_context) + if self.flex_context is None: + self.flex_context = {} if isinstance(self.flex_model, dict): if self.sensor.generic_asset.asset_type.name in storage_asset_types: @@ -1086,11 +1089,16 @@ def deserialize_flex_config(self): self.ensure_soc_min_max() # Now it's time to check if our flex configuration holds up to schemas - self.flex_model = StorageFlexModelSchema( + schema = StorageFlexModelSchema( start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model.get("soc-unit"), - ).load(self.flex_model) + schedule_resolution=self.resolution, + default_resolution=self.default_resolution, + ) + self.flex_model = schema.load(self.flex_model) + if schema.has_off_tick_soc_constraints: + self.enable_relax_soc_constraints() # Extend schedule period in case a target exceeds its end self.possibly_extend_end(soc_targets=self.flex_model.get("soc_targets")) @@ -1104,13 +1112,18 @@ def deserialize_flex_config(self): flex_model=sensor_flex_model["sensor_flex_model"], sensor=sensor_flex_model.get("sensor"), ) - self.flex_model[d] = StorageFlexModelSchema( + schema = StorageFlexModelSchema( start=self.start, sensor=sensor_flex_model.get("sensor"), default_soc_unit=sensor_flex_model["sensor_flex_model"].get( "soc-unit" ), - ).load(sensor_flex_model["sensor_flex_model"]) + schedule_resolution=self.resolution, + default_resolution=self.default_resolution, + ) + self.flex_model[d] = schema.load(sensor_flex_model["sensor_flex_model"]) + if schema.has_off_tick_soc_constraints: + self.enable_relax_soc_constraints() self.flex_model[d]["sensor"] = sensor_flex_model.get("sensor") self.flex_model[d]["asset"] = sensor_flex_model.get("asset") @@ -1125,37 +1138,14 @@ def deserialize_flex_config(self): f"Unsupported type of flex-model: '{type(self.flex_model)}'" ) + self.flex_context = FlexContextSchema().load(self.flex_context) + return self.flex_model - def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: + def enable_relax_soc_constraints(self) -> None: """Relax SoC constraints when off-tick SoC events require scheduling-tick projection.""" - if self.flex_context is None: - self.flex_context = {} - if not self.flex_model_has_off_tick_soc_constraints(): - return self.flex_context["relax-soc-constraints"] = True - def flex_model_has_off_tick_soc_constraints(self) -> bool: - if isinstance(self.flex_model, dict): - if not _should_project_off_tick_soc_constraints(self.sensor): - return False - resolution = _get_soc_constraint_resolution( - self.resolution, self.sensor, self.default_resolution - ) - return flex_model_has_off_tick_soc_constraints( - self.flex_model, resolution=resolution - ) - if isinstance(self.flex_model, list): - resolution = self.resolution or self.default_resolution - return any( - flex_model_has_off_tick_soc_constraints( - flex_model.get("sensor-flex-model", flex_model), - resolution=resolution, - ) - for flex_model in self.flex_model - ) - return False - def has_soc_at_start(self) -> bool: return ( "soc-at-start" in self.flex_model @@ -1974,55 +1964,6 @@ def create_constraint_violations_message(constraint_violations: list) -> str: return message -def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: - timestamp = pd.Timestamp(dt) - return timestamp == timestamp.floor(resolution) - - -def _get_soc_constraint_resolution( - schedule_resolution: timedelta | None, - sensor: Sensor | None, - default_resolution: timedelta, -) -> timedelta: - if schedule_resolution not in (None, timedelta(0)): - return schedule_resolution - if sensor is not None and sensor.event_resolution != timedelta(0): - return sensor.event_resolution - return default_resolution - - -def _should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: - return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) - - -def flex_model_has_off_tick_soc_constraints( - flex_model: dict, - resolution: timedelta | None, -) -> bool: - if resolution in (None, timedelta(0)): - return False - - for field_name in SOC_TIMED_EVENT_FIELDS: - field_value = flex_model.get(field_name) - if not isinstance(field_value, list): - continue - for soc_event in field_value: - if not isinstance(soc_event, dict): - continue - for timing_field in ("datetime", "start", "end"): - if soc_event.get(timing_field) is None: - continue - try: - is_on_tick = _is_on_schedule_tick( - soc_event[timing_field], resolution - ) - except (TypeError, ValueError): - continue - if not is_on_tick: - return True - return False - - def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: if isinstance(value, ur.Quantity): return value.to("MWh").magnitude @@ -2174,7 +2115,7 @@ def project_off_tick_soc_constraints( if isinstance(soc_targets, list): projected_targets = [] for soc_target in soc_targets: - if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( + if soc_target["start"] != soc_target["end"] or is_on_schedule_tick( soc_target["end"], resolution ): projected_targets.append(copy.copy(soc_target)) @@ -2221,7 +2162,7 @@ def project_off_tick_soc_constraints( else: soc_minimum_events = [] for soc_minimum in soc_minimum_events: - if soc_minimum["start"] != soc_minimum["end"] or _is_on_schedule_tick( + if soc_minimum["start"] != soc_minimum["end"] or is_on_schedule_tick( soc_minimum["end"], resolution ): continue @@ -2266,7 +2207,7 @@ def project_off_tick_soc_constraints( else: soc_maximum_events = [] for soc_maximum in soc_maximum_events: - if soc_maximum["start"] != soc_maximum["end"] or _is_on_schedule_tick( + if soc_maximum["start"] != soc_maximum["end"] or is_on_schedule_tick( soc_maximum["end"], resolution ): continue diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 8453c2149d..463762f059 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -6,6 +6,7 @@ from flask import current_app from marshmallow import ( Schema, + pre_load, post_load, validate, validates_schema, @@ -18,6 +19,11 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata +from flexmeasures.data.schemas.scheduling.utils import ( + flex_model_has_off_tick_soc_constraints, + get_soc_constraint_resolution, + should_project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.sensors import ( SensorReference, SensorReferenceSchema, @@ -253,12 +259,18 @@ def __init__( sensor: Sensor | None, *args, default_soc_unit: str | None = None, + schedule_resolution: timedelta | None = None, + default_resolution: timedelta = timedelta(minutes=15), **kwargs, ): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None + self.has_off_tick_soc_constraints = False + self.soc_constraint_resolution = get_soc_constraint_resolution( + schedule_resolution, sensor, default_resolution + ) self.flooring_resolution = ( sensor.event_resolution if sensor is not None @@ -306,6 +318,17 @@ def __init__( if field.startswith("soc_"): setattr(self.fields[field], "default_src_unit", default_soc_unit) + @pre_load + def detect_off_tick_soc_constraints(self, data: dict, **kwargs) -> dict: + self.has_off_tick_soc_constraints = ( + isinstance(data, dict) + and should_project_off_tick_soc_constraints(self.sensor) + and flex_model_has_off_tick_soc_constraints( + data, resolution=self.soc_constraint_resolution + ) + ) + return data + @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): # skip check if the flex-model does not define a sensor: the StorageScheduler will not base its resolution on this flex-model diff --git a/flexmeasures/data/schemas/scheduling/utils.py b/flexmeasures/data/schemas/scheduling/utils.py new file mode 100644 index 0000000000..07d15a07ca --- /dev/null +++ b/flexmeasures/data/schemas/scheduling/utils.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from datetime import datetime, timedelta + +import pandas as pd + +from flexmeasures import Sensor + + +SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") + + +def is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: + timestamp = pd.Timestamp(dt) + return timestamp == timestamp.floor(resolution) + + +def get_soc_constraint_resolution( + schedule_resolution: timedelta | None, + sensor: Sensor | None, + default_resolution: timedelta, +) -> timedelta: + if schedule_resolution not in (None, timedelta(0)): + return schedule_resolution + if sensor is not None and sensor.event_resolution != timedelta(0): + return sensor.event_resolution + return default_resolution + + +def should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: + return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) + + +def flex_model_has_off_tick_soc_constraints( + flex_model: dict, + resolution: timedelta | None, +) -> bool: + if resolution in (None, timedelta(0)): + return False + + for field_name in SOC_TIMED_EVENT_FIELDS: + field_value = flex_model.get( + field_name, flex_model.get(field_name.replace("-", "_")) + ) + if not isinstance(field_value, list): + continue + for soc_event in field_value: + if not isinstance(soc_event, dict): + continue + for timing_field in ("datetime", "start", "end"): + if soc_event.get(timing_field) is None: + continue + try: + is_on_tick = is_on_schedule_tick( + soc_event[timing_field], resolution + ) + except (TypeError, ValueError): + continue + if not is_on_tick: + return True + return False From b3b2d2b18604e3cd29801f5f3e5977ad6eb46c8d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 12:59:40 +0100 Subject: [PATCH 52/52] refactor: make soc projection policy explicit Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 259 +++++++++++-------- 1 file changed, 148 insertions(+), 111 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 66fe9ea3b2..b109943a60 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2,8 +2,9 @@ import re import copy +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Type +from typing import Literal, Type import pandas as pd import numpy as np @@ -50,6 +51,36 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] +SocBoundType = Literal["min", "max"] +SocProjectionTick = Literal["previous", "next"] +SocCapacityType = Literal["consumption", "production"] +SocCapacityPeriod = Literal["before", "after"] + + +@dataclass(frozen=True) +class SocProjectionRule: + bound_type: SocBoundType + tick: SocProjectionTick + capacity: SocCapacityType + period: SocCapacityPeriod + sign: int + + +SOC_PROJECTION_POLICIES = { + "soc-targets": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("max", "previous", "production", "before", +1), + ), + "soc-minima": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("min", "next", "production", "after", -1), + ), + "soc-maxima": ( + SocProjectionRule("max", "previous", "production", "before", +1), + SocProjectionRule("max", "next", "consumption", "after", +1), + ), +} + class MetaStorageScheduler(Scheduler): """This class defines the constraints of a schedule for a storage device from the @@ -2027,6 +2058,83 @@ def _energy_capacity_between( return energy +def _tick_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, +) -> pd.Timestamp: + return previous_tick if rule.tick == "previous" else next_tick + + +def _capacity_for_projection_rule( + rule: SocProjectionRule, + consumption_capacity: pd.Series, + production_capacity: pd.Series, +) -> pd.Series: + return ( + consumption_capacity if rule.capacity == "consumption" else production_capacity + ) + + +def _capacity_period_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + event_time: pd.Timestamp, + next_tick: pd.Timestamp, +) -> tuple[pd.Timestamp, pd.Timestamp]: + if rule.period == "before": + return previous_tick, event_time + return event_time, next_tick + + +def _clamp_projected_soc_value( + value: float, + rule: SocProjectionRule, + soc_min: float | None, + soc_max: float | None, +) -> float: + if rule.bound_type == "min": + return _clamp_soc_min(value, soc_min) + return _clamp_soc_max(value, soc_max) + + +def _add_projected_soc_bound( + projected_minima: list[dict[str, datetime | float]], + projected_maxima: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + rule: SocProjectionRule, + event_time: pd.Timestamp, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: float | None, + soc_max: float | None, +) -> None: + capacity_start, capacity_end = _capacity_period_for_projection_rule( + rule, previous_tick, event_time, next_tick + ) + capacity = _capacity_for_projection_rule( + rule, consumption_capacity, production_capacity + ) + projected_value = _soc_value_in_mwh(soc_event["value"]) + rule.sign * ( + _energy_capacity_between(capacity, capacity_start, capacity_end, resolution) + ) + projected_soc_events = ( + projected_minima if rule.bound_type == "min" else projected_maxima + ) + _add_soc_bound( + projected_soc_events, + _soc_event_at( + soc_event, + _tick_for_projection_rule(rule, previous_tick, next_tick), + _clamp_projected_soc_value(projected_value, rule, soc_min, soc_max), + ), + bound_type=rule.bound_type, + ) + + def _add_soc_bound( soc_events: list[dict[str, datetime | float]], soc_event: dict[str, datetime | float], @@ -2126,126 +2234,55 @@ def project_off_tick_soc_constraints( next_tick = target_time.ceil(resolution) target_value = _soc_value_in_mwh(soc_target["value"]) - charge_before_target = _energy_capacity_between( - consumption_capacity, previous_tick, target_time, resolution - ) - discharge_before_target = _energy_capacity_between( - production_capacity, previous_tick, target_time, resolution - ) - projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) - _add_soc_bound( - projected_minima, - _soc_event_at( + for rule in SOC_PROJECTION_POLICIES["soc-targets"]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, soc_target, + rule, + target_time, previous_tick, - _clamp_soc_min(target_value - charge_before_target, soc_min_value), - ), - bound_type="min", - ) - _add_soc_bound( - projected_maxima, - _soc_event_at( - soc_target, - previous_tick, - _clamp_soc_max( - target_value + discharge_before_target, soc_max_value - ), - ), - bound_type="max", - ) + next_tick, + consumption_capacity, + production_capacity, + resolution, + soc_min_value, + soc_max_value, + ) else: projected_targets = soc_targets - if isinstance(soc_minima, list): - soc_minimum_events = copy.deepcopy(soc_minima) - else: - soc_minimum_events = [] - for soc_minimum in soc_minimum_events: - if soc_minimum["start"] != soc_minimum["end"] or is_on_schedule_tick( - soc_minimum["end"], resolution - ): + for field_name, soc_events in ( + ("soc-minima", soc_minima), + ("soc-maxima", soc_maxima), + ): + if not isinstance(soc_events, list): continue + for soc_event in copy.deepcopy(soc_events): + if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( + soc_event["end"], resolution + ): + continue - minimum_time = pd.Timestamp(soc_minimum["end"]) - previous_tick = minimum_time.floor(resolution) - next_tick = minimum_time.ceil(resolution) - minimum_value = _soc_value_in_mwh(soc_minimum["value"]) - _add_soc_bound( - projected_minima, - _soc_event_at( - soc_minimum, - previous_tick, - _clamp_soc_min( - minimum_value - - _energy_capacity_between( - consumption_capacity, previous_tick, minimum_time, resolution - ), - soc_min_value, - ), - ), - bound_type="min", - ) - _add_soc_bound( - projected_minima, - _soc_event_at( - soc_minimum, - next_tick, - _clamp_soc_min( - minimum_value - - _energy_capacity_between( - production_capacity, minimum_time, next_tick, resolution - ), + event_time = pd.Timestamp(soc_event["end"]) + previous_tick = event_time.floor(resolution) + next_tick = event_time.ceil(resolution) + for rule in SOC_PROJECTION_POLICIES[field_name]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, + soc_event, + rule, + event_time, + previous_tick, + next_tick, + consumption_capacity, + production_capacity, + resolution, soc_min_value, - ), - ), - bound_type="min", - ) - - if isinstance(soc_maxima, list): - soc_maximum_events = copy.deepcopy(soc_maxima) - else: - soc_maximum_events = [] - for soc_maximum in soc_maximum_events: - if soc_maximum["start"] != soc_maximum["end"] or is_on_schedule_tick( - soc_maximum["end"], resolution - ): - continue - - maximum_time = pd.Timestamp(soc_maximum["end"]) - previous_tick = maximum_time.floor(resolution) - next_tick = maximum_time.ceil(resolution) - maximum_value = _soc_value_in_mwh(soc_maximum["value"]) - _add_soc_bound( - projected_maxima, - _soc_event_at( - soc_maximum, - previous_tick, - _clamp_soc_max( - maximum_value - + _energy_capacity_between( - production_capacity, previous_tick, maximum_time, resolution - ), soc_max_value, - ), - ), - bound_type="max", - ) - _add_soc_bound( - projected_maxima, - _soc_event_at( - soc_maximum, - next_tick, - _clamp_soc_max( - maximum_value - + _energy_capacity_between( - consumption_capacity, maximum_time, next_tick, resolution - ), - soc_max_value, - ), - ), - bound_type="max", - ) + ) return ( projected_targets,