diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 0f41c0a1d4..ded4c6604f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,8 +12,8 @@ v1.0.0 | July XX, 2026 New features ------------- +* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_ and `PR #2194 `_] * In the UI, asset and sensor lists can be filtered by ID prefix through API-backed search fields [see `PR #2231 `_] -* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] * Sensor references in flex-model and flex-context support various ways of filtering by source [see `PR #2209 `_] * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 6bb300a0c0..6456e3720d 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -100,6 +100,26 @@ Each belief links to a sensor and a data source. Here are two examples: See also :ref:`one_or_multiple_sensors` for guidance on when such beliefs are best recorded on one shared sensor and when separate sensors are preferable. +.. _projecting_scheduling_constraints: + +Projecting scheduling constraints to a fixed resolution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Schedulers operate on a fixed scheduling resolution, such as 15 minutes. +This means the optimization problem can enforce state-of-charge constraints only on the scheduling ticks implied by that resolution. +When a storage flex-model contains point-like ``soc-targets``, ``soc-minima`` or ``soc-maxima`` between two scheduling ticks, FlexMeasures projects those constraints onto the surrounding ticks instead of simply flooring them. + +For an off-tick ``soc-target``, the target value is moved to the next scheduling tick as an exact target. +The previous tick receives lower and upper bounds that reflect how much the asset could still charge or discharge between the previous tick and the original target time. +For off-tick ``soc-minima``, both surrounding ticks receive lower bounds that preserve whether the requested minimum can still be reached. +For off-tick ``soc-maxima``, both surrounding ticks receive upper bounds with the same reachability logic. + +The projection uses the ``consumption-capacity`` and ``production-capacity`` active at the relevant ticks. +If multiple projected lower bounds land on the same tick, the highest lower bound is kept. +If multiple projected upper bounds land on the same tick, the lowest upper bound is kept. +Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted. + + .. _signs_of_power_beliefs: About signs of power & energy values diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 6399d867af..7f711cc270 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -253,6 +253,8 @@ For more details on the possible formats for field values, see :ref:`variable_qu .. [#minimum_overlap] In case this field defines partially overlapping time periods, the minimum value is selected. See :ref:`variable_quantities`. +.. [#projecting_scheduling_constraints] Off-tick ``soc-targets``, ``soc-minima`` and ``soc-maxima`` are projected to the surrounding scheduling ticks. See :ref:`projecting_scheduling_constraints`. + For more details on the possible formats for field values, see :ref:`variable_quantities`. Usually, not the whole flexibility model is needed. diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 18596d64aa..3d3c0b4213 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -246,7 +246,7 @@ def test_upload_sensor_data_with_unit_conversion_success( len(beliefs) == expected_num_beliefs ), f"Fetched {len(beliefs)} beliefs from the database, expecting {expected_num_beliefs}." - assert [b.event_value for b in beliefs] == expected_event_values + assert [b.event_value for b in beliefs] == pytest.approx(expected_event_values) @pytest.mark.parametrize( diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4c59a0ffa9..7ae4824ee7 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2,8 +2,9 @@ import re import copy +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Type +from typing import Literal, Type import pandas as pd import numpy as np @@ -35,6 +36,10 @@ FlexContextSchema, MultiSensorFlexModelSchema, ) +from flexmeasures.data.schemas.scheduling.utils import ( + is_on_schedule_tick, + should_project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField from flexmeasures.utils.calculations import ( integrate_time_series, @@ -46,6 +51,36 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] +SocBoundType = Literal["min", "max"] +SocProjectionTick = Literal["previous", "next"] +SocCapacityType = Literal["consumption", "production"] +SocCapacityPeriod = Literal["before", "after"] + + +@dataclass(frozen=True) +class SocProjectionRule: + bound_type: SocBoundType + tick: SocProjectionTick + capacity: SocCapacityType + period: SocCapacityPeriod + sign: int + + +SOC_PROJECTION_POLICIES = { + "soc-targets": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("max", "previous", "production", "before", +1), + ), + "soc-minima": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("min", "next", "production", "after", -1), + ), + "soc-maxima": ( + SocProjectionRule("max", "previous", "production", "before", +1), + SocProjectionRule("max", "next", "consumption", "after", +1), + ), +} + class MetaStorageScheduler(Scheduler): """This class defines the constraints of a schedule for a storage device from the @@ -661,22 +696,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint soc_maxima[d] = None - if soc_at_start[d] is not None: - device_constraints[d] = add_storage_constraints( - start, - end, - resolution, - soc_at_start[d], - soc_targets[d], - soc_maxima[d], - soc_minima[d], - soc_max[d], - soc_min[d], - ) - else: - # No need to validate non-existing storage constraints - skip_validation = True - power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity( variable_quantity=power_capacity_in_mw[d], unit="MW", @@ -692,6 +711,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if sensor_d is not None and sensor_d.get_attribute( "is_strictly_non_positive" ): + production_capacity_d = pd.Series( + 0, index=power_capacity_in_mw[d].index + ) device_constraints[d]["derivative min"] = 0 else: production_capacity_d = get_continuous_series_sensor_or_quantity( @@ -760,6 +782,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if sensor_d is not None and sensor_d.get_attribute( "is_strictly_non_negative" ): + consumption_capacity_d = pd.Series( + 0, index=power_capacity_in_mw[d].index + ) device_constraints[d]["derivative max"] = 0 else: consumption_capacity_d = get_continuous_series_sensor_or_quantity( @@ -824,6 +849,40 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # consumption-capacity will become a hard constraint device_constraints[d]["derivative max"] = consumption_capacity_d + if soc_at_start[d] is not None: + if should_project_off_tick_soc_constraints(sensor_d): + ( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + ) = project_off_tick_soc_constraints( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + consumption_capacity_d, + production_capacity_d, + resolution, + soc_min[d], + soc_max[d], + ) + + storage_constraints = add_storage_constraints( + start, + end, + resolution, + soc_at_start[d], + soc_targets[d], + soc_maxima[d], + soc_minima[d], + soc_max[d], + soc_min[d], + ) + for column in ("equals", "min", "max"): + device_constraints[d][column] = storage_constraints[column] + else: + # No need to validate non-existing storage constraints + skip_validation = True + all_stock_delta = [] for is_usage, soc_delta in zip([False, True], [soc_gain[d], soc_usage[d]]): @@ -1058,18 +1117,24 @@ def deserialize_flex_config(self): self.flex_model = {} self.collect_flex_config() - self.flex_context = FlexContextSchema().load(self.flex_context) + if self.flex_context is None: + self.flex_context = {} if isinstance(self.flex_model, dict): if self.sensor.generic_asset.asset_type.name in storage_asset_types: self.ensure_soc_at_start() # Now it's time to check if our flex configuration holds up to schemas - self.flex_model = StorageFlexModelSchema( + schema = StorageFlexModelSchema( start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model.get("soc-unit"), - ).load(self.flex_model) + schedule_resolution=self.resolution, + default_resolution=self.default_resolution, + ) + self.flex_model = schema.load(self.flex_model) + if schema.has_off_tick_soc_constraints: + self.enable_relax_soc_constraints() # Extend schedule period in case a target exceeds its end self.possibly_extend_end(soc_targets=self.flex_model.get("soc_targets")) @@ -1082,13 +1147,18 @@ def deserialize_flex_config(self): flex_model=sensor_flex_model["sensor_flex_model"], sensor=sensor_flex_model.get("sensor"), ) - self.flex_model[d] = StorageFlexModelSchema( + schema = StorageFlexModelSchema( start=self.start, sensor=sensor_flex_model.get("sensor"), default_soc_unit=sensor_flex_model["sensor_flex_model"].get( "soc-unit" ), - ).load(sensor_flex_model["sensor_flex_model"]) + schedule_resolution=self.resolution, + default_resolution=self.default_resolution, + ) + self.flex_model[d] = schema.load(sensor_flex_model["sensor_flex_model"]) + if schema.has_off_tick_soc_constraints: + self.enable_relax_soc_constraints() self.flex_model[d]["sensor"] = sensor_flex_model.get("sensor") self.flex_model[d]["asset"] = sensor_flex_model.get("asset") @@ -1103,8 +1173,14 @@ def deserialize_flex_config(self): f"Unsupported type of flex-model: '{type(self.flex_model)}'" ) + self.flex_context = FlexContextSchema().load(self.flex_context) + return self.flex_model + def enable_relax_soc_constraints(self) -> None: + """Relax SoC constraints when off-tick SoC events require scheduling-tick projection.""" + self.flex_context["relax-soc-constraints"] = True + def has_soc_at_start(self) -> bool: return ( "soc-at-start" in self.flex_model @@ -1991,6 +2067,302 @@ def create_constraint_violations_message(constraint_violations: list) -> str: return message +def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: + if isinstance(value, ur.Quantity): + return value.to("MWh").magnitude + return float(value) + + +def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float | None: + if value is None: + return None + return _soc_value_in_mwh(value) + + +def _clamp_soc_min(value: float, soc_min: float | None) -> float: + return value if soc_min is None else max(soc_min, value) + + +def _clamp_soc_max(value: float, soc_max: float | None) -> float: + return value if soc_max is None else min(soc_max, value) + + +def _soc_event_at( + soc_event: dict[str, datetime | float], + dt: pd.Timestamp, + value: float, +) -> dict[str, datetime | float]: + shifted_event = copy.copy(soc_event) + shifted_event["value"] = value + shifted_event["start"] = dt.to_pydatetime() + shifted_event["end"] = dt.to_pydatetime() + if "datetime" in shifted_event: + shifted_event["datetime"] = dt.to_pydatetime() + shifted_event.pop("duration", None) + return shifted_event + + +def _energy_capacity_between( + capacity: pd.Series, + start: pd.Timestamp, + end: pd.Timestamp, + resolution: timedelta, +) -> float: + if end <= start: + return 0 + + if capacity.index.tz is not None: + start = start.tz_convert(capacity.index.tz) + end = end.tz_convert(capacity.index.tz) + + capacity = capacity.astype(float).fillna(0) + tick = start.floor(resolution) + energy = 0.0 + while tick < end: + next_tick = tick + resolution + overlap_start = max(start, tick) + overlap_end = min(end, next_tick) + if overlap_end > overlap_start and tick in capacity.index: + energy += float(capacity.loc[tick]) * ( + (overlap_end - overlap_start) / pd.Timedelta(hours=1) + ) + tick = next_tick + return energy + + +def _tick_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, +) -> pd.Timestamp: + return previous_tick if rule.tick == "previous" else next_tick + + +def _capacity_for_projection_rule( + rule: SocProjectionRule, + consumption_capacity: pd.Series, + production_capacity: pd.Series, +) -> pd.Series: + return ( + consumption_capacity if rule.capacity == "consumption" else production_capacity + ) + + +def _capacity_period_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + event_time: pd.Timestamp, + next_tick: pd.Timestamp, +) -> tuple[pd.Timestamp, pd.Timestamp]: + if rule.period == "before": + return previous_tick, event_time + return event_time, next_tick + + +def _clamp_projected_soc_value( + value: float, + rule: SocProjectionRule, + soc_min: float | None, + soc_max: float | None, +) -> float: + if rule.bound_type == "min": + return _clamp_soc_min(value, soc_min) + return _clamp_soc_max(value, soc_max) + + +def _add_projected_soc_bound( + projected_minima: list[dict[str, datetime | float]], + projected_maxima: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + rule: SocProjectionRule, + event_time: pd.Timestamp, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: float | None, + soc_max: float | None, +) -> None: + capacity_start, capacity_end = _capacity_period_for_projection_rule( + rule, previous_tick, event_time, next_tick + ) + capacity = _capacity_for_projection_rule( + rule, consumption_capacity, production_capacity + ) + projected_value = _soc_value_in_mwh(soc_event["value"]) + rule.sign * ( + _energy_capacity_between(capacity, capacity_start, capacity_end, resolution) + ) + projected_soc_events = ( + projected_minima if rule.bound_type == "min" else projected_maxima + ) + _add_soc_bound( + projected_soc_events, + _soc_event_at( + soc_event, + _tick_for_projection_rule(rule, previous_tick, next_tick), + _clamp_projected_soc_value(projected_value, rule, soc_min, soc_max), + ), + bound_type=rule.bound_type, + ) + + +def _add_soc_bound( + soc_events: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + bound_type: str, +) -> None: + for existing_event in soc_events: + if existing_event.get("start") == soc_event.get("start") and existing_event.get( + "end" + ) == soc_event.get("end"): + existing_value = _soc_value_in_mwh(existing_event["value"]) + soc_value = _soc_value_in_mwh(soc_event["value"]) + existing_event["value"] = ( + max(existing_value, soc_value) + if bound_type == "min" + else min(existing_value, soc_value) + ) + return + soc_events.append(soc_event) + + +def _projected_soc_events_or_original( + original_soc_events: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + projected_soc_events: list[dict[str, datetime | float]], +) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: + if isinstance(original_soc_events, list): + return projected_soc_events + if original_soc_events is None and projected_soc_events: + return projected_soc_events + return original_soc_events + + +def project_off_tick_soc_constraints( + soc_targets: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_maxima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_minima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: ur.Quantity | float | None, + soc_max: ur.Quantity | float | None, +) -> tuple[ + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, +]: + """Project off-tick point-like SoC constraints onto scheduling ticks. + + The scheduler can only enforce constraints at its fixed scheduling resolution. + Point-like ``soc-targets``, ``soc-minima`` and ``soc-maxima`` that fall between + two scheduling ticks are therefore replaced by constraints on the previous and + next tick that preserve reachability using the available charge and discharge + capacity between the original event time and those ticks. + + ``soc-targets`` are projected to an exact target on the next tick, plus + capacity-adjusted lower and upper bounds on the previous tick. ``soc-minima`` + become lower bounds on both surrounding ticks, and ``soc-maxima`` become upper + bounds on both surrounding ticks. If multiple projected bounds land on the same + tick, the stricter lower or upper bound is kept. + + Returns ``(soc_targets, soc_maxima, soc_minima)`` with projected list-based + timed events. Non-list specifications such as sensors, series, fixed + quantities, or ``None`` are returned unchanged unless projected bounds need to + be added to a missing list. + """ + + if not any( + isinstance(soc_events, list) and soc_events + for soc_events in (soc_targets, soc_maxima, soc_minima) + ): + return soc_targets, soc_maxima, soc_minima + + projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] + + soc_min_value = _optional_soc_value_in_mwh(soc_min) + soc_max_value = _optional_soc_value_in_mwh(soc_max) + + if isinstance(soc_targets, list): + projected_targets = [] + for soc_target in soc_targets: + if soc_target["start"] != soc_target["end"] or is_on_schedule_tick( + soc_target["end"], resolution + ): + projected_targets.append(copy.copy(soc_target)) + continue + + target_time = pd.Timestamp(soc_target["end"]) + previous_tick = target_time.floor(resolution) + next_tick = target_time.ceil(resolution) + target_value = _soc_value_in_mwh(soc_target["value"]) + + projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) + for rule in SOC_PROJECTION_POLICIES["soc-targets"]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, + soc_target, + rule, + target_time, + previous_tick, + next_tick, + consumption_capacity, + production_capacity, + resolution, + soc_min_value, + soc_max_value, + ) + else: + projected_targets = soc_targets + + for field_name, soc_events in ( + ("soc-minima", soc_minima), + ("soc-maxima", soc_maxima), + ): + if not isinstance(soc_events, list): + continue + for soc_event in copy.deepcopy(soc_events): + if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( + soc_event["end"], resolution + ): + continue + + event_time = pd.Timestamp(soc_event["end"]) + previous_tick = event_time.floor(resolution) + next_tick = event_time.ceil(resolution) + for rule in SOC_PROJECTION_POLICIES[field_name]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, + soc_event, + rule, + event_time, + previous_tick, + next_tick, + consumption_capacity, + production_capacity, + resolution, + soc_min_value, + soc_max_value, + ) + + return ( + projected_targets, + _projected_soc_events_or_original(soc_maxima, projected_maxima), + _projected_soc_events_or_original(soc_minima, projected_minima), + ) + + def build_device_soc_values( soc_values: ur.Quantity | list[dict[str, datetime | float]] | pd.Series | None, soc_at_start: float, diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index ed8c09be82..a68a35f6d8 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -7,7 +7,10 @@ import pandas as pd from flexmeasures.data.models.planning import Scheduler -from flexmeasures.data.models.planning.storage import StorageScheduler +from flexmeasures.data.models.planning.storage import ( + StorageScheduler, + project_off_tick_soc_constraints, +) from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.planning.tests.utils import ( @@ -290,6 +293,338 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours +def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets, db): + """Off-tick targets become a next-tick target and a reachable previous-tick bound.""" + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + }, + ) + + _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) + storage_constraints = device_constraints[0].tz_convert(tz) + + assert pd.isna( + storage_constraints.loc[start, "equals"] + ), "off-tick targets should not become exact constraints on the previous tick" + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0.992 * 4 + ), "previous tick should allow charging the missing 0.008 MWh before the target time" + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "next tick should carry the projected exact target" + + +def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( + add_battery_assets, db +): + """Off-tick projection also applies when the scheduled sensor is instantaneous.""" + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + instantaneous_power_sensor = Sensor( + name="instantaneous-power", + generic_asset=battery.generic_asset, + event_resolution=timedelta(0), + unit="MW", + ) + db.session.add(instantaneous_power_sensor) + db.session.flush() + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + instantaneous_power_sensor, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + }, + ) + + _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) + storage_constraints = device_constraints[0].tz_convert(tz) + + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0.992 * 4 + ), "instantaneous sensors should still project the previous-tick minimum" + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "instantaneous sensors should still project the exact target to the next tick" + + +@pytest.mark.parametrize( + "soc_minima, soc_maxima, expected_previous_value, expected_next_value", + [ + ( + [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 1, + } + ], + None, + 0.992, + 1, + ), + ( + None, + [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 0.5, + } + ], + 0.5, + 0.502, + ), + ], +) +def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( + soc_minima, + soc_maxima, + expected_previous_value, + expected_next_value, +): + """Off-tick minima and maxima are projected as reachable bounds on surrounding ticks.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=soc_maxima, + soc_minima=soc_minima, + consumption_capacity=capacity, + production_capacity=pd.Series(0, index=capacity.index), + resolution=resolution, + soc_min=0, + soc_max=1, + ) + + projected_events = projected_minima or projected_maxima + + assert _soc_event_value_at(projected_events, previous_tick) == pytest.approx( + expected_previous_value + ), "previous tick should use the capacity-adjusted projected SoC bound" + assert _soc_event_value_at(projected_events, next_tick) == pytest.approx( + expected_next_value + ), "next tick should use the projected SoC bound implied by reachability" + + +def test_off_tick_soc_projection_accepts_missing_global_bounds(): + """Missing global SoC bounds leave projected off-tick bounds unclamped.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( + soc_targets=[ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 0.5, + } + ], + soc_maxima=None, + soc_minima=None, + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=None, + soc_max=None, + ) + + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 0.492 + ), "missing global soc-min should not clamp the projected previous-tick minimum" + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx( + 0.508 + ), "missing global soc-max should not clamp the projected previous-tick maximum" + + +def _soc_event_value_at(events, dt): + matches = [ + event + for event in events + if pd.Timestamp(event["start"]) == dt and pd.Timestamp(event["end"]) == dt + ] + assert len(matches) == 1, "projection should create exactly one event per tick" + return matches[0]["value"] + + +def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): + """Projected bounds sharing a tick keep the stricter minimum or maximum.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 4)), + "start": tz.localize(datetime(2015, 1, 1, 17, 4)), + "end": tz.localize(datetime(2015, 1, 1, 17, 4)), + "value": 0.8, + }, + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 8)), + "start": tz.localize(datetime(2015, 1, 1, 17, 8)), + "end": tz.localize(datetime(2015, 1, 1, 17, 8)), + "value": 0.6, + }, + ], + soc_minima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 4)), + "start": tz.localize(datetime(2015, 1, 1, 17, 4)), + "end": tz.localize(datetime(2015, 1, 1, 17, 4)), + "value": 0.4, + }, + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 8)), + "start": tz.localize(datetime(2015, 1, 1, 17, 8)), + "end": tz.localize(datetime(2015, 1, 1, 17, 8)), + "value": 0.7, + }, + ], + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=0, + soc_max=1, + ) + + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 0.7 + ), "merged minima should keep the stricter lower bound on the previous tick" + assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx( + 0.7 + ), "merged minima should keep the stricter lower bound on the next tick" + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx( + 0.6 + ), "merged maxima should keep the stricter upper bound on the previous tick" + assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx( + 0.6 + ), "merged maxima should keep the stricter upper bound on the next tick" + + +def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): + """Off-tick SoC constraints enable relaxation because projection can add bounds.""" + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "relax-soc-constraints": False, + }, + ) + + scheduler.deserialize_config() + + assert ( + scheduler.flex_context["relax_soc_constraints"] is True + ), "off-tick SoC constraints should automatically enable SoC relaxation" + assert ( + scheduler.flex_context["soc_minima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a minima breach price" + assert ( + scheduler.flex_context["soc_maxima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a maxima breach price" + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 21c7765f18..15a7afa5a2 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -247,7 +247,7 @@ def to_dict(self): SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. If a ``soc-minima-breach-price`` is defined, the ``soc-minima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example. [#projecting_scheduling_constraints]_""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { @@ -260,7 +260,7 @@ def to_dict(self): SOC_MAXIMA = MetaData( description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. If a ``soc-maxima-breach-price`` is defined, the ``soc-maxima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#minimum_overlap]_""", +Otherwise, they become hard constraints. [#minimum_overlap]_ [#projecting_scheduling_constraints]_""", example=[ { "value": "51 kWh", @@ -272,7 +272,7 @@ def to_dict(self): SOC_TARGETS = MetaData( description=""" Exact set point(s) of the storage's state of charge that the scheduler needs to realize. -These are hard constraints, which means that any infeasible state-of-charge targets would prevent a complete schedule from being computed. +These are hard constraints, which means that any infeasible state-of-charge targets would prevent a complete schedule from being computed. [#projecting_scheduling_constraints]_ """, example=[{"datetime": "2024-02-05T08:00:00+01:00", "value": "3.2 kWh"}], ) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 8453c2149d..463762f059 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -6,6 +6,7 @@ from flask import current_app from marshmallow import ( Schema, + pre_load, post_load, validate, validates_schema, @@ -18,6 +19,11 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata +from flexmeasures.data.schemas.scheduling.utils import ( + flex_model_has_off_tick_soc_constraints, + get_soc_constraint_resolution, + should_project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.sensors import ( SensorReference, SensorReferenceSchema, @@ -253,12 +259,18 @@ def __init__( sensor: Sensor | None, *args, default_soc_unit: str | None = None, + schedule_resolution: timedelta | None = None, + default_resolution: timedelta = timedelta(minutes=15), **kwargs, ): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None + self.has_off_tick_soc_constraints = False + self.soc_constraint_resolution = get_soc_constraint_resolution( + schedule_resolution, sensor, default_resolution + ) self.flooring_resolution = ( sensor.event_resolution if sensor is not None @@ -306,6 +318,17 @@ def __init__( if field.startswith("soc_"): setattr(self.fields[field], "default_src_unit", default_soc_unit) + @pre_load + def detect_off_tick_soc_constraints(self, data: dict, **kwargs) -> dict: + self.has_off_tick_soc_constraints = ( + isinstance(data, dict) + and should_project_off_tick_soc_constraints(self.sensor) + and flex_model_has_off_tick_soc_constraints( + data, resolution=self.soc_constraint_resolution + ) + ) + return data + @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): # skip check if the flex-model does not define a sensor: the StorageScheduler will not base its resolution on this flex-model diff --git a/flexmeasures/data/schemas/scheduling/utils.py b/flexmeasures/data/schemas/scheduling/utils.py new file mode 100644 index 0000000000..07d15a07ca --- /dev/null +++ b/flexmeasures/data/schemas/scheduling/utils.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from datetime import datetime, timedelta + +import pandas as pd + +from flexmeasures import Sensor + + +SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") + + +def is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: + timestamp = pd.Timestamp(dt) + return timestamp == timestamp.floor(resolution) + + +def get_soc_constraint_resolution( + schedule_resolution: timedelta | None, + sensor: Sensor | None, + default_resolution: timedelta, +) -> timedelta: + if schedule_resolution not in (None, timedelta(0)): + return schedule_resolution + if sensor is not None and sensor.event_resolution != timedelta(0): + return sensor.event_resolution + return default_resolution + + +def should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: + return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) + + +def flex_model_has_off_tick_soc_constraints( + flex_model: dict, + resolution: timedelta | None, +) -> bool: + if resolution in (None, timedelta(0)): + return False + + for field_name in SOC_TIMED_EVENT_FIELDS: + field_value = flex_model.get( + field_name, flex_model.get(field_name.replace("-", "_")) + ) + if not isinstance(field_value, list): + continue + for soc_event in field_value: + if not isinstance(soc_event, dict): + continue + for timing_field in ("datetime", "start", "end"): + if soc_event.get(timing_field) is None: + continue + try: + is_on_tick = is_on_schedule_tick( + soc_event[timing_field], resolution + ) + except (TypeError, ValueError): + continue + if not is_on_tick: + return True + return False diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 3d4450580e..afcde75fef 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime import pytz import pytest @@ -14,7 +14,6 @@ StorageFlexModelSchema, DBStorageFlexModelSchema, ) -from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.schemas.sensors import TimedEventSchema, VariableQuantityField @@ -163,24 +162,28 @@ def test_process_scheduler_flex_model_process_type(db, app, setup_dummy_sensors) assert process_scheduler_flex_model["process_type"] == ProcessType.SHIFTABLE -def test_storage_flex_model_schema_does_not_floor_instantaneous_sensor( - db, app, dummy_asset +def test_storage_flex_model_schema_preserves_off_tick_soc_datetimes( + db, app, setup_dummy_sensors ): - sensor = Sensor( - "instantaneous power sensor", - generic_asset=dummy_asset, - event_resolution=timedelta(0), - unit="MW", - ) - db.session.add(sensor) - db.session.flush() + sensor1, _, _, _ = setup_dummy_sensors schema = StorageFlexModelSchema( - sensor=sensor, + sensor=sensor1, start=datetime(2023, 1, 1, tzinfo=pytz.UTC), ) - assert schema.flooring_resolution is None + flex_model = schema.load( + { + "soc-at-start": "0 MWh", + "soc-targets": [ + {"datetime": "2023-01-01T00:04:40+00:00", "value": "1 MWh"} + ], + } + ) + + assert flex_model["soc_targets"][0]["datetime"] == pd.Timestamp( + "2023-01-01T00:04:40+00:00" + ) @pytest.mark.parametrize( diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 7d3c06fe4b..190f2c577c 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -20,6 +20,11 @@ def serialize_variable_quantity(value): return VariableQuantityDumpSchema().dump({"value": value})["value"] +def assert_quantity_matches(actual_quantity, expected_quantity) -> None: + assert str(actual_quantity.units) == str(expected_quantity.units) + assert actual_quantity.magnitude == pytest.approx(expected_quantity.magnitude) + + @pytest.mark.parametrize( "src_quantity, dst_unit, fails, exp_dst_quantity", [ @@ -93,11 +98,12 @@ def test_quantity_or_sensor_deserialize( try: dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): - assert dst_quantity == ur.Quantity(exp_dst_quantity) + assert_quantity_matches(dst_quantity, ur.Quantity(exp_dst_quantity)) assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): - assert dst_quantity[0]["value"] == ur.Quantity(exp_dst_quantity) - assert str(dst_quantity[0]["value"]) == exp_dst_quantity + assert_quantity_matches( + dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) + ) assert not fails except ValidationError as e: assert fails, e