From 59008f7babfc98a23eb535848c4abb361ecaa157 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 2 Jun 2026 13:55:51 +0200 Subject: [PATCH 01/29] Add simple data operations --- .../ibex/core/data_manipulation_methods.py | 46 ++++++++++++++++ .../ibex/data_source/imas_python_source.py | 7 +++ .../data_source/imas_python_source_utils.py | 38 +++++++++++++ .../endpoints/schemas/request_data_schemas.py | 37 ++++++++++++- backend/tests/test_data_endpoints.py | 29 ++++++++++ backend/tests/test_data_manipulation.py | 43 +++++++++++++++ docs/source/conf.py | 4 +- .../backend_development/data_manipulation.rst | 53 +++++++++++++++++++ 8 files changed, 254 insertions(+), 3 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index b35e330e..65d42b19 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -170,3 +170,49 @@ class DataManipulationMethodsResponse(BaseModel): data_smoothing_description.method_parameters.append(data_smoothing_method_parameter) available_methods.data_manipulation_methods.append(data_smoothing_description) + +# ====================== SIMPLE DATA OPERATIONS ====================== + +simple_data_operations_description = DataManipulationOperation( + name="Simple Data Operations", + description="Sequence of scalar operations applied to the dataset in this order: addition, multiplication, division, exponentiation, and root.", + method_parameters=[], +) + +data_addition_scalar_parameter = DataManipulationParameter( + human_readable_name="Addition", + name="addition_addend", + description="Scalar value added to every data point. Executed first.", +) + +data_multiplication_scalar_parameter = DataManipulationParameter( + human_readable_name="Multiplication", + name="multiplication_factor", + description="Scalar value used to multiply every data point. Executed after addition.", +) + +data_division_scalar_parameter = DataManipulationParameter( + human_readable_name="Division", + name="division_divisor", + description="Scalar value used as the divisor for every data point. Executed after multiplication.", +) + +data_exponentiation_exponent_parameter = DataManipulationParameter( + human_readable_name="Exponentiation", + name="exponentiation_exponent", + description="Scalar exponent used to raise the input data to a power. Executed after division.", +) + +data_root_degree_parameter = DataManipulationParameter( + human_readable_name="Root", + name="root_degree", + description="Scalar degree used to compute the nth root of the input data. Executed last.", +) + +simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) +simple_data_operations_description.method_parameters.append(data_root_degree_parameter) + +available_methods.data_manipulation_methods.append(simple_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index fa85793e..2696e856 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -49,6 +49,7 @@ calculate_coordinate_shapes, apply_savgol_filter, apply_gaussian_filter, + apply_simple_operations, ) from ibex.core.data_manipulation_methods import SmoothingMethod, InterpolationMethod from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel @@ -785,6 +786,12 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # FE expects data's first dimension to be connected with second dimension, thus this transformation data_to_be_returned = transform_2D_data(data_to_be_returned) + # ============= BEGIN simple operations ============ + + data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) + + # ============= END simple operations ============= + # ============= BEGIN data smoothing ============ if plot_data_query.smoothing_method is not None: diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 795e88a0..12ce1f5f 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -1,4 +1,5 @@ from functools import reduce +from typing import Any import numpy as np from imas.ids_primitive import IDSNumericArray @@ -65,6 +66,43 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): raise InvalidParametersException(msg) +def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): + """ + Apply simple scalar operations to data. + Operations are applied in the following order: + addition, multiplication, division, exponentiation, root. + """ + + if isinstance(data, list): + return [apply_simple_operations(x, plot_data_query) for x in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + result = data + + if plot_data_query.addition_addend is not None: + result = result + plot_data_query.addition_addend + + if plot_data_query.multiplication_factor is not None: + result = result * plot_data_query.multiplication_factor + + if plot_data_query.division_divisor is not None: + if plot_data_query.division_divisor == 0: + raise InvalidParametersException("division_divisor cannot be 0") + result = result / plot_data_query.division_divisor + + if plot_data_query.exponentiation_exponent is not None: + result = np.power(result, plot_data_query.exponentiation_exponent) + + if plot_data_query.root_degree is not None: + if plot_data_query.root_degree == 0: + raise InvalidParametersException("root_degree cannot be 0") + result = np.power(result, 1 / plot_data_query.root_degree) + + return result + else: + msg = "Simple operations can be executed only on numeric arrays, not single values or strings." + raise InvalidParametersException(msg) + + def union_arrays(data: list): return reduce(np.union1d, data) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index f4b93643..43991f8b 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -81,6 +81,29 @@ class GaussianSmoothingParameters(BaseModel): ) +class SimpleOperationsParameters(BaseModel): + addition_addend: float | None = Field( + default=None, + description="Scalar value added to every data point.", + ) + multiplication_factor: float | None = Field( + default=None, + description="Scalar value used to multiply every data point.", + ) + division_divisor: float | None = Field( + default=None, + description="Scalar value used as the divisor for every data point.", + ) + exponentiation_exponent: float | None = Field( + default=None, + description="Scalar exponent used to raise the input data to a power.", + ) + root_degree: float | None = Field( + default=None, + description="Scalar degree used to compute the nth root of the input data.", + ) + + class PlotDataBasicParameters(BaseModel): """...""" @@ -94,7 +117,12 @@ class PlotDataBasicParameters(BaseModel): smoothing_method: SmoothingMethod | None = Field(default=None, description="Smoothing method to be used") -class PlotDataRequestModel(PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters): +class PlotDataRequestModel( + PlotDataBasicParameters, + SavgolSmoothingParameters, + GaussianSmoothingParameters, + SimpleOperationsParameters, +): @model_validator(mode="after") def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": if self.smoothing_method == SmoothingMethod.GAUSSIAN_FILTER and self.gaussian_smoothing_sigma is None: @@ -112,3 +140,10 @@ def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": ) return self + + @model_validator(mode="after") + def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": + if self.division_divisor == 0: + raise ValueError("division_divisor cannot be 0") + + return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 030fd616..463cc66f 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -96,6 +96,35 @@ def test_plot_data_with_savgol_smoothing(entry_path): assert response_body["data"]["value"] == pytest.approx([0.99, 2.0, 3.0, 4.0, 5.0], 0.1) +def test_plot_data_with_simple_operations(entry_path): + cases = [ + ( + {"addition_addend": 2, "multiplication_factor": 3}, + [9.0, 12.0, 15.0, 18.0, 21.0], + ), + ( + {"division_divisor": 2}, + [0.5, 1.0, 1.5, 2.0, 2.5], + ), + ( + {"exponentiation_exponent": 2}, + [1.0, 4.0, 9.0, 16.0, 25.0], + ), + ( + {"root_degree": 2}, + [1.0, 1.41421356237, 1.73205080757, 2.0, 2.2360679775], + ), + ] + + for params, expected in cases: + parameters = {"uri": f"imas:hdf5?path={entry_path}#core_profiles/time", **params} + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx(expected) + + def test_plot_data_smoothing_with_wrong_target_node(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", # targetet quantity must be time-based diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index c987bc9f..0eda2d69 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,9 +1,13 @@ import numpy as np import pytest +from types import SimpleNamespace +from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, + apply_simple_operations, ) +from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel def test_apply_gaussian_smoothing(): @@ -47,3 +51,42 @@ def test_apply_savitzky_golay_smoothing(): ), 0.1, ) + + +@pytest.mark.parametrize( + ("request_kwargs", "data", "expected"), + [ + ({"addition_addend": 2}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), + ({"multiplication_factor": 3}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), + ({"division_divisor": 2}, np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), + ({"exponentiation_exponent": 2}, np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), + ({"root_degree": 2}, np.array([1.0, 4.0, 9.0]), np.array([1.0, 2.0, 3.0])), + ], +) +def test_apply_simple_operations(request_kwargs, data, expected): + request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", **request_kwargs) + + assert np.asarray(expected) == pytest.approx(apply_simple_operations(data, request)) + + +def test_apply_simple_operations_recurses_over_lists(): + request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", addition_addend=1) + data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] + + result = apply_simple_operations(data, request) + + assert np.asarray(result[0]) == pytest.approx([2.0, 3.0]) + assert np.asarray(result[1]) == pytest.approx([4.0, 5.0]) + + +def test_apply_simple_operations_rejects_division_by_zero(): + request = SimpleNamespace( + division_divisor=0, + addition_addend=None, + multiplication_factor=None, + exponentiation_exponent=None, + root_degree=None, + ) + + with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + apply_simple_operations(np.array([1.0, 2.0]), request) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3860d5a5..ed8bbfe2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -294,8 +294,8 @@ # Configuration of sphinx.ext.mathjax # https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.mathjax -autodoc_pydantic_model_show_json = True -autodoc_pydantic_model_show_config_summary = True +autodoc_pydantic_model_show_json = False +autodoc_pydantic_model_show_config_summary = False def escape_underscores(string): diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index dc08be09..08385bad 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -9,6 +9,14 @@ Introduction The IBEX backend provides a range of data manipulation techniques that directly affect the shape and appearance of the resulting plots. These operations are applied as part of the ``/data/plot_data/`` request flow and allow the backend to transform datasets before they are returned to the frontend. +The backend applies the manipulation stages in this order: + +1. simple data operations +2. data smoothing +3. data interpolation +4. downsampling + +This means later stages operate on the output of earlier ones when the corresponding request parameters are enabled. Data smoothing --------------- @@ -85,3 +93,48 @@ Savitzky-Golay smoothing: curl -X 'GET' \ '/data/plot_data?uri=&smoothing_method=savitzky_golay_filter&savgol_smoothing_window_length=5&savgol_smoothing_polyorder=2' \ -H 'accept: application/json' + + +Simple scalar operations +------------------------ + +IBEX also supports a sequence of scalar operations that can be applied to the returned dataset: + +* addition +* multiplication +* division +* exponentiation +* root + +These operations are executed in that order. In practice, the backend applies them sequentially to the numerical data before any smoothing or resampling step. + +The corresponding request parameters are: + +* ``addition_addend`` +* ``multiplication_factor`` +* ``division_divisor`` +* ``exponentiation_exponent`` +* ``root_degree`` + +Division by zero is rejected by the backend. + +Example usage +~~~~~~~~~~~~~~ + +The following examples demonstrate how simple data operations can be enabled for testing purposes. + +Single operation: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&addition_addend=2' \ + -H 'accept: application/json' + +Two operations: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&addition_addend=2&multiplication_factor=3' \ + -H 'accept: application/json' From c86a6f23fc3e51cef4722249f7593ab5f38c6e15 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 10:23:09 +0200 Subject: [PATCH 02/29] Add subtraction. Check parameters before operation. --- backend/ibex/core/data_manipulation_methods.py | 11 +++++++++-- backend/ibex/data_source/imas_python_source.py | 12 +++++++++++- backend/ibex/data_source/imas_python_source_utils.py | 5 ++++- .../ibex/endpoints/schemas/request_data_schemas.py | 4 ++++ backend/tests/test_data_manipulation.py | 2 ++ .../backend_development/data_manipulation.rst | 2 ++ 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 65d42b19..fc633eb4 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -175,7 +175,7 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description = DataManipulationOperation( name="Simple Data Operations", - description="Sequence of scalar operations applied to the dataset in this order: addition, multiplication, division, exponentiation, and root.", + description="Sequence of scalar operations applied to the dataset in this order: addition, subtraction, multiplication, division, exponentiation, and root.", method_parameters=[], ) @@ -185,10 +185,16 @@ class DataManipulationMethodsResponse(BaseModel): description="Scalar value added to every data point. Executed first.", ) +data_subtraction_subtrahend_parameter = DataManipulationParameter( + human_readable_name="Subtraction", + name="subtraction_subtrahend", + description="Scalar value subtracted from every data point. Executed after addition.", +) + data_multiplication_scalar_parameter = DataManipulationParameter( human_readable_name="Multiplication", name="multiplication_factor", - description="Scalar value used to multiply every data point. Executed after addition.", + description="Scalar value used to multiply every data point. Executed after subtraction.", ) data_division_scalar_parameter = DataManipulationParameter( @@ -210,6 +216,7 @@ class DataManipulationMethodsResponse(BaseModel): ) simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_subtraction_subtrahend_parameter) simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 2696e856..1f185c13 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -788,7 +788,17 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= BEGIN simple operations ============ - data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) + if any( + [ + plot_data_query.addition_addend is not None, + plot_data_query.subtraction_subtrahend is not None, + plot_data_query.multiplication_factor is not None, + plot_data_query.division_divisor is not None, + plot_data_query.exponentiation_exponent is not None, + plot_data_query.root_degree is not None, + ] + ): + data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) # ============= END simple operations ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 12ce1f5f..800bb2c4 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -70,7 +70,7 @@ def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): """ Apply simple scalar operations to data. Operations are applied in the following order: - addition, multiplication, division, exponentiation, root. + addition, subtraction, multiplication, division, exponentiation, root. """ if isinstance(data, list): @@ -81,6 +81,9 @@ def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): if plot_data_query.addition_addend is not None: result = result + plot_data_query.addition_addend + if plot_data_query.subtraction_subtrahend is not None: + result = result - plot_data_query.subtraction_subtrahend + if plot_data_query.multiplication_factor is not None: result = result * plot_data_query.multiplication_factor diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 43991f8b..23c6386f 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -86,6 +86,10 @@ class SimpleOperationsParameters(BaseModel): default=None, description="Scalar value added to every data point.", ) + subtraction_subtrahend: float | None = Field( + default=None, + description="Scalar value subtracted from every data point.", + ) multiplication_factor: float | None = Field( default=None, description="Scalar value used to multiply every data point.", diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 0eda2d69..0dcec80b 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -57,6 +57,7 @@ def test_apply_savitzky_golay_smoothing(): ("request_kwargs", "data", "expected"), [ ({"addition_addend": 2}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), + ({"subtraction_subtrahend": 1}, np.array([3.0, 4.0, 5.0]), np.array([2.0, 3.0, 4.0])), ({"multiplication_factor": 3}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), ({"division_divisor": 2}, np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), ({"exponentiation_exponent": 2}, np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), @@ -83,6 +84,7 @@ def test_apply_simple_operations_rejects_division_by_zero(): request = SimpleNamespace( division_divisor=0, addition_addend=None, + subtraction_subtrahend=None, multiplication_factor=None, exponentiation_exponent=None, root_degree=None, diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index 08385bad..e0e02599 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -101,6 +101,7 @@ Simple scalar operations IBEX also supports a sequence of scalar operations that can be applied to the returned dataset: * addition +* subtraction * multiplication * division * exponentiation @@ -111,6 +112,7 @@ These operations are executed in that order. In practice, the backend applies th The corresponding request parameters are: * ``addition_addend`` +* ``subtraction_subtrahend`` * ``multiplication_factor`` * ``division_divisor`` * ``exponentiation_exponent`` From e67f0f1a9581f0a1fe88cc4fbc59e783aa44b370 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 12:34:08 +0200 Subject: [PATCH 03/29] Add operation priorities --- .../ibex/core/data_manipulation_methods.py | 82 +++++++++++++++++-- .../data_source/imas_python_source_utils.py | 77 ++++++++++++----- .../endpoints/schemas/request_data_schemas.py | 36 ++++++++ backend/tests/test_data_endpoints.py | 5 ++ backend/tests/test_data_manipulation.py | 25 ++++-- 5 files changed, 187 insertions(+), 38 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index fc633eb4..730befc5 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -43,6 +43,8 @@ class DataManipulationParameter(BaseModel): human_readable_name: str name: str description: str + type: str + default: Optional[str] = None possible_values: Optional[list[PossibleValue]] = None @@ -76,12 +78,15 @@ class DataManipulationMethodsResponse(BaseModel): human_readable_name="Interpolate over", name="interpolate_over", description="List of URIs to gather coordinates from, for interpolation", + type="list[string]", ) data_interpolation_method_parameter = DataManipulationParameter( human_readable_name="Interpolation method", name="interpolation_method", description="List of URIs to gather coordinates from, for interpolation", + type="string", + default=InterpolationMethod.EXACT_VALUE, possible_values=[ PossibleValue( value=InterpolationMethod.EXACT_VALUE, @@ -116,6 +121,7 @@ class DataManipulationMethodsResponse(BaseModel): human_readable_name="Smoothing method", name="smoothing_method", description="Method to be used in data smoothing process", + type="string", possible_values=[ PossibleValue( value=SmoothingMethod.GAUSSIAN_FILTER, @@ -175,51 +181,113 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description = DataManipulationOperation( name="Simple Data Operations", - description="Sequence of scalar operations applied to the dataset in this order: addition, subtraction, multiplication, division, exponentiation, and root.", + description="Sequence of scalar operations applied to the dataset. " + "Execution order is determined by the *_priority parameters. " + "Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6.", method_parameters=[], ) data_addition_scalar_parameter = DataManipulationParameter( human_readable_name="Addition", name="addition_addend", - description="Scalar value added to every data point. Executed first.", + description="Scalar value added to every data point.", + type="number", ) data_subtraction_subtrahend_parameter = DataManipulationParameter( human_readable_name="Subtraction", name="subtraction_subtrahend", - description="Scalar value subtracted from every data point. Executed after addition.", + description="Scalar value subtracted from every data point.", + type="number", ) data_multiplication_scalar_parameter = DataManipulationParameter( human_readable_name="Multiplication", name="multiplication_factor", - description="Scalar value used to multiply every data point. Executed after subtraction.", + description="Scalar value used to multiply every data point.", + type="number", ) data_division_scalar_parameter = DataManipulationParameter( human_readable_name="Division", name="division_divisor", - description="Scalar value used as the divisor for every data point. Executed after multiplication.", + description="Scalar value used as the divisor for every data point.", + type="number", ) data_exponentiation_exponent_parameter = DataManipulationParameter( human_readable_name="Exponentiation", name="exponentiation_exponent", - description="Scalar exponent used to raise the input data to a power. Executed after division.", + description="Scalar exponent used to raise the input data to a power.", + type="number", ) data_root_degree_parameter = DataManipulationParameter( human_readable_name="Root", name="root_degree", - description="Scalar degree used to compute the nth root of the input data. Executed last.", + description="Scalar degree used to compute the nth root of the input data.", + type="number", +) + +data_addition_priority_parameter = DataManipulationParameter( + human_readable_name="Addition priority", + name="addition_priority", + description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", + type="int", + default="1", +) + +data_subtraction_priority_parameter = DataManipulationParameter( + human_readable_name="Subtraction priority", + name="subtraction_priority", + description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", + type="int", + default="2", +) + +data_multiplication_priority_parameter = DataManipulationParameter( + human_readable_name="Multiplication priority", + name="multiplication_priority", + description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", + type="int", + default="3", +) + +data_division_priority_parameter = DataManipulationParameter( + human_readable_name="Division priority", + name="division_priority", + description="Execution order priority for division. Lower value = earlier execution. Default: 4.", + type="int", + default="4", +) + +data_exponentiation_priority_parameter = DataManipulationParameter( + human_readable_name="Exponentiation priority", + name="exponentiation_priority", + description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", + type="int", + default="5", +) + +data_root_priority_parameter = DataManipulationParameter( + human_readable_name="Root priority", + name="root_priority", + description="Execution order priority for root. Lower value = earlier execution. Default: 6.", + type="int", + default="6", ) simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_addition_priority_parameter) simple_data_operations_description.method_parameters.append(data_subtraction_subtrahend_parameter) +simple_data_operations_description.method_parameters.append(data_subtraction_priority_parameter) simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_multiplication_priority_parameter) simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_division_priority_parameter) simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) +simple_data_operations_description.method_parameters.append(data_exponentiation_priority_parameter) simple_data_operations_description.method_parameters.append(data_root_degree_parameter) +simple_data_operations_description.method_parameters.append(data_root_priority_parameter) available_methods.data_manipulation_methods.append(simple_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 800bb2c4..3be95215 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -66,39 +66,72 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): raise InvalidParametersException(msg) +def _safe_division(data, divisor): + if divisor == 0: + raise InvalidParametersException("division_divisor cannot be 0") + return data / divisor + + def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): """ Apply simple scalar operations to data. - Operations are applied in the following order: - addition, subtraction, multiplication, division, exponentiation, root. + Execution order is determined by the *_priority parameters from the request. + Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6. """ + _SIMPLE_OPERATION_DEFS = [ + { + "value_field": "addition_addend", + "priority_field": "addition_priority", + "default_priority": 1, + "func": lambda r, v: r + v, + }, + { + "value_field": "subtraction_subtrahend", + "priority_field": "subtraction_priority", + "default_priority": 2, + "func": lambda r, v: r - v, + }, + { + "value_field": "multiplication_factor", + "priority_field": "multiplication_priority", + "default_priority": 3, + "func": lambda r, v: r * v, + }, + { + "value_field": "division_divisor", + "priority_field": "division_priority", + "default_priority": 4, + "func": lambda r, v: _safe_division(r, v), + }, + { + "value_field": "exponentiation_exponent", + "priority_field": "exponentiation_priority", + "default_priority": 5, + "func": lambda r, v: np.power(r, v), + }, + { + "value_field": "root_degree", + "priority_field": "root_priority", + "default_priority": 6, + "func": lambda r, v: np.power(r, 1 / v), + }, + ] + if isinstance(data, list): return [apply_simple_operations(x, plot_data_query) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): result = data - if plot_data_query.addition_addend is not None: - result = result + plot_data_query.addition_addend - - if plot_data_query.subtraction_subtrahend is not None: - result = result - plot_data_query.subtraction_subtrahend - - if plot_data_query.multiplication_factor is not None: - result = result * plot_data_query.multiplication_factor - - if plot_data_query.division_divisor is not None: - if plot_data_query.division_divisor == 0: - raise InvalidParametersException("division_divisor cannot be 0") - result = result / plot_data_query.division_divisor - - if plot_data_query.exponentiation_exponent is not None: - result = np.power(result, plot_data_query.exponentiation_exponent) + operations = [] + for op in _SIMPLE_OPERATION_DEFS: + value = getattr(plot_data_query, op["value_field"]) + if value is not None: + priority = getattr(plot_data_query, op["priority_field"]) or op["default_priority"] + operations.append({"priority": priority, "func": op["func"], "value": value}) - if plot_data_query.root_degree is not None: - if plot_data_query.root_degree == 0: - raise InvalidParametersException("root_degree cannot be 0") - result = np.power(result, 1 / plot_data_query.root_degree) + for op in sorted(operations, key=lambda x: x["priority"]): + result = op["func"](result, op["value"]) return result else: diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 23c6386f..84ee0e63 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -86,26 +86,50 @@ class SimpleOperationsParameters(BaseModel): default=None, description="Scalar value added to every data point.", ) + addition_priority: int | None = Field( + default=None, + description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", + ) subtraction_subtrahend: float | None = Field( default=None, description="Scalar value subtracted from every data point.", ) + subtraction_priority: int | None = Field( + default=None, + description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", + ) multiplication_factor: float | None = Field( default=None, description="Scalar value used to multiply every data point.", ) + multiplication_priority: int | None = Field( + default=None, + description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", + ) division_divisor: float | None = Field( default=None, description="Scalar value used as the divisor for every data point.", ) + division_priority: int | None = Field( + default=None, + description="Execution order priority for division. Lower value = earlier execution. Default: 4.", + ) exponentiation_exponent: float | None = Field( default=None, description="Scalar exponent used to raise the input data to a power.", ) + exponentiation_priority: int | None = Field( + default=None, + description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", + ) root_degree: float | None = Field( default=None, description="Scalar degree used to compute the nth root of the input data.", ) + root_priority: int | None = Field( + default=None, + description="Execution order priority for root. Lower value = earlier execution. Default: 6.", + ) class PlotDataBasicParameters(BaseModel): @@ -150,4 +174,16 @@ def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": if self.division_divisor == 0: raise ValueError("division_divisor cannot be 0") + priorities = [ + self.addition_priority, + self.subtraction_priority, + self.multiplication_priority, + self.division_priority, + self.exponentiation_priority, + self.root_priority, + ] + defined_priorities = [p for p in priorities if p is not None] + if len(defined_priorities) != len(set(defined_priorities)): + raise ValueError("operation priorities must be unique") + return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 463cc66f..e390f92a 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -97,11 +97,16 @@ def test_plot_data_with_savgol_smoothing(entry_path): def test_plot_data_with_simple_operations(entry_path): + # core_profiles.time = [1,2,3,4,5] (float) cases = [ ( {"addition_addend": 2, "multiplication_factor": 3}, [9.0, 12.0, 15.0, 18.0, 21.0], ), + ( + {"addition_addend": 2, "multiplication_factor": 3, "addition_priority": 6, "multiplication_priority": 1}, + [5.0, 8.0, 11.0, 14.0, 17.0], + ), ( {"division_divisor": 2}, [0.5, 1.0, 1.5, 2.0, 2.5], diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 0dcec80b..dbe94c18 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from types import SimpleNamespace from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, @@ -81,14 +80,22 @@ def test_apply_simple_operations_recurses_over_lists(): def test_apply_simple_operations_rejects_division_by_zero(): - request = SimpleNamespace( - division_divisor=0, - addition_addend=None, - subtraction_subtrahend=None, - multiplication_factor=None, - exponentiation_exponent=None, - root_degree=None, - ) + request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): apply_simple_operations(np.array([1.0, 2.0]), request) + + +def test_apply_simple_operations_uses_priority_order(): + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + addition_addend=1, + multiplication_factor=2, + addition_priority=2, + multiplication_priority=1, + ) + data = np.array([5.0]) + # default order: add then multiply -> (5+1)*2 = 12 + # priority order: multiply then add -> (5*2)+1 = 11 + result = apply_simple_operations(data, request) + assert result == pytest.approx([11.0]) From fc9c0b33095c7638f48966b2b1600cf70eb24df4 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 12:49:38 +0200 Subject: [PATCH 04/29] Fix broken test --- backend/tests/test_data_manipulation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index dbe94c18..163dc649 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -7,6 +7,7 @@ apply_simple_operations, ) from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel +from pydantic_core._pydantic_core import ValidationError def test_apply_gaussian_smoothing(): @@ -80,10 +81,8 @@ def test_apply_simple_operations_recurses_over_lists(): def test_apply_simple_operations_rejects_division_by_zero(): - request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) - - with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): - apply_simple_operations(np.array([1.0, 2.0]), request) + with pytest.raises(ValidationError, match="division_divisor cannot be 0"): + PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) def test_apply_simple_operations_uses_priority_order(): From 1c7e665ac9828491049e5b4dd811f467e2f477bf Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 12:59:02 +0200 Subject: [PATCH 05/29] Apply linter --- backend/tests/test_data_manipulation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 163dc649..5ee91c1c 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, From 85c3a2b989145e09fc53b0e3b62515f21e18691f Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 16 Jun 2026 12:01:00 +0200 Subject: [PATCH 06/29] Signal operations (WIP) --- .../ibex/data_source/imas_python_source.py | 55 +++++++++++++++++- .../data_source/imas_python_source_utils.py | 40 ++++++++++++- .../endpoints/schemas/request_data_schemas.py | 58 +++++++++++++++++++ backend/tests/test_data_manipulation.py | 53 +++++++++++++++++ 4 files changed, 203 insertions(+), 3 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 1f185c13..60e1004f 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -50,6 +50,7 @@ apply_savgol_filter, apply_gaussian_filter, apply_simple_operations, + apply_signal_operations, ) from ibex.core.data_manipulation_methods import SmoothingMethod, InterpolationMethod from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel @@ -831,7 +832,6 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= END data smoothing ============= # ============= BEGIN resample data onto new time vector ============= - def convert_to_lists(data): if isinstance(data, list): return [convert_to_lists(d) for d in data] @@ -840,6 +840,10 @@ def convert_to_lists(data): else: return data + # list of dicts used when combining signals after + # {uri:str, data:list[*], interpolated_data: list[*]} + others_signals_data = {} + if plot_data_query.interpolate_over: # =================== GATHER ALL COORDINATES =================== original_coord_values = [] @@ -849,6 +853,17 @@ def convert_to_lists(data): original_coord_values.append(sorted(set(flatten(c["value"])))) original_coord_values.reverse() + store_other_signals_data = False # used for signal combining after interpolation + if any( + [ + plot_data_query.signal_addition_addend_uri, + plot_data_query.signal_subtraction_subtrahend_uri, + plot_data_query.signal_multiplication_factor_uri, + plot_data_query.signal_division_divisor_uri, + ] + ): + store_other_signals_data = True + for _uri in plot_data_query.interpolate_over: _uri_obj = IMAS_URI(_uri) @@ -861,7 +876,17 @@ def convert_to_lists(data): new_plot_data_query.uri = _uri new_plot_data_query.interpolate_over = None new_plot_data_query.smoothing_method = None - interpolate_to_coordinates = self.get_plot_data(new_plot_data_query)["data"]["coordinates"] + # interpolate_to will be used later with signal combining + interpolate_to = self.get_plot_data(new_plot_data_query)["data"] + interpolate_to_coordinates = interpolate_to["coordinates"] + if store_other_signals_data: + others_signals_data[_uri] = { + "uri": _uri, + "data": interpolate_to["value"], + "coordinates": [ + sorted(set(flatten(convert_to_lists(c["value"])))) for c in interpolate_to_coordinates + ], + } if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): message = "Interpolation error. Source and target nodes have different number of coordinates." @@ -911,6 +936,32 @@ def convert_to_lists(data): # ============= END resample data onto new time vector ============= + # ============= BEGIN signal operations ============= + + _SIGNAL_URI_FIELDS = [ + "signal_addition_addend_uri", + "signal_subtraction_subtrahend_uri", + "signal_multiplication_factor_uri", + "signal_division_divisor_uri", + ] + + for field in _SIGNAL_URI_FIELDS: + signal_uris = getattr(plot_data_query, field) + if signal_uris: + for signal_uri in signal_uris: + if "interpolated_data" not in others_signals_data[signal_uri]: + signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) + signal_data = resample_data_without_interpolation( + tuple(others_signals_data[signal_uri]["coordinates"]), + signal_data, + tuple(common_coords_values), + ) + others_signals_data[uri]["interpolated_data"] = signal_data + + data_to_be_returned = apply_signal_operations(data_to_be_returned, plot_data_query, others_signals_data) + + # ============= END signal operations ============= + try: original_data_shape = np.asarray(data_to_be_returned).shape except ValueError: diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 3be95215..bb70a63e 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -67,7 +67,10 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): def _safe_division(data, divisor): - if divisor == 0: + if isinstance(divisor, np.ndarray): + if np.any(divisor == 0): + raise InvalidParametersException("division_divisor cannot be 0") + elif divisor == 0: raise InvalidParametersException("division_divisor cannot be 0") return data / divisor @@ -139,6 +142,41 @@ def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): raise InvalidParametersException(msg) +def apply_signal_operations(data: list | np.ndarray, plot_data_query: Any, signal_data_by_uri: dict): + """ + Apply signal operations to data. + + :param data: The input data array. + :param plot_data_query: Request model with signal URI fields. + :param signal_data_by_uri: Dict mapping signal URIs to their interpolated data arrays. + """ + # signal_data_by_uri: { + # uri : + # data : + # interpolated_data : + # } + + _SIGNAL_OPERATION_DEFS = [ + ("signal_addition_addend_uri", "signal_addition_priority", 1, lambda r, v: r + v), + ("signal_subtraction_subtrahend_uri", "signal_subtraction_priority", 2, lambda r, v: r - v), + ("signal_multiplication_factor_uri", "signal_multiplication_priority", 3, lambda r, v: r * v), + ("signal_division_divisor_uri", "signal_division_priority", 4, lambda r, v: _safe_division(r, v)), + ] + + operations = [] + for uri_field, priority_field, default_priority, func in _SIGNAL_OPERATION_DEFS: + uris = getattr(plot_data_query, uri_field) + if uris: + for uri in uris: + priority = getattr(plot_data_query, priority_field) or default_priority + signal_data = signal_data_by_uri[uri] + operations.append({"priority": priority, "func": func, "value": signal_data}) + + for op in sorted(operations, key=lambda x: x["priority"]): + data = op["func"](data, op["value"]) + return data + + def union_arrays(data: list): return reduce(np.union1d, data) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 84ee0e63..6072aa8a 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -132,6 +132,41 @@ class SimpleOperationsParameters(BaseModel): ) +class SignalOperationsParameters(BaseModel): + signal_addition_addend_uri: list[str] | None = Field( + default=None, + description="URIs of signals to be added to the data.", + ) + signal_addition_priority: int | None = Field( + default=None, + description="Execution order priority for signal addition. Lower value = earlier execution. Default: 1.", + ) + signal_subtraction_subtrahend_uri: list[str] | None = Field( + default=None, + description="URIs of signals to be subtracted from the data.", + ) + signal_subtraction_priority: int | None = Field( + default=None, + description="Execution order priority for signal subtraction. Lower value = earlier execution. Default: 2.", + ) + signal_multiplication_factor_uri: list[str] | None = Field( + default=None, + description="URIs of signals used to multiply the data.", + ) + signal_multiplication_priority: int | None = Field( + default=None, + description="Execution order priority for signal multiplication. Lower value = earlier execution. Default: 3.", + ) + signal_division_divisor_uri: list[str] | None = Field( + default=None, + description="URIs of signals used as the divisor for the data.", + ) + signal_division_priority: int | None = Field( + default=None, + description="Execution order priority for signal division. Lower value = earlier execution. Default: 4.", + ) + + class PlotDataBasicParameters(BaseModel): """...""" @@ -150,6 +185,7 @@ class PlotDataRequestModel( SavgolSmoothingParameters, GaussianSmoothingParameters, SimpleOperationsParameters, + SignalOperationsParameters, ): @model_validator(mode="after") def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": @@ -174,6 +210,28 @@ def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": if self.division_divisor == 0: raise ValueError("division_divisor cannot be 0") + signal_uri_lists = [ + self.signal_addition_addend_uri, + self.signal_subtraction_subtrahend_uri, + self.signal_multiplication_factor_uri, + self.signal_division_divisor_uri, + ] + for uri_list in signal_uri_lists: + if uri_list: + for signal_uri in uri_list: + if not self.interpolate_over or signal_uri not in self.interpolate_over: + raise ValueError(f"Signal URI '{signal_uri}' must be listed in interpolate_over") + + signal_priorities = [ + self.signal_addition_priority, + self.signal_subtraction_priority, + self.signal_multiplication_priority, + self.signal_division_priority, + ] + defined_signal_priorities = [p for p in signal_priorities if p is not None] + if len(defined_signal_priorities) != len(set(defined_signal_priorities)): + raise ValueError("signal operation priorities must be unique") + priorities = [ self.addition_priority, self.subtraction_priority, diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 5ee91c1c..6dfc70ee 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -3,12 +3,65 @@ from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, + apply_signal_operations, apply_simple_operations, ) from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel from pydantic_core._pydantic_core import ValidationError +def test_apply_signal_operations_addition(): + addend_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([1.0, 2.0, 3.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[addend_uri], + signal_addition_addend_uri=[addend_uri], + ) + signal_data_by_uri = {addend_uri: np.array([10.0, 20.0, 30.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [11.0, 22.0, 33.0]) + + +def test_apply_signal_operations_subtraction(): + subtrahend_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([10.0, 20.0, 30.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[subtrahend_uri], + signal_subtraction_subtrahend_uri=[subtrahend_uri], + ) + signal_data_by_uri = {subtrahend_uri: np.array([1.0, 2.0, 3.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [9.0, 18.0, 27.0]) + + +def test_apply_signal_operations_multiplication(): + factor_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([1.0, 2.0, 3.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[factor_uri], + signal_multiplication_factor_uri=[factor_uri], + ) + signal_data_by_uri = {factor_uri: np.array([2.0, 3.0, 4.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [2.0, 6.0, 12.0]) + + +def test_apply_signal_operations_division(): + divisor_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([10.0, 20.0, 30.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[divisor_uri], + signal_division_divisor_uri=[divisor_uri], + ) + signal_data_by_uri = {divisor_uri: np.array([2.0, 5.0, 6.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [5.0, 4.0, 5.0]) + + def test_apply_gaussian_smoothing(): data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1]) From a105b666d5f0cabad3924eaffcc52bb9e777d9d9 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 16 Jun 2026 13:51:30 +0200 Subject: [PATCH 07/29] Basic operations order (WIP) --- .../ibex/core/data_manipulation_methods.py | 138 +++++------------- .../ibex/data_source/imas_python_source.py | 13 +- .../data_source/imas_python_source_utils.py | 78 +++------- .../endpoints/schemas/request_data_schemas.py | 75 +--------- backend/tests/test_data_endpoints.py | 10 +- backend/tests/test_data_manipulation.py | 49 +++---- 6 files changed, 85 insertions(+), 278 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 730befc5..d768259f 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -46,6 +46,8 @@ class DataManipulationParameter(BaseModel): type: str default: Optional[str] = None possible_values: Optional[list[PossibleValue]] = None + group_label: Optional[str] = None + fields: Optional[list["DataManipulationParameter"]] = None class DataManipulationOperation(BaseModel): @@ -62,6 +64,7 @@ class DataManipulationMethodsResponse(BaseModel): data_manipulation_methods: list[DataManipulationOperation] +DataManipulationParameter.model_rebuild() available_methods = DataManipulationMethodsResponse(data_manipulation_methods=[]) # ====================== DATA INTERPOLATION ====================== @@ -181,113 +184,40 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description = DataManipulationOperation( name="Simple Data Operations", - description="Sequence of scalar operations applied to the dataset. " - "Execution order is determined by the *_priority parameters. " - "Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6.", + description="Ordered list of scalar operations applied to the dataset. " + "Execution order is determined by the order of parameters in the request.", method_parameters=[], ) -data_addition_scalar_parameter = DataManipulationParameter( - human_readable_name="Addition", - name="addition_addend", - description="Scalar value added to every data point.", - type="number", -) - -data_subtraction_subtrahend_parameter = DataManipulationParameter( - human_readable_name="Subtraction", - name="subtraction_subtrahend", - description="Scalar value subtracted from every data point.", - type="number", -) - -data_multiplication_scalar_parameter = DataManipulationParameter( - human_readable_name="Multiplication", - name="multiplication_factor", - description="Scalar value used to multiply every data point.", - type="number", -) - -data_division_scalar_parameter = DataManipulationParameter( - human_readable_name="Division", - name="division_divisor", - description="Scalar value used as the divisor for every data point.", - type="number", -) - -data_exponentiation_exponent_parameter = DataManipulationParameter( - human_readable_name="Exponentiation", - name="exponentiation_exponent", - description="Scalar exponent used to raise the input data to a power.", - type="number", -) - -data_root_degree_parameter = DataManipulationParameter( - human_readable_name="Root", - name="root_degree", - description="Scalar degree used to compute the nth root of the input data.", - type="number", -) - -data_addition_priority_parameter = DataManipulationParameter( - human_readable_name="Addition priority", - name="addition_priority", - description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", - type="int", - default="1", -) - -data_subtraction_priority_parameter = DataManipulationParameter( - human_readable_name="Subtraction priority", - name="subtraction_priority", - description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", - type="int", - default="2", -) - -data_multiplication_priority_parameter = DataManipulationParameter( - human_readable_name="Multiplication priority", - name="multiplication_priority", - description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", - type="int", - default="3", -) - -data_division_priority_parameter = DataManipulationParameter( - human_readable_name="Division priority", - name="division_priority", - description="Execution order priority for division. Lower value = earlier execution. Default: 4.", - type="int", - default="4", -) - -data_exponentiation_priority_parameter = DataManipulationParameter( - human_readable_name="Exponentiation priority", - name="exponentiation_priority", - description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", - type="int", - default="5", -) - -data_root_priority_parameter = DataManipulationParameter( - human_readable_name="Root priority", - name="root_priority", - description="Execution order priority for root. Lower value = earlier execution. Default: 6.", - type="int", - default="6", +data_operations_parameter = DataManipulationParameter( + human_readable_name="Operations", + name="operations", + description="Ordered list of scalar operations applied to every data point.", + type="list[object]", + group_label="Operation", + fields=[ + DataManipulationParameter( + human_readable_name="Type", + name="operation_type", + description="Type of operation", + type="string", + possible_values=[ + PossibleValue(value="add", description="Addition"), + PossibleValue(value="sub", description="Subtraction"), + PossibleValue(value="mul", description="Multiplication"), + PossibleValue(value="div", description="Division"), + PossibleValue(value="pow", description="Exponentiation"), + PossibleValue(value="root", description="Nth root"), + ], + ), + DataManipulationParameter( + human_readable_name="Value", + name="operation_value", + description="Scalar value for the operation", + type="number", + ), + ], ) -simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) -simple_data_operations_description.method_parameters.append(data_addition_priority_parameter) -simple_data_operations_description.method_parameters.append(data_subtraction_subtrahend_parameter) -simple_data_operations_description.method_parameters.append(data_subtraction_priority_parameter) -simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) -simple_data_operations_description.method_parameters.append(data_multiplication_priority_parameter) -simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) -simple_data_operations_description.method_parameters.append(data_division_priority_parameter) -simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) -simple_data_operations_description.method_parameters.append(data_exponentiation_priority_parameter) -simple_data_operations_description.method_parameters.append(data_root_degree_parameter) -simple_data_operations_description.method_parameters.append(data_root_priority_parameter) - +simple_data_operations_description.method_parameters.append(data_operations_parameter) available_methods.data_manipulation_methods.append(simple_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 1f185c13..6a77c6ad 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -788,17 +788,8 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= BEGIN simple operations ============ - if any( - [ - plot_data_query.addition_addend is not None, - plot_data_query.subtraction_subtrahend is not None, - plot_data_query.multiplication_factor is not None, - plot_data_query.division_divisor is not None, - plot_data_query.exponentiation_exponent is not None, - plot_data_query.root_degree is not None, - ] - ): - data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) + if plot_data_query.operations is not None: + data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query.operations) # ============= END simple operations ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 3be95215..e24b37a0 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -1,5 +1,4 @@ from functools import reduce -from typing import Any import numpy as np from imas.ids_primitive import IDSNumericArray @@ -72,67 +71,32 @@ def _safe_division(data, divisor): return data / divisor -def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): - """ - Apply simple scalar operations to data. - Execution order is determined by the *_priority parameters from the request. - Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6. - """ +_OP_FUNCS = { + "add": lambda r, v: r + v, + "sub": lambda r, v: r - v, + "mul": lambda r, v: r * v, + "div": lambda r, v: _safe_division(r, v), + "pow": lambda r, v: np.power(r, v), + "root": lambda r, v: np.power(r, 1 / v), +} - _SIMPLE_OPERATION_DEFS = [ - { - "value_field": "addition_addend", - "priority_field": "addition_priority", - "default_priority": 1, - "func": lambda r, v: r + v, - }, - { - "value_field": "subtraction_subtrahend", - "priority_field": "subtraction_priority", - "default_priority": 2, - "func": lambda r, v: r - v, - }, - { - "value_field": "multiplication_factor", - "priority_field": "multiplication_priority", - "default_priority": 3, - "func": lambda r, v: r * v, - }, - { - "value_field": "division_divisor", - "priority_field": "division_priority", - "default_priority": 4, - "func": lambda r, v: _safe_division(r, v), - }, - { - "value_field": "exponentiation_exponent", - "priority_field": "exponentiation_priority", - "default_priority": 5, - "func": lambda r, v: np.power(r, v), - }, - { - "value_field": "root_degree", - "priority_field": "root_priority", - "default_priority": 6, - "func": lambda r, v: np.power(r, 1 / v), - }, - ] +def apply_simple_operations(data: list | np.ndarray, operations: list[str]): + """ + Apply simple scalar operations to data in the order given. + Each operation is a string in the format 'type:value', e.g. 'add:10', 'mul:5'. + """ if isinstance(data, list): - return [apply_simple_operations(x, plot_data_query) for x in data] + return [apply_simple_operations(x, operations) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): result = data - - operations = [] - for op in _SIMPLE_OPERATION_DEFS: - value = getattr(plot_data_query, op["value_field"]) - if value is not None: - priority = getattr(plot_data_query, op["priority_field"]) or op["default_priority"] - operations.append({"priority": priority, "func": op["func"], "value": value}) - - for op in sorted(operations, key=lambda x: x["priority"]): - result = op["func"](result, op["value"]) - + for op_str in operations: + op_type, value_str = op_str.split(":", 1) + value = float(value_str) + func = _OP_FUNCS.get(op_type) + if func is None: + raise InvalidParametersException(f"Unknown operation type: {op_type}") + result = func(result, value) return result else: msg = "Simple operations can be executed only on numeric arrays, not single values or strings." diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 84ee0e63..4c0bc2a6 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -81,57 +81,6 @@ class GaussianSmoothingParameters(BaseModel): ) -class SimpleOperationsParameters(BaseModel): - addition_addend: float | None = Field( - default=None, - description="Scalar value added to every data point.", - ) - addition_priority: int | None = Field( - default=None, - description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", - ) - subtraction_subtrahend: float | None = Field( - default=None, - description="Scalar value subtracted from every data point.", - ) - subtraction_priority: int | None = Field( - default=None, - description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", - ) - multiplication_factor: float | None = Field( - default=None, - description="Scalar value used to multiply every data point.", - ) - multiplication_priority: int | None = Field( - default=None, - description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", - ) - division_divisor: float | None = Field( - default=None, - description="Scalar value used as the divisor for every data point.", - ) - division_priority: int | None = Field( - default=None, - description="Execution order priority for division. Lower value = earlier execution. Default: 4.", - ) - exponentiation_exponent: float | None = Field( - default=None, - description="Scalar exponent used to raise the input data to a power.", - ) - exponentiation_priority: int | None = Field( - default=None, - description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", - ) - root_degree: float | None = Field( - default=None, - description="Scalar degree used to compute the nth root of the input data.", - ) - root_priority: int | None = Field( - default=None, - description="Execution order priority for root. Lower value = earlier execution. Default: 6.", - ) - - class PlotDataBasicParameters(BaseModel): """...""" @@ -143,13 +92,16 @@ class PlotDataBasicParameters(BaseModel): downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") smoothing_method: SmoothingMethod | None = Field(default=None, description="Smoothing method to be used") + operations: Optional[List[str]] = Field( + default=None, + description="Ordered list of scalar operations in format 'type:value' e.g. 'add:10'", + ) class PlotDataRequestModel( PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters, - SimpleOperationsParameters, ): @model_validator(mode="after") def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": @@ -168,22 +120,3 @@ def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": ) return self - - @model_validator(mode="after") - def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": - if self.division_divisor == 0: - raise ValueError("division_divisor cannot be 0") - - priorities = [ - self.addition_priority, - self.subtraction_priority, - self.multiplication_priority, - self.division_priority, - self.exponentiation_priority, - self.root_priority, - ] - defined_priorities = [p for p in priorities if p is not None] - if len(defined_priorities) != len(set(defined_priorities)): - raise ValueError("operation priorities must be unique") - - return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index e390f92a..791a325e 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -100,23 +100,23 @@ def test_plot_data_with_simple_operations(entry_path): # core_profiles.time = [1,2,3,4,5] (float) cases = [ ( - {"addition_addend": 2, "multiplication_factor": 3}, + {"operations": ["add:2", "mul:3"]}, [9.0, 12.0, 15.0, 18.0, 21.0], ), ( - {"addition_addend": 2, "multiplication_factor": 3, "addition_priority": 6, "multiplication_priority": 1}, + {"operations": ["mul:3", "add:2"]}, [5.0, 8.0, 11.0, 14.0, 17.0], ), ( - {"division_divisor": 2}, + {"operations": ["div:2"]}, [0.5, 1.0, 1.5, 2.0, 2.5], ), ( - {"exponentiation_exponent": 2}, + {"operations": ["pow:2"]}, [1.0, 4.0, 9.0, 16.0, 25.0], ), ( - {"root_degree": 2}, + {"operations": ["root:2"]}, [1.0, 1.41421356237, 1.73205080757, 2.0, 2.2360679775], ), ] diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 5ee91c1c..d0eeae7f 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,12 +1,11 @@ import numpy as np import pytest +from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, apply_simple_operations, ) -from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel -from pydantic_core._pydantic_core import ValidationError def test_apply_gaussian_smoothing(): @@ -53,47 +52,37 @@ def test_apply_savitzky_golay_smoothing(): @pytest.mark.parametrize( - ("request_kwargs", "data", "expected"), + ("operations", "data", "expected"), [ - ({"addition_addend": 2}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), - ({"subtraction_subtrahend": 1}, np.array([3.0, 4.0, 5.0]), np.array([2.0, 3.0, 4.0])), - ({"multiplication_factor": 3}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), - ({"division_divisor": 2}, np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), - ({"exponentiation_exponent": 2}, np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), - ({"root_degree": 2}, np.array([1.0, 4.0, 9.0]), np.array([1.0, 2.0, 3.0])), + (["add:2"], np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), + (["sub:1"], np.array([3.0, 4.0, 5.0]), np.array([2.0, 3.0, 4.0])), + (["mul:3"], np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), + (["div:2"], np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), + (["pow:2"], np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), + (["root:2"], np.array([1.0, 4.0, 9.0]), np.array([1.0, 2.0, 3.0])), ], ) -def test_apply_simple_operations(request_kwargs, data, expected): - request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", **request_kwargs) - - assert np.asarray(expected) == pytest.approx(apply_simple_operations(data, request)) +def test_apply_simple_operations(operations, data, expected): + assert np.asarray(expected) == pytest.approx(apply_simple_operations(data, operations)) def test_apply_simple_operations_recurses_over_lists(): - request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", addition_addend=1) data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] - - result = apply_simple_operations(data, request) - + result = apply_simple_operations(data, ["add:1"]) assert np.asarray(result[0]) == pytest.approx([2.0, 3.0]) assert np.asarray(result[1]) == pytest.approx([4.0, 5.0]) def test_apply_simple_operations_rejects_division_by_zero(): - with pytest.raises(ValidationError, match="division_divisor cannot be 0"): - PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) + with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + apply_simple_operations(np.array([1.0]), ["div:0"]) -def test_apply_simple_operations_uses_priority_order(): - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - addition_addend=1, - multiplication_factor=2, - addition_priority=2, - multiplication_priority=1, - ) +def test_apply_simple_operations_uses_order(): data = np.array([5.0]) - # default order: add then multiply -> (5+1)*2 = 12 - # priority order: multiply then add -> (5*2)+1 = 11 - result = apply_simple_operations(data, request) + # mul then add -> (5*2)+1 = 11 + result = apply_simple_operations(data, ["mul:2", "add:1"]) assert result == pytest.approx([11.0]) + # add then mul -> (5+1)*2 = 12 + result = apply_simple_operations(data, ["add:1", "mul:2"]) + assert result == pytest.approx([12.0]) From f5da1cfce42e5c94655f33ea5a953e5568fe0883 Mon Sep 17 00:00:00 2001 From: wasikj Date: Wed, 17 Jun 2026 10:42:12 +0200 Subject: [PATCH 08/29] Fix wrong description --- backend/ibex/core/data_manipulation_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index d768259f..74696422 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -87,7 +87,7 @@ class DataManipulationMethodsResponse(BaseModel): data_interpolation_method_parameter = DataManipulationParameter( human_readable_name="Interpolation method", name="interpolation_method", - description="List of URIs to gather coordinates from, for interpolation", + description="Method used during data interpolation", type="string", default=InterpolationMethod.EXACT_VALUE, possible_values=[ From 6639d458b14c3d8e09274146170f11a28d39e875 Mon Sep 17 00:00:00 2001 From: wasikj Date: Wed, 17 Jun 2026 10:54:56 +0200 Subject: [PATCH 09/29] Update docstring and test. --- .../data_source/imas_python_source_utils.py | 23 +++++++++++-------- backend/tests/test_data_manipulation.py | 6 ++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index e24b37a0..29ab3be2 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -71,21 +71,24 @@ def _safe_division(data, divisor): return data / divisor -_OP_FUNCS = { - "add": lambda r, v: r + v, - "sub": lambda r, v: r - v, - "mul": lambda r, v: r * v, - "div": lambda r, v: _safe_division(r, v), - "pow": lambda r, v: np.power(r, v), - "root": lambda r, v: np.power(r, 1 / v), -} - - def apply_simple_operations(data: list | np.ndarray, operations: list[str]): """ Apply simple scalar operations to data in the order given. Each operation is a string in the format 'type:value', e.g. 'add:10', 'mul:5'. + :param data: Input data + :param operations: List of operations and operands divided by colon (:) + :return: Data after operation """ + + _OP_FUNCS = { + "add": lambda r, v: r + v, + "sub": lambda r, v: r - v, + "mul": lambda r, v: r * v, + "div": lambda r, v: _safe_division(r, v), + "pow": lambda r, v: np.power(r, v), + "root": lambda r, v: np.power(r, 1 / v), + } + if isinstance(data, list): return [apply_simple_operations(x, operations) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index d0eeae7f..2ff43a52 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -68,9 +68,9 @@ def test_apply_simple_operations(operations, data, expected): def test_apply_simple_operations_recurses_over_lists(): data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] - result = apply_simple_operations(data, ["add:1"]) - assert np.asarray(result[0]) == pytest.approx([2.0, 3.0]) - assert np.asarray(result[1]) == pytest.approx([4.0, 5.0]) + result = apply_simple_operations(data, ["add:1", "mul:2", "add:3"]) + assert np.asarray(result[0]) == pytest.approx([7.0, 9.0]) + assert np.asarray(result[1]) == pytest.approx([11.0, 13.0]) def test_apply_simple_operations_rejects_division_by_zero(): From d6b6227340f1cb3dbdb69e90d1219331a569e53e Mon Sep 17 00:00:00 2001 From: wasikj Date: Thu, 18 Jun 2026 08:29:09 +0200 Subject: [PATCH 10/29] Signal data operations (WIP) --- .../ibex/core/data_manipulation_methods.py | 43 ++++++++++++ .../ibex/data_source/imas_python_source.py | 51 +++++++------- .../data_source/imas_python_source_utils.py | 60 ++++++++-------- .../endpoints/schemas/request_data_schemas.py | 4 ++ backend/tests/test_data_manipulation.py | 60 +++++++++------- .../backend_development/data_manipulation.rst | 68 +++++++++++++++++++ 6 files changed, 205 insertions(+), 81 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 74696422..2ba58f04 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -221,3 +221,46 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description.method_parameters.append(data_operations_parameter) available_methods.data_manipulation_methods.append(simple_data_operations_description) + +# ====================== SIGNAL DATA OPERATIONS ====================== + +signal_data_operations_description = DataManipulationOperation( + name="Signal Data Operations", + description="Ordered list of signal operations applied to the dataset. " + "Execution order is determined by the order of parameters in the request. " + "Each value is a URI pointing to a signal.", + method_parameters=[], +) + +signal_data_operations_parameter = DataManipulationParameter( + human_readable_name="Operations", + name="operations", + description="Ordered list of signal operations applied to every data point.", + type="list[object]", + group_label="Operation", + fields=[ + DataManipulationParameter( + human_readable_name="Type", + name="operation_type", + description="Type of operation", + type="string", + possible_values=[ + PossibleValue(value="add", description="Addition"), + PossibleValue(value="sub", description="Subtraction"), + PossibleValue(value="mul", description="Multiplication"), + PossibleValue(value="div", description="Division"), + PossibleValue(value="pow", description="Exponentiation"), + PossibleValue(value="root", description="Nth root"), + ], + ), + DataManipulationParameter( + human_readable_name="Signal URI", + name="operation_value", + description="URI to signal data for the operation", + type="string", + ), + ], +) + +signal_data_operations_description.method_parameters.append(signal_data_operations_parameter) +available_methods.data_manipulation_methods.append(signal_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 4223eb54..2194e434 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -845,14 +845,7 @@ def convert_to_lists(data): original_coord_values.reverse() store_other_signals_data = False # used for signal combining after interpolation - if any( - [ - plot_data_query.signal_addition_addend_uri, - plot_data_query.signal_subtraction_subtrahend_uri, - plot_data_query.signal_multiplication_factor_uri, - plot_data_query.signal_division_divisor_uri, - ] - ): + if plot_data_query.signal_operations: store_other_signals_data = True for _uri in plot_data_query.interpolate_over: @@ -867,6 +860,7 @@ def convert_to_lists(data): new_plot_data_query.uri = _uri new_plot_data_query.interpolate_over = None new_plot_data_query.smoothing_method = None + new_plot_data_query.signal_operations = None # interpolate_to will be used later with signal combining interpolate_to = self.get_plot_data(new_plot_data_query)["data"] interpolate_to_coordinates = interpolate_to["coordinates"] @@ -929,27 +923,30 @@ def convert_to_lists(data): # ============= BEGIN signal operations ============= - _SIGNAL_URI_FIELDS = [ - "signal_addition_addend_uri", - "signal_subtraction_subtrahend_uri", - "signal_multiplication_factor_uri", - "signal_division_divisor_uri", - ] + if plot_data_query.signal_operations: + # Collect set of signal URIs that will be used in signal operations + signal_op_uris = set() + for op_str in (plot_data_query.signal_operations or []): + _, op_uri = op_str.split(":", 1) + signal_op_uris.add(op_uri) - for field in _SIGNAL_URI_FIELDS: - signal_uris = getattr(plot_data_query, field) - if signal_uris: - for signal_uri in signal_uris: - if "interpolated_data" not in others_signals_data[signal_uri]: - signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) - signal_data = resample_data_without_interpolation( - tuple(others_signals_data[signal_uri]["coordinates"]), - signal_data, - tuple(common_coords_values), - ) - others_signals_data[uri]["interpolated_data"] = signal_data + # For each signal not yet interpolated — interpolate it onto the common coordinate grid + for signal_uri in signal_op_uris: + if signal_uri not in others_signals_data: + continue + if "interpolated_data" not in others_signals_data[signal_uri]: + signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) + signal_data = resample_data_without_interpolation( + tuple(others_signals_data[signal_uri]["coordinates"]), + signal_data, + tuple(common_coords_values), + ) + others_signals_data[signal_uri]["interpolated_data"] = signal_data - data_to_be_returned = apply_signal_operations(data_to_be_returned, plot_data_query, others_signals_data) + # Apply signal operations (addition, subtraction, multiplication, etc.) + data_to_be_returned = apply_signal_operations( + data_to_be_returned, plot_data_query.signal_operations, others_signals_data + ) # ============= END signal operations ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index fed0079d..fd28d447 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -109,39 +109,39 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): raise InvalidParametersException(msg) -def apply_signal_operations(data: list | np.ndarray, plot_data_query: Any, signal_data_by_uri: dict): +def apply_signal_operations(data: list | np.ndarray, operations: list[str], signal_data_by_uri: dict): """ - Apply signal operations to data. - - :param data: The input data array. - :param plot_data_query: Request model with signal URI fields. + Apply signal operations to data in the order given. + Each operation is a string in the format 'type:uri', e.g. 'add:some/imas/uri', 'mul:other/uri'. + :param data: Input data + :param operations: List of operations and URIs divided by colon (:) :param signal_data_by_uri: Dict mapping signal URIs to their interpolated data arrays. + :return: Data after operation """ - # signal_data_by_uri: { - # uri : - # data : - # interpolated_data : - # } - - _SIGNAL_OPERATION_DEFS = [ - ("signal_addition_addend_uri", "signal_addition_priority", 1, lambda r, v: r + v), - ("signal_subtraction_subtrahend_uri", "signal_subtraction_priority", 2, lambda r, v: r - v), - ("signal_multiplication_factor_uri", "signal_multiplication_priority", 3, lambda r, v: r * v), - ("signal_division_divisor_uri", "signal_division_priority", 4, lambda r, v: _safe_division(r, v)), - ] - - operations = [] - for uri_field, priority_field, default_priority, func in _SIGNAL_OPERATION_DEFS: - uris = getattr(plot_data_query, uri_field) - if uris: - for uri in uris: - priority = getattr(plot_data_query, priority_field) or default_priority - signal_data = signal_data_by_uri[uri] - operations.append({"priority": priority, "func": func, "value": signal_data}) - - for op in sorted(operations, key=lambda x: x["priority"]): - data = op["func"](data, op["value"]) - return data + _OP_FUNCS = { + "add": lambda r, v: r + v, + "sub": lambda r, v: r - v, + "mul": lambda r, v: r * v, + "div": lambda r, v: _safe_division(r, v), + "pow": lambda r, v: np.power(r, v), + "root": lambda r, v: np.power(r, 1 / v), + } + + if isinstance(data, list): + return [apply_signal_operations(x, operations, signal_data_by_uri) for x in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + result = data + for op_str in operations: + op_type, uri = op_str.split(":", 1) + value = signal_data_by_uri[uri] + func = _OP_FUNCS.get(op_type) + if func is None: + raise InvalidParametersException(f"Unknown operation type: {op_type}") + result = func(result, value) + return result + else: + msg = "Signal operations can be executed only on numeric arrays, not single values or strings." + raise InvalidParametersException(msg) def union_arrays(data: list): diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 4c0bc2a6..bea85b54 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -96,6 +96,10 @@ class PlotDataBasicParameters(BaseModel): default=None, description="Ordered list of scalar operations in format 'type:value' e.g. 'add:10'", ) + signal_operations: Optional[List[str]] = Field( + default=None, + description="Ordered list of signal operations in format 'type:uri' e.g. 'add:some/imas/uri'", + ) class PlotDataRequestModel( diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index e7968b8e..845b51be 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -12,52 +12,36 @@ def test_apply_signal_operations_addition(): addend_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([1.0, 2.0, 3.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[addend_uri], - signal_addition_addend_uri=[addend_uri], - ) + operations = [f"add:{addend_uri}"] signal_data_by_uri = {addend_uri: np.array([10.0, 20.0, 30.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [11.0, 22.0, 33.0]) def test_apply_signal_operations_subtraction(): subtrahend_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([10.0, 20.0, 30.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[subtrahend_uri], - signal_subtraction_subtrahend_uri=[subtrahend_uri], - ) + operations = [f"sub:{subtrahend_uri}"] signal_data_by_uri = {subtrahend_uri: np.array([1.0, 2.0, 3.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [9.0, 18.0, 27.0]) def test_apply_signal_operations_multiplication(): factor_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([1.0, 2.0, 3.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[factor_uri], - signal_multiplication_factor_uri=[factor_uri], - ) + operations = [f"mul:{factor_uri}"] signal_data_by_uri = {factor_uri: np.array([2.0, 3.0, 4.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [2.0, 6.0, 12.0]) def test_apply_signal_operations_division(): divisor_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([10.0, 20.0, 30.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[divisor_uri], - signal_division_divisor_uri=[divisor_uri], - ) + operations = [f"div:{divisor_uri}"] signal_data_by_uri = {divisor_uri: np.array([2.0, 5.0, 6.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [5.0, 4.0, 5.0]) @@ -139,3 +123,31 @@ def test_apply_simple_operations_uses_order(): # add then mul -> (5+1)*2 = 12 result = apply_simple_operations(data, ["add:1", "mul:2"]) assert result == pytest.approx([12.0]) + + +def test_apply_signal_operations_uses_order(): + uri_a = "some/uri/a" + uri_b = "some/uri/b" + data = np.array([5.0]) + signal_data_by_uri = {uri_a: np.array([2.0]), uri_b: np.array([1.0])} + # mul then add -> (5*2)+1 = 11 + result = apply_signal_operations(data, [f"mul:{uri_a}", f"add:{uri_b}"], signal_data_by_uri) + assert result == pytest.approx([11.0]) + # add then mul -> (5+1)*2 = 12 + result = apply_signal_operations(data, [f"add:{uri_b}", f"mul:{uri_a}"], signal_data_by_uri) + assert result == pytest.approx([12.0]) + + +def test_apply_signal_operations_recurses_over_lists(): + uri = "some/uri" + data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] + signal_data_by_uri = {uri: np.array([2.0, 3.0])} + result = apply_signal_operations(data, [f"add:{uri}", f"mul:{uri}"], signal_data_by_uri) + assert np.asarray(result[0]) == pytest.approx([6.0, 15.0]) + assert np.asarray(result[1]) == pytest.approx([10.0, 21.0]) + + +def test_apply_signal_operations_rejects_division_by_zero(): + uri = "some/uri" + with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + apply_signal_operations(np.array([1.0]), [f"div:{uri}"], {uri: np.array([0.0])}) diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index e0e02599..22b84e89 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -18,6 +18,11 @@ The backend applies the manipulation stages in this order: This means later stages operate on the output of earlier ones when the corresponding request parameters are enabled. +.. note:: + + Signal operations are applied after data interpolation and before downsampling. + See the dedicated section below for details. + Data smoothing --------------- @@ -140,3 +145,66 @@ Two operations: curl -X 'GET' \ '/data/plot_data?uri=&addition_addend=2&multiplication_factor=3' \ -H 'accept: application/json' + + +Signal operations +------------------ + +IBEX supports element-wise arithmetic between two signals (datasets) during a ``/data/plot_data`` request. +Unlike simple scalar operations which use fixed numeric values, signal operations use **other IMAS signals** as operands. +This enables combining data from different IMAS nodes arithmetically, for example subtracting a background signal from a measurement. + +Signal operations are applied **after data interpolation** and **before downsampling**. +When a signal referenced in a signal operation was not already interpolated via the ``interpolate_over`` parameter, it is interpolated on the fly onto the common coordinate grid during this step. +Operations are applied in the order they appear in the request. + +Configuration +~~~~~~~~~~~~~~ + +Signal operations are configured through the ``signal_operations`` parameter of the ``/data/plot_data/`` endpoint. +It accepts a list of strings in the format ``type:uri``, where ``type`` is one of the supported operations and ``uri`` is the IMAS URI of the operand signal. + +The following operation types are supported: + +* ``add`` — addition +* ``sub`` — subtraction +* ``mul`` — multiplication +* ``div`` — division +* ``pow`` — exponentiation +* ``root`` — Nth root + +Division by zero is rejected by the backend. + +The full list of available operations can be retrieved from the ``/info/data_manipulation_methods/`` endpoint. + +Implementation +~~~~~~~~~~~~~~~ + +Signal operations are applied in ``apply_signal_operations()`` in +``backend/ibex/data_source/imas_python_source_utils.py``. + +The function iterates over the list of operation strings in order, looking up each +operand signal's interpolated data from a dictionary populated during the interpolation step. For each operation the corresponding arithmetic function is applied element-wise between the current result and the operand signal data. + +The implementation handles both flat arrays and nested lists of arrays (higher-dimensional data) recursively. + +Example usage +~~~~~~~~~~~~~~ + +The following examples demonstrate how signal operations can be enabled for testing purposes. + +Addition of two signals: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&signal_operations=add:' \ + -H 'accept: application/json' + +Multiple signal operations (subtraction then multiplication): + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&signal_operations=sub:&signal_operations=mul:' \ + -H 'accept: application/json' From d79fac57fecf557a2c3264a21ca201af996a58da Mon Sep 17 00:00:00 2001 From: wasikj Date: Thu, 18 Jun 2026 09:18:18 +0200 Subject: [PATCH 11/29] Update docs. Mini-fixes. --- .../data_source/imas_python_source_utils.py | 3 +- backend/tests/test_data_manipulation.py | 2 +- .../backend_development/data_manipulation.rst | 64 +++++++++++++------ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 29ab3be2..4e22c52c 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -67,7 +67,7 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): def _safe_division(data, divisor): if divisor == 0: - raise InvalidParametersException("division_divisor cannot be 0") + raise InvalidParametersException("Division by zero is not allowed") return data / divisor @@ -79,7 +79,6 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): :param operations: List of operations and operands divided by colon (:) :return: Data after operation """ - _OP_FUNCS = { "add": lambda r, v: r + v, "sub": lambda r, v: r - v, diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 2ff43a52..78ca8633 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -74,7 +74,7 @@ def test_apply_simple_operations_recurses_over_lists(): def test_apply_simple_operations_rejects_division_by_zero(): - with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + with pytest.raises(InvalidParametersException, match="Division by zero is not allowed"): apply_simple_operations(np.array([1.0]), ["div:0"]) diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index e0e02599..39263c23 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -95,48 +95,70 @@ Savitzky-Golay smoothing: -H 'accept: application/json' -Simple scalar operations +Simple data operations ------------------------ -IBEX also supports a sequence of scalar operations that can be applied to the returned dataset: +IBEX supports a sequence of scalar operations that can be applied element-wise to every data point in the returned dataset. +Simple data operations use fixed numeric values. -* addition -* subtraction -* multiplication -* division -* exponentiation -* root +The following operation types are supported: -These operations are executed in that order. In practice, the backend applies them sequentially to the numerical data before any smoothing or resampling step. +* ``add`` — addition +* ``sub`` — subtraction +* ``mul`` — multiplication +* ``div`` — division +* ``pow`` — exponentiation +* ``root`` — Nth root -The corresponding request parameters are: +Operations are applied **in the order they appear** in the request. +The backend executes them sequentially on the numerical data **before** smoothing, interpolation, or downsampling. -* ``addition_addend`` -* ``subtraction_subtrahend`` -* ``multiplication_factor`` -* ``division_divisor`` -* ``exponentiation_exponent`` -* ``root_degree`` +Configuration +~~~~~~~~~~~~~~ + +Simple data operations are configured through the ``operations`` parameter of the ``/data/plot_data/`` endpoint. +It accepts a list of strings in the format ``type:value``, for example ``add:10`` or ``mul:2.5``. + +The full list of available operations can be retrieved from the ``/info/data_manipulation_methods/`` endpoint. + +Division by zero is rejected by the backend with an ``InvalidParametersException`` ("Division by zero is not allowed"). + +Implementation +~~~~~~~~~~~~~~~ -Division by zero is rejected by the backend. +Simple data operations are applied in ``apply_simple_operations()`` in +``backend/ibex/data_source/imas_python_source_utils.py``. + +The function iterates over the list of operation strings in order, splitting each on the colon to extract the operation type and the scalar value. +For each operation the corresponding arithmetic lambda is applied element-wise to the data array. + +The implementation handles both flat arrays and nested lists of arrays (higher-dimensional data) recursively. Example usage ~~~~~~~~~~~~~~ The following examples demonstrate how simple data operations can be enabled for testing purposes. -Single operation: +Single operation (addition by 2): + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&operations=add:2' \ + -H 'accept: application/json' + +Two chained operations (add then multiply): .. code-block:: bash curl -X 'GET' \ - '/data/plot_data?uri=&addition_addend=2' \ + '/data/plot_data?uri=&operations=add:2&operations=mul:3' \ -H 'accept: application/json' -Two operations: +Order matters (multiply then add yields different result): .. code-block:: bash curl -X 'GET' \ - '/data/plot_data?uri=&addition_addend=2&multiplication_factor=3' \ + '/data/plot_data?uri=&operations=mul:3&operations=add:2' \ -H 'accept: application/json' From 4e3cec7cdff4997603b2b7711509b5f13f859d55 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 16 Jun 2026 12:01:00 +0200 Subject: [PATCH 12/29] Signal operations (WIP) --- .../ibex/data_source/imas_python_source.py | 55 ++++++++++++++++++- .../data_source/imas_python_source_utils.py | 40 +++++++++++++- .../endpoints/schemas/request_data_schemas.py | 41 ++++++++++++++ backend/tests/test_data_manipulation.py | 53 ++++++++++++++++++ 4 files changed, 186 insertions(+), 3 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 6a77c6ad..4223eb54 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -50,6 +50,7 @@ apply_savgol_filter, apply_gaussian_filter, apply_simple_operations, + apply_signal_operations, ) from ibex.core.data_manipulation_methods import SmoothingMethod, InterpolationMethod from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel @@ -822,7 +823,6 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= END data smoothing ============= # ============= BEGIN resample data onto new time vector ============= - def convert_to_lists(data): if isinstance(data, list): return [convert_to_lists(d) for d in data] @@ -831,6 +831,10 @@ def convert_to_lists(data): else: return data + # list of dicts used when combining signals after + # {uri:str, data:list[*], interpolated_data: list[*]} + others_signals_data = {} + if plot_data_query.interpolate_over: # =================== GATHER ALL COORDINATES =================== original_coord_values = [] @@ -840,6 +844,17 @@ def convert_to_lists(data): original_coord_values.append(sorted(set(flatten(c["value"])))) original_coord_values.reverse() + store_other_signals_data = False # used for signal combining after interpolation + if any( + [ + plot_data_query.signal_addition_addend_uri, + plot_data_query.signal_subtraction_subtrahend_uri, + plot_data_query.signal_multiplication_factor_uri, + plot_data_query.signal_division_divisor_uri, + ] + ): + store_other_signals_data = True + for _uri in plot_data_query.interpolate_over: _uri_obj = IMAS_URI(_uri) @@ -852,7 +867,17 @@ def convert_to_lists(data): new_plot_data_query.uri = _uri new_plot_data_query.interpolate_over = None new_plot_data_query.smoothing_method = None - interpolate_to_coordinates = self.get_plot_data(new_plot_data_query)["data"]["coordinates"] + # interpolate_to will be used later with signal combining + interpolate_to = self.get_plot_data(new_plot_data_query)["data"] + interpolate_to_coordinates = interpolate_to["coordinates"] + if store_other_signals_data: + others_signals_data[_uri] = { + "uri": _uri, + "data": interpolate_to["value"], + "coordinates": [ + sorted(set(flatten(convert_to_lists(c["value"])))) for c in interpolate_to_coordinates + ], + } if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): message = "Interpolation error. Source and target nodes have different number of coordinates." @@ -902,6 +927,32 @@ def convert_to_lists(data): # ============= END resample data onto new time vector ============= + # ============= BEGIN signal operations ============= + + _SIGNAL_URI_FIELDS = [ + "signal_addition_addend_uri", + "signal_subtraction_subtrahend_uri", + "signal_multiplication_factor_uri", + "signal_division_divisor_uri", + ] + + for field in _SIGNAL_URI_FIELDS: + signal_uris = getattr(plot_data_query, field) + if signal_uris: + for signal_uri in signal_uris: + if "interpolated_data" not in others_signals_data[signal_uri]: + signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) + signal_data = resample_data_without_interpolation( + tuple(others_signals_data[signal_uri]["coordinates"]), + signal_data, + tuple(common_coords_values), + ) + others_signals_data[uri]["interpolated_data"] = signal_data + + data_to_be_returned = apply_signal_operations(data_to_be_returned, plot_data_query, others_signals_data) + + # ============= END signal operations ============= + try: original_data_shape = np.asarray(data_to_be_returned).shape except ValueError: diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 4e22c52c..8dcc492b 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -66,7 +66,10 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): def _safe_division(data, divisor): - if divisor == 0: + if isinstance(divisor, np.ndarray): + if np.any(divisor == 0): + raise InvalidParametersException("Division by zero is not allowed") + elif divisor == 0: raise InvalidParametersException("Division by zero is not allowed") return data / divisor @@ -105,6 +108,41 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): raise InvalidParametersException(msg) +def apply_signal_operations(data: list | np.ndarray, plot_data_query: Any, signal_data_by_uri: dict): + """ + Apply signal operations to data. + + :param data: The input data array. + :param plot_data_query: Request model with signal URI fields. + :param signal_data_by_uri: Dict mapping signal URIs to their interpolated data arrays. + """ + # signal_data_by_uri: { + # uri : + # data : + # interpolated_data : + # } + + _SIGNAL_OPERATION_DEFS = [ + ("signal_addition_addend_uri", "signal_addition_priority", 1, lambda r, v: r + v), + ("signal_subtraction_subtrahend_uri", "signal_subtraction_priority", 2, lambda r, v: r - v), + ("signal_multiplication_factor_uri", "signal_multiplication_priority", 3, lambda r, v: r * v), + ("signal_division_divisor_uri", "signal_division_priority", 4, lambda r, v: _safe_division(r, v)), + ] + + operations = [] + for uri_field, priority_field, default_priority, func in _SIGNAL_OPERATION_DEFS: + uris = getattr(plot_data_query, uri_field) + if uris: + for uri in uris: + priority = getattr(plot_data_query, priority_field) or default_priority + signal_data = signal_data_by_uri[uri] + operations.append({"priority": priority, "func": func, "value": signal_data}) + + for op in sorted(operations, key=lambda x: x["priority"]): + data = op["func"](data, op["value"]) + return data + + def union_arrays(data: list): return reduce(np.union1d, data) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 4c0bc2a6..bc77fa4e 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -120,3 +120,44 @@ def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": ) return self + + @model_validator(mode="after") + def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": + if self.division_divisor == 0: + raise ValueError("division_divisor cannot be 0") + + signal_uri_lists = [ + self.signal_addition_addend_uri, + self.signal_subtraction_subtrahend_uri, + self.signal_multiplication_factor_uri, + self.signal_division_divisor_uri, + ] + for uri_list in signal_uri_lists: + if uri_list: + for signal_uri in uri_list: + if not self.interpolate_over or signal_uri not in self.interpolate_over: + raise ValueError(f"Signal URI '{signal_uri}' must be listed in interpolate_over") + + signal_priorities = [ + self.signal_addition_priority, + self.signal_subtraction_priority, + self.signal_multiplication_priority, + self.signal_division_priority, + ] + defined_signal_priorities = [p for p in signal_priorities if p is not None] + if len(defined_signal_priorities) != len(set(defined_signal_priorities)): + raise ValueError("signal operation priorities must be unique") + + priorities = [ + self.addition_priority, + self.subtraction_priority, + self.multiplication_priority, + self.division_priority, + self.exponentiation_priority, + self.root_priority, + ] + defined_priorities = [p for p in priorities if p is not None] + if len(defined_priorities) != len(set(defined_priorities)): + raise ValueError("operation priorities must be unique") + + return self diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 78ca8633..fa94ba6b 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -4,10 +4,63 @@ from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, + apply_signal_operations, apply_simple_operations, ) +def test_apply_signal_operations_addition(): + addend_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([1.0, 2.0, 3.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[addend_uri], + signal_addition_addend_uri=[addend_uri], + ) + signal_data_by_uri = {addend_uri: np.array([10.0, 20.0, 30.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [11.0, 22.0, 33.0]) + + +def test_apply_signal_operations_subtraction(): + subtrahend_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([10.0, 20.0, 30.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[subtrahend_uri], + signal_subtraction_subtrahend_uri=[subtrahend_uri], + ) + signal_data_by_uri = {subtrahend_uri: np.array([1.0, 2.0, 3.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [9.0, 18.0, 27.0]) + + +def test_apply_signal_operations_multiplication(): + factor_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([1.0, 2.0, 3.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[factor_uri], + signal_multiplication_factor_uri=[factor_uri], + ) + signal_data_by_uri = {factor_uri: np.array([2.0, 3.0, 4.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [2.0, 6.0, 12.0]) + + +def test_apply_signal_operations_division(): + divisor_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([10.0, 20.0, 30.0]) + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + interpolate_over=[divisor_uri], + signal_division_divisor_uri=[divisor_uri], + ) + signal_data_by_uri = {divisor_uri: np.array([2.0, 5.0, 6.0])} + result = apply_signal_operations(data, request, signal_data_by_uri) + assert np.allclose(result, [5.0, 4.0, 5.0]) + + def test_apply_gaussian_smoothing(): data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1]) From 25a4b1d385a983f1d5c37f1774bfb3597afaa38f Mon Sep 17 00:00:00 2001 From: wasikj Date: Thu, 18 Jun 2026 08:29:09 +0200 Subject: [PATCH 13/29] Signal data operations (WIP) --- .../ibex/core/data_manipulation_methods.py | 43 ++++++++++++ .../ibex/data_source/imas_python_source.py | 51 +++++++------- .../data_source/imas_python_source_utils.py | 60 ++++++++-------- .../endpoints/schemas/request_data_schemas.py | 4 ++ backend/tests/test_data_manipulation.py | 60 +++++++++------- .../backend_development/data_manipulation.rst | 68 +++++++++++++++++++ 6 files changed, 205 insertions(+), 81 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 74696422..2ba58f04 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -221,3 +221,46 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description.method_parameters.append(data_operations_parameter) available_methods.data_manipulation_methods.append(simple_data_operations_description) + +# ====================== SIGNAL DATA OPERATIONS ====================== + +signal_data_operations_description = DataManipulationOperation( + name="Signal Data Operations", + description="Ordered list of signal operations applied to the dataset. " + "Execution order is determined by the order of parameters in the request. " + "Each value is a URI pointing to a signal.", + method_parameters=[], +) + +signal_data_operations_parameter = DataManipulationParameter( + human_readable_name="Operations", + name="operations", + description="Ordered list of signal operations applied to every data point.", + type="list[object]", + group_label="Operation", + fields=[ + DataManipulationParameter( + human_readable_name="Type", + name="operation_type", + description="Type of operation", + type="string", + possible_values=[ + PossibleValue(value="add", description="Addition"), + PossibleValue(value="sub", description="Subtraction"), + PossibleValue(value="mul", description="Multiplication"), + PossibleValue(value="div", description="Division"), + PossibleValue(value="pow", description="Exponentiation"), + PossibleValue(value="root", description="Nth root"), + ], + ), + DataManipulationParameter( + human_readable_name="Signal URI", + name="operation_value", + description="URI to signal data for the operation", + type="string", + ), + ], +) + +signal_data_operations_description.method_parameters.append(signal_data_operations_parameter) +available_methods.data_manipulation_methods.append(signal_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 4223eb54..2194e434 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -845,14 +845,7 @@ def convert_to_lists(data): original_coord_values.reverse() store_other_signals_data = False # used for signal combining after interpolation - if any( - [ - plot_data_query.signal_addition_addend_uri, - plot_data_query.signal_subtraction_subtrahend_uri, - plot_data_query.signal_multiplication_factor_uri, - plot_data_query.signal_division_divisor_uri, - ] - ): + if plot_data_query.signal_operations: store_other_signals_data = True for _uri in plot_data_query.interpolate_over: @@ -867,6 +860,7 @@ def convert_to_lists(data): new_plot_data_query.uri = _uri new_plot_data_query.interpolate_over = None new_plot_data_query.smoothing_method = None + new_plot_data_query.signal_operations = None # interpolate_to will be used later with signal combining interpolate_to = self.get_plot_data(new_plot_data_query)["data"] interpolate_to_coordinates = interpolate_to["coordinates"] @@ -929,27 +923,30 @@ def convert_to_lists(data): # ============= BEGIN signal operations ============= - _SIGNAL_URI_FIELDS = [ - "signal_addition_addend_uri", - "signal_subtraction_subtrahend_uri", - "signal_multiplication_factor_uri", - "signal_division_divisor_uri", - ] + if plot_data_query.signal_operations: + # Collect set of signal URIs that will be used in signal operations + signal_op_uris = set() + for op_str in (plot_data_query.signal_operations or []): + _, op_uri = op_str.split(":", 1) + signal_op_uris.add(op_uri) - for field in _SIGNAL_URI_FIELDS: - signal_uris = getattr(plot_data_query, field) - if signal_uris: - for signal_uri in signal_uris: - if "interpolated_data" not in others_signals_data[signal_uri]: - signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) - signal_data = resample_data_without_interpolation( - tuple(others_signals_data[signal_uri]["coordinates"]), - signal_data, - tuple(common_coords_values), - ) - others_signals_data[uri]["interpolated_data"] = signal_data + # For each signal not yet interpolated — interpolate it onto the common coordinate grid + for signal_uri in signal_op_uris: + if signal_uri not in others_signals_data: + continue + if "interpolated_data" not in others_signals_data[signal_uri]: + signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) + signal_data = resample_data_without_interpolation( + tuple(others_signals_data[signal_uri]["coordinates"]), + signal_data, + tuple(common_coords_values), + ) + others_signals_data[signal_uri]["interpolated_data"] = signal_data - data_to_be_returned = apply_signal_operations(data_to_be_returned, plot_data_query, others_signals_data) + # Apply signal operations (addition, subtraction, multiplication, etc.) + data_to_be_returned = apply_signal_operations( + data_to_be_returned, plot_data_query.signal_operations, others_signals_data + ) # ============= END signal operations ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 8dcc492b..da6e5f77 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -108,39 +108,39 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): raise InvalidParametersException(msg) -def apply_signal_operations(data: list | np.ndarray, plot_data_query: Any, signal_data_by_uri: dict): +def apply_signal_operations(data: list | np.ndarray, operations: list[str], signal_data_by_uri: dict): """ - Apply signal operations to data. - - :param data: The input data array. - :param plot_data_query: Request model with signal URI fields. + Apply signal operations to data in the order given. + Each operation is a string in the format 'type:uri', e.g. 'add:some/imas/uri', 'mul:other/uri'. + :param data: Input data + :param operations: List of operations and URIs divided by colon (:) :param signal_data_by_uri: Dict mapping signal URIs to their interpolated data arrays. + :return: Data after operation """ - # signal_data_by_uri: { - # uri : - # data : - # interpolated_data : - # } - - _SIGNAL_OPERATION_DEFS = [ - ("signal_addition_addend_uri", "signal_addition_priority", 1, lambda r, v: r + v), - ("signal_subtraction_subtrahend_uri", "signal_subtraction_priority", 2, lambda r, v: r - v), - ("signal_multiplication_factor_uri", "signal_multiplication_priority", 3, lambda r, v: r * v), - ("signal_division_divisor_uri", "signal_division_priority", 4, lambda r, v: _safe_division(r, v)), - ] - - operations = [] - for uri_field, priority_field, default_priority, func in _SIGNAL_OPERATION_DEFS: - uris = getattr(plot_data_query, uri_field) - if uris: - for uri in uris: - priority = getattr(plot_data_query, priority_field) or default_priority - signal_data = signal_data_by_uri[uri] - operations.append({"priority": priority, "func": func, "value": signal_data}) - - for op in sorted(operations, key=lambda x: x["priority"]): - data = op["func"](data, op["value"]) - return data + _OP_FUNCS = { + "add": lambda r, v: r + v, + "sub": lambda r, v: r - v, + "mul": lambda r, v: r * v, + "div": lambda r, v: _safe_division(r, v), + "pow": lambda r, v: np.power(r, v), + "root": lambda r, v: np.power(r, 1 / v), + } + + if isinstance(data, list): + return [apply_signal_operations(x, operations, signal_data_by_uri) for x in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + result = data + for op_str in operations: + op_type, uri = op_str.split(":", 1) + value = signal_data_by_uri[uri] + func = _OP_FUNCS.get(op_type) + if func is None: + raise InvalidParametersException(f"Unknown operation type: {op_type}") + result = func(result, value) + return result + else: + msg = "Signal operations can be executed only on numeric arrays, not single values or strings." + raise InvalidParametersException(msg) def union_arrays(data: list): diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index bc77fa4e..2232f3fd 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -96,6 +96,10 @@ class PlotDataBasicParameters(BaseModel): default=None, description="Ordered list of scalar operations in format 'type:value' e.g. 'add:10'", ) + signal_operations: Optional[List[str]] = Field( + default=None, + description="Ordered list of signal operations in format 'type:uri' e.g. 'add:some/imas/uri'", + ) class PlotDataRequestModel( diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index fa94ba6b..2dc81cdb 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -12,52 +12,36 @@ def test_apply_signal_operations_addition(): addend_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([1.0, 2.0, 3.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[addend_uri], - signal_addition_addend_uri=[addend_uri], - ) + operations = [f"add:{addend_uri}"] signal_data_by_uri = {addend_uri: np.array([10.0, 20.0, 30.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [11.0, 22.0, 33.0]) def test_apply_signal_operations_subtraction(): subtrahend_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([10.0, 20.0, 30.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[subtrahend_uri], - signal_subtraction_subtrahend_uri=[subtrahend_uri], - ) + operations = [f"sub:{subtrahend_uri}"] signal_data_by_uri = {subtrahend_uri: np.array([1.0, 2.0, 3.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [9.0, 18.0, 27.0]) def test_apply_signal_operations_multiplication(): factor_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([1.0, 2.0, 3.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[factor_uri], - signal_multiplication_factor_uri=[factor_uri], - ) + operations = [f"mul:{factor_uri}"] signal_data_by_uri = {factor_uri: np.array([2.0, 3.0, 4.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [2.0, 6.0, 12.0]) def test_apply_signal_operations_division(): divisor_uri = "imas:hdf5?path=/dummy/interpolation_db_2#equilibrium/time_slice[:]/profiles_2d[:]/psi" data = np.array([10.0, 20.0, 30.0]) - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - interpolate_over=[divisor_uri], - signal_division_divisor_uri=[divisor_uri], - ) + operations = [f"div:{divisor_uri}"] signal_data_by_uri = {divisor_uri: np.array([2.0, 5.0, 6.0])} - result = apply_signal_operations(data, request, signal_data_by_uri) + result = apply_signal_operations(data, operations, signal_data_by_uri) assert np.allclose(result, [5.0, 4.0, 5.0]) @@ -139,3 +123,31 @@ def test_apply_simple_operations_uses_order(): # add then mul -> (5+1)*2 = 12 result = apply_simple_operations(data, ["add:1", "mul:2"]) assert result == pytest.approx([12.0]) + + +def test_apply_signal_operations_uses_order(): + uri_a = "some/uri/a" + uri_b = "some/uri/b" + data = np.array([5.0]) + signal_data_by_uri = {uri_a: np.array([2.0]), uri_b: np.array([1.0])} + # mul then add -> (5*2)+1 = 11 + result = apply_signal_operations(data, [f"mul:{uri_a}", f"add:{uri_b}"], signal_data_by_uri) + assert result == pytest.approx([11.0]) + # add then mul -> (5+1)*2 = 12 + result = apply_signal_operations(data, [f"add:{uri_b}", f"mul:{uri_a}"], signal_data_by_uri) + assert result == pytest.approx([12.0]) + + +def test_apply_signal_operations_recurses_over_lists(): + uri = "some/uri" + data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] + signal_data_by_uri = {uri: np.array([2.0, 3.0])} + result = apply_signal_operations(data, [f"add:{uri}", f"mul:{uri}"], signal_data_by_uri) + assert np.asarray(result[0]) == pytest.approx([6.0, 15.0]) + assert np.asarray(result[1]) == pytest.approx([10.0, 21.0]) + + +def test_apply_signal_operations_rejects_division_by_zero(): + uri = "some/uri" + with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + apply_signal_operations(np.array([1.0]), [f"div:{uri}"], {uri: np.array([0.0])}) diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index 39263c23..aa664d50 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -18,6 +18,11 @@ The backend applies the manipulation stages in this order: This means later stages operate on the output of earlier ones when the corresponding request parameters are enabled. +.. note:: + + Signal operations are applied after data interpolation and before downsampling. + See the dedicated section below for details. + Data smoothing --------------- @@ -162,3 +167,66 @@ Order matters (multiply then add yields different result): curl -X 'GET' \ '/data/plot_data?uri=&operations=mul:3&operations=add:2' \ -H 'accept: application/json' + + +Signal operations +------------------ + +IBEX supports element-wise arithmetic between two signals (datasets) during a ``/data/plot_data`` request. +Unlike simple scalar operations which use fixed numeric values, signal operations use **other IMAS signals** as operands. +This enables combining data from different IMAS nodes arithmetically, for example subtracting a background signal from a measurement. + +Signal operations are applied **after data interpolation** and **before downsampling**. +When a signal referenced in a signal operation was not already interpolated via the ``interpolate_over`` parameter, it is interpolated on the fly onto the common coordinate grid during this step. +Operations are applied in the order they appear in the request. + +Configuration +~~~~~~~~~~~~~~ + +Signal operations are configured through the ``signal_operations`` parameter of the ``/data/plot_data/`` endpoint. +It accepts a list of strings in the format ``type:uri``, where ``type`` is one of the supported operations and ``uri`` is the IMAS URI of the operand signal. + +The following operation types are supported: + +* ``add`` — addition +* ``sub`` — subtraction +* ``mul`` — multiplication +* ``div`` — division +* ``pow`` — exponentiation +* ``root`` — Nth root + +Division by zero is rejected by the backend. + +The full list of available operations can be retrieved from the ``/info/data_manipulation_methods/`` endpoint. + +Implementation +~~~~~~~~~~~~~~~ + +Signal operations are applied in ``apply_signal_operations()`` in +``backend/ibex/data_source/imas_python_source_utils.py``. + +The function iterates over the list of operation strings in order, looking up each +operand signal's interpolated data from a dictionary populated during the interpolation step. For each operation the corresponding arithmetic function is applied element-wise between the current result and the operand signal data. + +The implementation handles both flat arrays and nested lists of arrays (higher-dimensional data) recursively. + +Example usage +~~~~~~~~~~~~~~ + +The following examples demonstrate how signal operations can be enabled for testing purposes. + +Addition of two signals: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&signal_operations=add:' \ + -H 'accept: application/json' + +Multiple signal operations (subtraction then multiplication): + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&signal_operations=sub:&signal_operations=mul:' \ + -H 'accept: application/json' From 1455de1d49bcf737bec4d8141b0bc631a5b100b6 Mon Sep 17 00:00:00 2001 From: wasikj Date: Thu, 18 Jun 2026 11:18:40 +0200 Subject: [PATCH 14/29] Delete unused validator --- .../endpoints/schemas/request_data_schemas.py | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 2232f3fd..bea85b54 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -124,44 +124,3 @@ def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": ) return self - - @model_validator(mode="after") - def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": - if self.division_divisor == 0: - raise ValueError("division_divisor cannot be 0") - - signal_uri_lists = [ - self.signal_addition_addend_uri, - self.signal_subtraction_subtrahend_uri, - self.signal_multiplication_factor_uri, - self.signal_division_divisor_uri, - ] - for uri_list in signal_uri_lists: - if uri_list: - for signal_uri in uri_list: - if not self.interpolate_over or signal_uri not in self.interpolate_over: - raise ValueError(f"Signal URI '{signal_uri}' must be listed in interpolate_over") - - signal_priorities = [ - self.signal_addition_priority, - self.signal_subtraction_priority, - self.signal_multiplication_priority, - self.signal_division_priority, - ] - defined_signal_priorities = [p for p in signal_priorities if p is not None] - if len(defined_signal_priorities) != len(set(defined_signal_priorities)): - raise ValueError("signal operation priorities must be unique") - - priorities = [ - self.addition_priority, - self.subtraction_priority, - self.multiplication_priority, - self.division_priority, - self.exponentiation_priority, - self.root_priority, - ] - defined_priorities = [p for p in priorities if p is not None] - if len(defined_priorities) != len(set(defined_priorities)): - raise ValueError("operation priorities must be unique") - - return self From ae27462735884fa1f785acb550e7637b4e1d2e6a Mon Sep 17 00:00:00 2001 From: wasikj Date: Thu, 18 Jun 2026 11:51:59 +0200 Subject: [PATCH 15/29] Fix broken tests. --- backend/ibex/data_source/imas_python_source.py | 2 +- backend/ibex/data_source/imas_python_source_utils.py | 4 ++-- backend/tests/test_data_manipulation.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 2194e434..f9610c56 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -926,7 +926,7 @@ def convert_to_lists(data): if plot_data_query.signal_operations: # Collect set of signal URIs that will be used in signal operations signal_op_uris = set() - for op_str in (plot_data_query.signal_operations or []): + for op_str in plot_data_query.signal_operations or []: _, op_uri = op_str.split(":", 1) signal_op_uris.add(op_uri) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index fd28d447..072125fe 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -68,9 +68,9 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): def _safe_division(data, divisor): if isinstance(divisor, np.ndarray): if np.any(divisor == 0): - raise InvalidParametersException("division_divisor cannot be 0") + raise InvalidParametersException("Division by zero is not allowed") elif divisor == 0: - raise InvalidParametersException("division_divisor cannot be 0") + raise InvalidParametersException("Division by zero is not allowed") return data / divisor diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 2dc81cdb..5b1d1d73 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -149,5 +149,5 @@ def test_apply_signal_operations_recurses_over_lists(): def test_apply_signal_operations_rejects_division_by_zero(): uri = "some/uri" - with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + with pytest.raises(InvalidParametersException, match="Division by zero is not allowed"): apply_signal_operations(np.array([1.0]), [f"div:{uri}"], {uri: np.array([0.0])}) From 93fd74d5b97b2c1479be5f5086dc685313666897 Mon Sep 17 00:00:00 2001 From: wasikj Date: Fri, 19 Jun 2026 14:11:39 +0200 Subject: [PATCH 16/29] Signal operations --- .../ibex/data_source/imas_python_source.py | 78 ++++++++++--- .../data_source/imas_python_source_utils.py | 39 ++++--- backend/tests/conftest.py | 5 +- backend/tests/test_data_endpoints.py | 107 ++++++++++++++++++ .../backend_development/data_manipulation.rst | 70 +++--------- 5 files changed, 208 insertions(+), 91 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index f9610c56..feb3e229 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -822,6 +822,11 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= END data smoothing ============= + try: + original_data_shape = np.asarray(data_to_be_returned).shape + except ValueError: + original_data_shape = "irregular" + # ============= BEGIN resample data onto new time vector ============= def convert_to_lists(data): if isinstance(data, list): @@ -867,10 +872,11 @@ def convert_to_lists(data): if store_other_signals_data: others_signals_data[_uri] = { "uri": _uri, - "data": interpolate_to["value"], + "data": pad_to_rectangular(interpolate_to["value"]), "coordinates": [ sorted(set(flatten(convert_to_lists(c["value"])))) for c in interpolate_to_coordinates ], + "shape": interpolate_to["shape"], } if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): @@ -922,38 +928,76 @@ def convert_to_lists(data): # ============= END resample data onto new time vector ============= # ============= BEGIN signal operations ============= + # + # Steps performed in this block: + # 1. Collect all signal URIs referenced in signal_operations + # 2. Pad data to rectangular if shape is irregular + # 3. For each signal URI, fetch and prepare data: + # a. If the signal was already interpolated (stored during + # interpolation phase), skip fetching + # b. Otherwise fetch the signal and check shape compatibility + # 4. Prepare interpolated_data for each signal: + # a. If interpolation was requested, resample onto common coords + # b. Otherwise use raw signal data directly (same shape path) + # 5. Build a flat uri->array dict and apply all signal operations if plot_data_query.signal_operations: - # Collect set of signal URIs that will be used in signal operations + # Step 1: extract unique signal URIs from operation strings signal_op_uris = set() for op_str in plot_data_query.signal_operations or []: _, op_uri = op_str.split(":", 1) signal_op_uris.add(op_uri) - # For each signal not yet interpolated — interpolate it onto the common coordinate grid + # Step 2: ensure rectangular data for downstream processing + if original_data_shape == "irregular": + data_to_be_returned = pad_to_rectangular(data_to_be_returned) + + # Step 3: fetch and prepare each signal referenced in operations for signal_uri in signal_op_uris: if signal_uri not in others_signals_data: - continue + # Signal was not pre-loaded during interpolation phase. + # Fetch it now and verify shape compatibility. + request = PlotDataRequestModel(uri=signal_uri) + other_signal = self.get_plot_data(request) + if ( + other_signal["data"]["shape"] != "irregular" + and other_signal["data"]["shape"] == original_data_shape + ): + others_signals_data[signal_uri] = { + "uri": request.uri, + "data": other_signal["data"]["value"], + "coordinates": [ + sorted(set(flatten(convert_to_lists(c["value"])))) + for c in other_signal["data"]["coordinates"] + ], + "shape": other_signal["data"]["shape"], + } + else: + msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape does not match. Try interpolating signal onto data's shape." + raise InvalidParametersException(msg) + + # Step 4: prepare interpolated_data (resampled or raw) if "interpolated_data" not in others_signals_data[signal_uri]: - signal_data = pad_to_rectangular(others_signals_data[signal_uri]["data"]) - signal_data = resample_data_without_interpolation( - tuple(others_signals_data[signal_uri]["coordinates"]), - signal_data, - tuple(common_coords_values), - ) - others_signals_data[signal_uri]["interpolated_data"] = signal_data + if plot_data_query.interpolate_over: + # Resample signal data onto the common coordinate grid + signal_data = resample_data_without_interpolation( + tuple(reversed(others_signals_data[signal_uri]["coordinates"])), + others_signals_data[signal_uri]["data"], + tuple(common_coords_values), + ) + else: + # No interpolation needed — use signal data as-is + signal_data = others_signals_data[signal_uri]["data"] + others_signals_data[signal_uri]["interpolated_data"] = np.array(signal_data) - # Apply signal operations (addition, subtraction, multiplication, etc.) + # Step 5: flatten dict and apply operations in order + signal_data_by_uri = {uri: info["interpolated_data"] for uri, info in others_signals_data.items()} data_to_be_returned = apply_signal_operations( - data_to_be_returned, plot_data_query.signal_operations, others_signals_data + data_to_be_returned, plot_data_query.signal_operations, signal_data_by_uri ) # ============= END signal operations ============= - try: - original_data_shape = np.asarray(data_to_be_returned).shape - except ValueError: - original_data_shape = "irregular" # Downsample only 1D data if first_value.metadata.ndim == 1: if coordinates_to_be_returned[0]["target"].split("/")[-1] == f"{node_path.split('/')[-1]}": diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 072125fe..da439ffb 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -127,9 +127,18 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign "root": lambda r, v: np.power(r, 1 / v), } + _NULL_REPLACEMENT = { + "add": 0, + "sub": 0, + "mul": 1, + "div": 1, + "pow": 1, + "root": 1, + } + if isinstance(data, list): - return [apply_signal_operations(x, operations, signal_data_by_uri) for x in data] - elif isinstance(data, (np.ndarray, IDSNumericArray)): + data = np.array(data) + if isinstance(data, (np.ndarray, IDSNumericArray)): result = data for op_str in operations: op_type, uri = op_str.split(":", 1) @@ -137,6 +146,15 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign func = _OP_FUNCS.get(op_type) if func is None: raise InvalidParametersException(f"Unknown operation type: {op_type}") + + # Replace NaN/null values with operation-specific neutral value + if isinstance(value, np.ndarray): + mask = np.isnan(value) + if np.any(mask): + value = np.where(mask, _NULL_REPLACEMENT.get(op_type, 0), value) + elif isinstance(value, float) and np.isnan(value): + value = _NULL_REPLACEMENT.get(op_type, 0) + result = func(result, value) return result else: @@ -165,14 +183,14 @@ def calculate_coordinate_shapes(shape: list[int], dims: int, n_coords: int): :param shape: Full shape of the data array (e.g. [4, 5, 10, 15]). :param dims: Number of base dimensions extracted from Data Dictionary (e.g. 2 for a 2D grid -> [4, 5]). :param n_coords: Number of coordinate dimensions generated by AoS presence in node path (e.g. [10, 15]). - :return: List of shapes for each coordinate. Includes: + :return: List of shapes for each coordinate. :raises ValueError: If ``len(shape) != dims + n_coords``. """ - if dims < 0 or dims >= len(shape): - raise ValueError("dims must be >= 0 and < len(shape)") + if dims < 0 or dims > len(shape): + raise ValueError("dims must be >= 0 and <= len(shape)") # Base dimensions (dimensions added by AoS in path), e.g. [4, 5] base = shape[:dims] @@ -312,16 +330,7 @@ def resample_data_without_interpolation(original_coords, data, target_coords): target_indices = [] for orig, target in zip(original_coords, target_coords): - # Build dictionary: coordinate -> target index - lookup = {} - for i, value in enumerate(target): - lookup[value] = i - - axis_indices = [] - - for value in orig: - axis_indices.append(lookup[value]) - + axis_indices = np.searchsorted(np.asarray(target), np.asarray(orig)) target_indices.append(axis_indices) # Create mesh diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index bdfcaeab..9e9c9737 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -11,6 +11,7 @@ @pytest.fixture(scope="session") def interpolation_entry_path_directory(tmp_path_factory): tmp_path = tmp_path_factory.mktemp("interpolation_testdb") + rand_generator = np.random.default_rng(2137) with imas.DBEntry(f"imas:hdf5?path={tmp_path}/interpolation_db_1", mode="w") as entry: eq = entry.factory.equilibrium() @@ -21,7 +22,7 @@ def interpolation_entry_path_directory(tmp_path_factory): for ts in eq.time_slice: ts.profiles_2d.resize(2) for p2d in ts.profiles_2d: - p2d.psi = np.asarray(np.random.rand(3, 3)) + p2d.psi = np.asarray(rand_generator.random((3, 3))) p2d.grid.dim1 = np.asarray([1.0, 2.0, 3.0]) p2d.grid.dim2 = np.asarray([1.0, 2.0, 3.0]) entry.put(eq) @@ -35,7 +36,7 @@ def interpolation_entry_path_directory(tmp_path_factory): for ts in eq.time_slice: ts.profiles_2d.resize(4) for p2d in ts.profiles_2d: - p2d.psi = np.asarray(np.random.rand(9, 3)) + p2d.psi = np.asarray(rand_generator.random((9, 3))) p2d.grid.dim1 = np.asarray([0.3, 0.6, 0.9, 1.2, 1.5, 1.8, 2.1, 2.4, 2.7]) p2d.grid.dim2 = np.asarray([1.0, 2.0, 3.0]) entry.put(eq) diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 791a325e..17a903c3 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -1,4 +1,5 @@ import pytest +import numpy as np def test_status_codes(entry_path): @@ -130,6 +131,112 @@ def test_plot_data_with_simple_operations(entry_path): assert response_body["data"]["value"] == pytest.approx(expected) +def test_plot_data_with_signal_operations(entry_path): + parameters = { + "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", + "signal_operations": [f"add:imas:hdf5?path={entry_path}#core_profiles/global_quantities/ip"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, 8.0, 10.0]) + + +def test_plot_data_with_signal_operations_same_shape_different_uris(interpolation_entry_path_directory): + db_names = [ + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", + ] + + parameters = { + "uri": f"{db_names[0]}#equilibrium/time_slice[0:2]/profiles_2d[0]/grid/dim2", + "signal_operations": [f"add:{db_names[1]}#equilibrium/time_slice[0:2]/profiles_2d[0]/grid/dim2"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == [[2.0, 4.0, 6.0], [2.0, 4.0, 6.0]] + + +def test_plot_data_with_signal_operations_and_interpolation(interpolation_entry_path_directory): + db_names = [ + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", + ] + + parameters = { + "uri": f"{db_names[0]}#equilibrium/time", + "signal_operations": [f"add:{db_names[1]}#equilibrium/time"], + "interpolate_over": [f"{db_names[1]}#equilibrium/time"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + # time1: [1, 2, 3, 4] + # time2: [1, 2, 3] + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, 4.0]) + + # reversed order + parameters = { + "uri": f"{db_names[1]}#equilibrium/time", + "signal_operations": [f"add:{db_names[0]}#equilibrium/time"], + "interpolate_over": [f"{db_names[0]}#equilibrium/time"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + # time1: [1, 2, 3] + # time2: [1, 2, 3, 4] + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, None]) + + +def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_entry_path_directory): + db_names = [ + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", + ] + + # ---- db_1 primary, db_2 operand ---- + # common coords: time=[1,2,3,4] (4), profiles_2d=[0,1,2,3] (4), + # dim2=[1,2,3] (3), dim1=12 values (union of both) + parameters = { + "uri": f"{db_names[0]}#equilibrium/time_slice[:]/profiles_2d[:]/psi", + "signal_operations": [f"add:{db_names[1]}#equilibrium/time_slice[:]/profiles_2d[:]/psi"], + "interpolate_over": [f"{db_names[1]}#equilibrium/time_slice[:]/profiles_2d[:]/psi"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + response_body = response.json() + + data = np.array(response_body["data"]["value"], dtype=float) + # data shape reflects common coordinates (reversed): [time, profiles_2d, dim2, dim1] + assert data.shape == (4, 4, 3, 12) + # db_1 has data at time=[1,2,3,4], profiles_2d=[0,1], dim2=[1,2,3], dim1=[1,2,3] + # Common dim1 has 1,2,3 at indices 3,7,11 → 4×2×3×3 = 72 non-NaN values + # Operand NaN replaced with 0, result = db_1 primary values + assert np.count_nonzero(~np.isnan(data)) == 72 + + # ---- reversed: db_2 primary, db_1 operand ---- + parameters = { + "uri": f"{db_names[1]}#equilibrium/time_slice[:]/profiles_2d[:]/psi", + "signal_operations": [f"add:{db_names[0]}#equilibrium/time_slice[:]/profiles_2d[:]/psi"], + "interpolate_over": [f"{db_names[0]}#equilibrium/time_slice[:]/profiles_2d[:]/psi"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + response_body = response.json() + + data = np.array(response_body["data"]["value"], dtype=float) + assert data.shape == (4, 4, 3, 12) + # db_2 has data at time=[1,2,3], profiles_2d=[0,1,2,3], dim2=[1,2,3], dim1 has 9 values + # Common dim1 has db_2's 9 values at indices [0,1,2,4,5,6,8,9,10] → 3×4×3×9 = 324 + assert np.count_nonzero(~np.isnan(data)) == 324 + + def test_plot_data_smoothing_with_wrong_target_node(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", # targetet quantity must be time-based diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index d2977d84..4b4d44fe 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -231,65 +231,21 @@ Multiple signal operations (subtraction then multiplication): '/data/plot_data?uri=&signal_operations=sub:&signal_operations=mul:' \ -H 'accept: application/json' +.. _`Signal operations details`: -Signal operations ------------------- -IBEX supports element-wise arithmetic between two signals (datasets) during a ``/data/plot_data`` request. -Unlike simple scalar operations which use fixed numeric values, signal operations use **other IMAS signals** as operands. -This enables combining data from different IMAS nodes arithmetically, for example subtracting a background signal from a measurement. +Null and NaN handling +~~~~~~~~~~~~~~~~~~~~~~ -Signal operations are applied **after data interpolation** and **before downsampling**. -When a signal referenced in a signal operation was not already interpolated via the ``interpolate_over`` parameter, it is interpolated on the fly onto the common coordinate grid during this step. -Operations are applied in the order they appear in the request. +Before applying each operation, the backend inspects the operand signal's data for NaN values. +If any NaN values are found, they are replaced with an **operation-specific neutral value** +that will not affect the result of the operation: -Configuration -~~~~~~~~~~~~~~ - -Signal operations are configured through the ``signal_operations`` parameter of the ``/data/plot_data/`` endpoint. -It accepts a list of strings in the format ``type:uri``, where ``type`` is one of the supported operations and ``uri`` is the IMAS URI of the operand signal. - -The following operation types are supported: - -* ``add`` — addition -* ``sub`` — subtraction -* ``mul`` — multiplication -* ``div`` — division -* ``pow`` — exponentiation -* ``root`` — Nth root - -Division by zero is rejected by the backend. - -The full list of available operations can be retrieved from the ``/info/data_manipulation_methods/`` endpoint. - -Implementation -~~~~~~~~~~~~~~~ +* ``add`` — neutral value ``0`` +* ``sub`` — neutral value ``0`` +* ``mul`` — neutral value ``1`` +* ``div`` — neutral value ``1`` +* ``pow`` — neutral value ``1`` +* ``root`` — neutral value ``1`` -Signal operations are applied in ``apply_signal_operations()`` in -``backend/ibex/data_source/imas_python_source_utils.py``. - -The function iterates over the list of operation strings in order, looking up each -operand signal's interpolated data from a dictionary populated during the interpolation step. For each operation the corresponding arithmetic function is applied element-wise between the current result and the operand signal data. - -The implementation handles both flat arrays and nested lists of arrays (higher-dimensional data) recursively. - -Example usage -~~~~~~~~~~~~~~ - -The following examples demonstrate how signal operations can be enabled for testing purposes. - -Addition of two signals: - -.. code-block:: bash - - curl -X 'GET' \ - '/data/plot_data?uri=&signal_operations=add:' \ - -H 'accept: application/json' - -Multiple signal operations (subtraction then multiplication): - -.. code-block:: bash - - curl -X 'GET' \ - '/data/plot_data?uri=&signal_operations=sub:&signal_operations=mul:' \ - -H 'accept: application/json' +This prevents NaN propagation through chained operations. From 3b8f226292c74b18586b2dbbcb6385088bfbca05 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 22 Jun 2026 08:42:24 +0200 Subject: [PATCH 17/29] Update tests. Fix bug. --- .../ibex/data_source/imas_python_source.py | 7 +-- backend/tests/conftest.py | 4 ++ backend/tests/test_data_endpoints.py | 55 +++++++++++++++++-- backend/tests/test_data_manipulation.py | 2 +- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index feb3e229..51468d3e 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -5,7 +5,6 @@ import imas # type: ignore import numpy as np # type: ignore import re # type: ignore -from copy import copy # type: ignore from idstools.database import DBMaster # type: ignore from imas.ids_metadata import IDSMetadata # type: ignore from imas.ids_primitive import ( @@ -861,11 +860,7 @@ def convert_to_lists(data): "IDS name and node path should be the same for source and target URI when interpolating data" ) - new_plot_data_query = copy(plot_data_query) - new_plot_data_query.uri = _uri - new_plot_data_query.interpolate_over = None - new_plot_data_query.smoothing_method = None - new_plot_data_query.signal_operations = None + new_plot_data_query = PlotDataRequestModel(uri=_uri) # interpolate_to will be used later with signal combining interpolate_to = self.get_plot_data(new_plot_data_query)["data"] interpolate_to_coordinates = interpolate_to["coordinates"] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 9e9c9737..76009454 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -18,6 +18,8 @@ def interpolation_entry_path_directory(tmp_path_factory): eq.ids_properties.homogeneous_time = 1 eq.time = np.asarray([1, 2, 3, 4], dtype=float) + eq.vacuum_toroidal_field.r0 = 1.0 + eq.vacuum_toroidal_field.b0 = np.asarray([0.1, 0.2, 0.3, 0.4], dtype=float) eq.time_slice.resize(4) for ts in eq.time_slice: ts.profiles_2d.resize(2) @@ -32,6 +34,8 @@ def interpolation_entry_path_directory(tmp_path_factory): eq.ids_properties.homogeneous_time = 1 eq.time = np.asarray([1, 2, 3], dtype=float) + eq.vacuum_toroidal_field.r0 = 1.0 + eq.vacuum_toroidal_field.b0 = np.asarray([0.1, 0.2, 0.3], dtype=float) eq.time_slice.resize(3) for ts in eq.time_slice: ts.profiles_2d.resize(4) diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 17a903c3..5aa790b2 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -200,9 +200,6 @@ def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_ent f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", ] - # ---- db_1 primary, db_2 operand ---- - # common coords: time=[1,2,3,4] (4), profiles_2d=[0,1,2,3] (4), - # dim2=[1,2,3] (3), dim1=12 values (union of both) parameters = { "uri": f"{db_names[0]}#equilibrium/time_slice[:]/profiles_2d[:]/psi", "signal_operations": [f"add:{db_names[1]}#equilibrium/time_slice[:]/profiles_2d[:]/psi"], @@ -239,7 +236,7 @@ def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_ent def test_plot_data_smoothing_with_wrong_target_node(entry_path): parameters = { - "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", # targetet quantity must be time-based + "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", # targeted quantity must be time-based "smoothing_method": "gaussian_filter", "gaussian_smoothing_sigma": 1, } @@ -313,6 +310,56 @@ def test_plot_data_requires_gaussian_sigma(entry_path): assert "gaussian_smoothing_sigma is required" in response.text +def test_combined_features(entry_path, interpolation_entry_path_directory): + """ + Single test exercising all data manipulation features: + simple operations, smoothing, interpolation (exact_value), and signal operations. + """ + # --- Part 1: simple ops + gaussian smoothing + signal ops (entry_path) --- + db = f"imas:hdf5?path={entry_path}" + parameters = { + "uri": f"{db}#core_profiles/global_quantities/ip", + "operations": ["add:2", "mul:3"], + "smoothing_method": "gaussian_filter", + "gaussian_smoothing_sigma": 1, + "signal_operations": [f"add:{db}#core_profiles/time"], + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx([11.28, 14.20, 18.0, 21.80, 24.72], 0.1) + + # --- Part 2: different simple ops + savgol smoothing + exact_value interpolation + signal ops --- + db_names = [ + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", + ] + + parameters = { + "uri": f"{db_names[0]}#equilibrium/vacuum_toroidal_field/b0", + "operations": ["mul:10", "pow:2"], + "smoothing_method": "savitzky-golay_filter", + "savgol_smoothing_window_length": 3, + "savgol_smoothing_polyorder": 1, + "signal_operations": [f"add:{db_names[1]}#equilibrium/vacuum_toroidal_field/b0"], + "interpolate_over": [f"{db_names[1]}#equilibrium/vacuum_toroidal_field/b0"], + "interpolation_method": "exact_value", + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + # db_1 b0: [0.1,0.2,0.3,0.4], mul:10+pow:2 -> [1,4,9,16] + # savgol wl=3 po=1 -> [0.6667,4.6667,9.6667,15.6667] + # exact_value interpolation on union [1,2,3,4] -> no change + # db_2 b0: [0.1,0.2,0.3] + # resampled to [1,2,3,4] with exact_value: [0.1,0.2,0.3,None->0] + # signal add: [0.6667+0.1, 4.6667+0.2, 9.6667+0.3, 15.6667+0] + # Result: [0.7667,4.8667,9.9667,15.6667] + assert response_body["data"]["value"] == pytest.approx([0.76, 4.86, 9.96, 15.66], 0.01) + + def test_plot_data_requires_savgol_window_length_and_polyorder(entry_path): base_parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/profiles_1d[:]/time", diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 5b1d1d73..2960292f 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -138,7 +138,7 @@ def test_apply_signal_operations_uses_order(): assert result == pytest.approx([12.0]) -def test_apply_signal_operations_recurses_over_lists(): +def test_apply_signal_operations_2D(): uri = "some/uri" data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] signal_data_by_uri = {uri: np.array([2.0, 3.0])} From 2499794e0a065a198c3c75b581753ec0e5f99c01 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 22 Jun 2026 08:54:13 +0200 Subject: [PATCH 18/29] Delete unused operations --- backend/ibex/core/data_manipulation_methods.py | 2 -- backend/ibex/data_source/imas_python_source_utils.py | 4 ---- 2 files changed, 6 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 2ba58f04..2954fd15 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -249,8 +249,6 @@ class DataManipulationMethodsResponse(BaseModel): PossibleValue(value="sub", description="Subtraction"), PossibleValue(value="mul", description="Multiplication"), PossibleValue(value="div", description="Division"), - PossibleValue(value="pow", description="Exponentiation"), - PossibleValue(value="root", description="Nth root"), ], ), DataManipulationParameter( diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index da439ffb..59c803dc 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -123,8 +123,6 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign "sub": lambda r, v: r - v, "mul": lambda r, v: r * v, "div": lambda r, v: _safe_division(r, v), - "pow": lambda r, v: np.power(r, v), - "root": lambda r, v: np.power(r, 1 / v), } _NULL_REPLACEMENT = { @@ -132,8 +130,6 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign "sub": 0, "mul": 1, "div": 1, - "pow": 1, - "root": 1, } if isinstance(data, list): From 56d30409d7a6141d6a2b2a56205735f168c45b42 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 30 Jun 2026 11:50:16 +0200 Subject: [PATCH 19/29] Apply linter. Fix bugs. --- .../data_source/imas_python_source_utils.py | 90 ++++++------------- backend/pyproject.toml | 6 +- backend/tests/test_data_endpoints.py | 6 +- 3 files changed, 31 insertions(+), 71 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index a5b6d1bf..abc8eda0 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -95,18 +95,21 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma, axis: int | None = Non raise InvalidParametersException(msg) -def _safe_division(data, divisor): - if divisor == 0: - raise InvalidParametersException("Division by zero is not allowed") - return data / divisor - - def _safe_root(data, exponent): if exponent == 0: raise InvalidParametersException("Root by zero is not allowed") return np.power(data, 1 / exponent) +def _safe_division(data, divisor): + if isinstance(divisor, np.ndarray): + if np.any(divisor == 0): + raise InvalidParametersException("Division by zero is not allowed") + elif divisor == 0: + raise InvalidParametersException("Division by zero is not allowed") + return data / divisor + + _SIMPLE_OPERATIONS_FUNCTIONS = { "add": _op.add, "sub": _op.sub, @@ -116,39 +119,19 @@ def _safe_root(data, exponent): "root": _safe_root, } +_SIGNAL_OPERATIONS_FUNCTIONS = { + "add": _op.add, + "sub": _op.sub, + "mul": _op.mul, + "div": _safe_division, +} -def apply_simple_operations(data: list | np.ndarray, operations: list[str]): - """ - Apply simple scalar operations to data in the order given. - Each operation is a string in the format 'type:value', e.g. 'add:10', 'mul:5'. - :param data: Input data - :param operations: List of operations and operands divided by colon (:) - :return: Data after operation - """ - if isinstance(data, list): - return [apply_simple_operations(x, operations) for x in data] - elif isinstance(data, (np.ndarray, IDSNumericArray)): - result = data - for op_str in operations: - op_type, value_str = op_str.split(":", 1) - value = float(value_str) - func = _SIMPLE_OPERATIONS_FUNCTIONS.get(op_type) - if func is None: - raise InvalidParametersException(f"Unknown operation type: {op_type}") - result = func(result, value) - return result - else: - msg = "Simple operations can be executed only on numeric arrays, not single values or strings." - raise InvalidParametersException(msg) - - -def _safe_division(data, divisor): - if isinstance(divisor, np.ndarray): - if np.any(divisor == 0): - raise InvalidParametersException("Division by zero is not allowed") - elif divisor == 0: - raise InvalidParametersException("Division by zero is not allowed") - return data / divisor +_SIGNAL_NULL_REPLACEMENT = { + "add": 0, + "sub": 0, + "mul": 1, + "div": 1, +} def apply_simple_operations(data: list | np.ndarray, operations: list[str]): @@ -159,16 +142,6 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): :param operations: List of operations and operands divided by colon (:) :return: Data after operation """ - - _OP_FUNCS = { - "add": lambda r, v: r + v, - "sub": lambda r, v: r - v, - "mul": lambda r, v: r * v, - "div": lambda r, v: _safe_division(r, v), - "pow": lambda r, v: np.power(r, v), - "root": lambda r, v: np.power(r, 1 / v), - } - if isinstance(data, list): return [apply_simple_operations(x, operations) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): @@ -176,7 +149,7 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): for op_str in operations: op_type, value_str = op_str.split(":", 1) value = float(value_str) - func = _OP_FUNCS.get(op_type) + func = _SIMPLE_OPERATIONS_FUNCTIONS.get(op_type) if func is None: raise InvalidParametersException(f"Unknown operation type: {op_type}") result = func(result, value) @@ -195,19 +168,6 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign :param signal_data_by_uri: Dict mapping signal URIs to their interpolated data arrays. :return: Data after operation """ - _OP_FUNCS = { - "add": lambda r, v: r + v, - "sub": lambda r, v: r - v, - "mul": lambda r, v: r * v, - "div": lambda r, v: _safe_division(r, v), - } - - _NULL_REPLACEMENT = { - "add": 0, - "sub": 0, - "mul": 1, - "div": 1, - } if isinstance(data, list): data = np.array(data) @@ -216,7 +176,7 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign for op_str in operations: op_type, uri = op_str.split(":", 1) value = signal_data_by_uri[uri] - func = _OP_FUNCS.get(op_type) + func = _SIGNAL_OPERATIONS_FUNCTIONS.get(op_type) if func is None: raise InvalidParametersException(f"Unknown operation type: {op_type}") @@ -224,9 +184,9 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign if isinstance(value, np.ndarray): mask = np.isnan(value) if np.any(mask): - value = np.where(mask, _NULL_REPLACEMENT.get(op_type, 0), value) + value = np.where(mask, _SIGNAL_NULL_REPLACEMENT.get(op_type, 0), value) elif isinstance(value, float) and np.isnan(value): - value = _NULL_REPLACEMENT.get(op_type, 0) + value = _SIGNAL_NULL_REPLACEMENT.get(op_type, 0) result = func(result, value) return result diff --git a/backend/pyproject.toml b/backend/pyproject.toml index bb02a7b4..715d9f85 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -57,9 +57,9 @@ dependencies = [ [project.optional-dependencies] # these self-dependencies are available since pip 21.2 all = [ - "ibex[test]", - "ibex[linting]", - "ibex[docs]", + "imas-ibex[test]", + "imas-ibex[linting]", + "imas-ibex[docs]", ] test = [ diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 2434fde9..b54f22b6 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -371,6 +371,7 @@ def test_plot_data_coordinate_aliases(entry_path, expected_unit): assert dim2_coordinate["name"].lower() == "theta" assert dim2_coordinate["unit"].lower() == expected_unit[1] + def test_plot_data_with_signal_operations(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", @@ -453,7 +454,7 @@ def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_ent # data shape reflects common coordinates (reversed): [time, profiles_2d, dim2, dim1] assert data.shape == (4, 4, 3, 12) # db_1 has data at time=[1,2,3,4], profiles_2d=[0,1], dim2=[1,2,3], dim1=[1,2,3] - # Common dim1 has 1,2,3 at indices 3,7,11 → 4×2×3×3 = 72 non-NaN values + # Common dim1 has 1,2,3 at indices 3,7,11 → 4*2*3*3 = 72 non-NaN values # Operand NaN replaced with 0, result = db_1 primary values assert np.count_nonzero(~np.isnan(data)) == 72 @@ -470,6 +471,5 @@ def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_ent data = np.array(response_body["data"]["value"], dtype=float) assert data.shape == (4, 4, 3, 12) # db_2 has data at time=[1,2,3], profiles_2d=[0,1,2,3], dim2=[1,2,3], dim1 has 9 values - # Common dim1 has db_2's 9 values at indices [0,1,2,4,5,6,8,9,10] → 3×4×3×9 = 324 + # Common dim1 has db_2's 9 values at indices [0,1,2,4,5,6,8,9,10] → 3*4*3*9 = 324 assert np.count_nonzero(~np.isnan(data)) == 324 - From 8cc64a4af318ff3ea6c5aee1c11310570a0ec7a0 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 30 Jun 2026 12:01:41 +0200 Subject: [PATCH 20/29] Add validator for signal operations. --- .../endpoints/schemas/request_data_schemas.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 93e04b99..f6eb0475 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field, model_validator from typing import Optional, List from ibex.core.data_manipulation_methods import available_methods, SmoothingMethod -from ibex.data_source.imas_python_source_utils import _SIMPLE_OPERATIONS_FUNCTIONS +from ibex.data_source.imas_python_source_utils import _SIMPLE_OPERATIONS_FUNCTIONS, _SIGNAL_OPERATIONS_FUNCTIONS from enum import Enum @@ -126,6 +126,22 @@ def validate_operations_format(self) -> "PlotDataRequestModel": raise ValueError(f"Invalid operation value: '{value_str}' in '{op}'. Value must be a number.") return self + @model_validator(mode="after") + def validate_signal_operations_format(self) -> "PlotDataRequestModel": + if self.signal_operations: + valid_operations = set(_SIGNAL_OPERATIONS_FUNCTIONS.keys()) + for op in self.signal_operations: + if ":" not in op: + raise ValueError(f"Invalid signal operation format: '{op}'. Expected 'operation:uri'") + op_type, uri = op.split(":", 1) + if op_type not in valid_operations: + raise ValueError( + f"Unknown signal operation type: '{op_type}'. Valid types: {', '.join(sorted(valid_operations))}" + ) + if not uri.strip(): + raise ValueError(f"Invalid signal operation: '{op}'. URI must not be empty.") + return self + @model_validator(mode="after") def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": if self.smoothing_method == SmoothingMethod.GAUSSIAN_FILTER and self.gaussian_smoothing_sigma is None: From 3d05817c5e57faa9a17b8281e867a245f812c36e Mon Sep 17 00:00:00 2001 From: Bartek Palak Date: Mon, 20 Jul 2026 13:50:40 +0200 Subject: [PATCH 21/29] Fix plot data shape after interpolation --- backend/ibex/data_source/imas_python_source.py | 11 ++++++++++- backend/tests/test_data_endpoints.py | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 7f0d1733..4a2a1842 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -1286,6 +1286,15 @@ def convert_to_lists(data): # ============= END signal operations ============= + # Interpolation and signal operations can change the data shape. + # The response's ``shape`` describes the processed data before + # downsampling, rather than the raw shape used for compatibility + # checks above. + try: + processed_data_shape = np.asarray(data_to_be_returned).shape + except ValueError: + processed_data_shape = "irregular" + # Downsample only 1D data if first_value.metadata.ndim == 1: if coordinates_to_be_returned[0]["target"].split("/")[-1] == f"{node_path.split('/')[-1]}": @@ -1318,7 +1327,7 @@ def convert_to_lists(data): "data": { "name": node_path.split("/")[-1], "unit": first_value.metadata.units, - "shape": original_data_shape, + "shape": processed_data_shape, "downsampled_shape": downsampled_shape, "ndim": first_value.metadata.ndim, "path": f"#{ids}/{node_path}", diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index b54f22b6..581e7656 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -419,6 +419,8 @@ def test_plot_data_with_signal_operations_and_interpolation(interpolation_entry_ # time2: [1, 2, 3] response_body = response.json() assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, 4.0]) + assert response_body["data"]["shape"] == [4] + assert response_body["data"]["downsampled_shape"] == [4] # reversed order parameters = { @@ -433,6 +435,9 @@ def test_plot_data_with_signal_operations_and_interpolation(interpolation_entry_ # time2: [1, 2, 3, 4] response_body = response.json() assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, None]) + # Interpolation expands the source from three to four samples. + assert response_body["data"]["shape"] == [4] + assert response_body["data"]["downsampled_shape"] == [4] def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_entry_path_directory): From e4f6f2c0d0e84c5ce527d47a3d1226f5a1799ca6 Mon Sep 17 00:00:00 2001 From: Bartek Palak Date: Mon, 20 Jul 2026 14:10:16 +0200 Subject: [PATCH 22/29] Preserve NaN values in signal operations --- .../data_source/imas_python_source_utils.py | 16 ------------- backend/tests/test_data_endpoints.py | 24 +++++++++---------- backend/tests/test_data_manipulation.py | 8 +++++++ 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index abc8eda0..ef357f54 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -126,14 +126,6 @@ def _safe_division(data, divisor): "div": _safe_division, } -_SIGNAL_NULL_REPLACEMENT = { - "add": 0, - "sub": 0, - "mul": 1, - "div": 1, -} - - def apply_simple_operations(data: list | np.ndarray, operations: list[str]): """ Apply simple scalar operations to data in the order given. @@ -180,14 +172,6 @@ def apply_signal_operations(data: list | np.ndarray, operations: list[str], sign if func is None: raise InvalidParametersException(f"Unknown operation type: {op_type}") - # Replace NaN/null values with operation-specific neutral value - if isinstance(value, np.ndarray): - mask = np.isnan(value) - if np.any(mask): - value = np.where(mask, _SIGNAL_NULL_REPLACEMENT.get(op_type, 0), value) - elif isinstance(value, float) and np.isnan(value): - value = _SIGNAL_NULL_REPLACEMENT.get(op_type, 0) - result = func(result, value) return result else: diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 581e7656..507333ca 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -316,10 +316,11 @@ def test_combined_features(entry_path, interpolation_entry_path_directory): # savgol wl=3 po=1 -> [0.6667,4.6667,9.6667,15.6667] # exact_value interpolation on union [1,2,3,4] -> no change # db_2 b0: [0.1,0.2,0.3] - # resampled to [1,2,3,4] with exact_value: [0.1,0.2,0.3,None->0] - # signal add: [0.6667+0.1, 4.6667+0.2, 9.6667+0.3, 15.6667+0] - # Result: [0.7667,4.8667,9.9667,15.6667] - assert response_body["data"]["value"] == pytest.approx([0.76, 4.86, 9.96, 15.66], 0.01) + # resampled to [1,2,3,4] with exact_value: [0.1,0.2,0.3,None] + # signal add propagates missing operand data: [0.7667,4.8667,9.9667,None] + values = response_body["data"]["value"] + assert values[:3] == pytest.approx([0.76, 4.86, 9.96], 0.01) + assert values[3] is None def test_plot_data_requires_savgol_window_length_and_polyorder(entry_path): @@ -418,7 +419,7 @@ def test_plot_data_with_signal_operations_and_interpolation(interpolation_entry_ # time1: [1, 2, 3, 4] # time2: [1, 2, 3] response_body = response.json() - assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, 4.0]) + assert response_body["data"]["value"] == [2.0, 4.0, 6.0, None] assert response_body["data"]["shape"] == [4] assert response_body["data"]["downsampled_shape"] == [4] @@ -434,7 +435,7 @@ def test_plot_data_with_signal_operations_and_interpolation(interpolation_entry_ # time1: [1, 2, 3] # time2: [1, 2, 3, 4] response_body = response.json() - assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, None]) + assert response_body["data"]["value"] == [2.0, 4.0, 6.0, None] # Interpolation expands the source from three to four samples. assert response_body["data"]["shape"] == [4] assert response_body["data"]["downsampled_shape"] == [4] @@ -458,10 +459,8 @@ def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_ent data = np.array(response_body["data"]["value"], dtype=float) # data shape reflects common coordinates (reversed): [time, profiles_2d, dim2, dim1] assert data.shape == (4, 4, 3, 12) - # db_1 has data at time=[1,2,3,4], profiles_2d=[0,1], dim2=[1,2,3], dim1=[1,2,3] - # Common dim1 has 1,2,3 at indices 3,7,11 → 4*2*3*3 = 72 non-NaN values - # Operand NaN replaced with 0, result = db_1 primary values - assert np.count_nonzero(~np.isnan(data)) == 72 + # db_1 and db_2 have disjoint valid dim1 locations after interpolation, so operand NaNs propagate. + assert np.count_nonzero(~np.isnan(data)) == 0 # ---- reversed: db_2 primary, db_1 operand ---- parameters = { @@ -475,6 +474,5 @@ def test_plot_data_with_signal_operations_and_interpolation_2d(interpolation_ent data = np.array(response_body["data"]["value"], dtype=float) assert data.shape == (4, 4, 3, 12) - # db_2 has data at time=[1,2,3], profiles_2d=[0,1,2,3], dim2=[1,2,3], dim1 has 9 values - # Common dim1 has db_2's 9 values at indices [0,1,2,4,5,6,8,9,10] → 3*4*3*9 = 324 - assert np.count_nonzero(~np.isnan(data)) == 324 + # db_1 and db_2 have disjoint valid dim1 locations after interpolation, so operand NaNs propagate. + assert np.count_nonzero(~np.isnan(data)) == 0 diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 3e067c75..9768146f 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -45,6 +45,14 @@ def test_apply_signal_operations_division(): assert np.allclose(result, [5.0, 4.0, 5.0]) +def test_apply_signal_operations_preserves_operand_nans(): + operand_uri = "imas:hdf5?path=/dummy/interpolation_db_1#equilibrium/time_slice[:]/profiles_2d[:]/psi" + data = np.array([10.0, 20.0, 30.0]) + signal_data_by_uri = {operand_uri: np.array([1.0, np.nan, 3.0])} + result = apply_signal_operations(data, [f"add:{operand_uri}"], signal_data_by_uri) + assert np.allclose(result, [11.0, np.nan, 33.0], equal_nan=True) + + def test_apply_gaussian_smoothing(): data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1]) From ad0a37bc908aed22417349c122b198a879622993 Mon Sep 17 00:00:00 2001 From: Bartek Palak Date: Mon, 20 Jul 2026 15:20:37 +0200 Subject: [PATCH 23/29] Propagate units through signal operations --- .../ibex/data_source/imas_python_source.py | 13 +++++++++- .../data_source/imas_python_source_utils.py | 24 +++++++++++++++++++ backend/tests/test_data_endpoints.py | 12 ++++++++++ backend/tests/test_data_manipulation.py | 16 +++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 4a2a1842..b14b919c 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -51,6 +51,7 @@ apply_gaussian_filter, apply_simple_operations, apply_signal_operations, + combine_signal_units, ) from ibex.core.data_manipulation_methods import SmoothingMethod, InterpolationMethod from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel @@ -1054,6 +1055,7 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: } coordinates_to_be_returned.append(c) first_value = find_first_value_in_list(ids_data) + result_unit = first_value.metadata.units or "" data_to_be_returned = convert_ids_data_into_numpy_array(ids_data) if first_value.metadata.ndim == 2: @@ -1169,6 +1171,7 @@ def convert_to_lists(data): sorted(set(flatten(convert_to_lists(c["value"])))) for c in interpolate_to_coordinates ], "shape": interpolate_to["shape"], + "unit": interpolate_to["unit"], } if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): @@ -1259,6 +1262,7 @@ def convert_to_lists(data): for c in other_signal["data"]["coordinates"] ], "shape": other_signal["data"]["shape"], + "unit": other_signal["data"]["unit"], } else: msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape does not match. Try interpolating signal onto data's shape." @@ -1283,6 +1287,13 @@ def convert_to_lists(data): data_to_be_returned = apply_signal_operations( data_to_be_returned, plot_data_query.signal_operations, signal_data_by_uri ) + for operation in plot_data_query.signal_operations: + operation_type, signal_uri = operation.split(":", 1) + result_unit = combine_signal_units( + result_unit, + others_signals_data[signal_uri]["unit"], + operation_type, + ) # ============= END signal operations ============= @@ -1326,7 +1337,7 @@ def convert_to_lists(data): result = { "data": { "name": node_path.split("/")[-1], - "unit": first_value.metadata.units, + "unit": result_unit, "shape": processed_data_shape, "downsampled_shape": downsampled_shape, "ndim": first_value.metadata.ndim, diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index ef357f54..fab79391 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -126,6 +126,30 @@ def _safe_division(data, divisor): "div": _safe_division, } + +def combine_signal_units(left_unit: str, right_unit: str, operation: str) -> str: + """Return the unit produced by a binary signal operation.""" + left_unit = left_unit or "" + right_unit = right_unit or "" + + if operation in {"add", "sub"}: + return left_unit + if operation == "mul": + if not left_unit: + return right_unit + if not right_unit: + return left_unit + return f"{left_unit}*{right_unit}" + if operation == "div": + if left_unit == right_unit: + return "" + if not right_unit: + return left_unit + return f"{left_unit}/{right_unit}" + + raise InvalidParametersException(f"Unknown operation type: {operation}") + + def apply_simple_operations(data: list | np.ndarray, operations: list[str]): """ Apply simple scalar operations to data in the order given. diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 507333ca..d7f0b652 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -385,6 +385,18 @@ def test_plot_data_with_signal_operations(entry_path): assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, 8.0, 10.0]) +@pytest.mark.parametrize(("operation", "expected_unit"), [("mul", "s*s"), ("div", "")]) +def test_plot_data_with_signal_operations_updates_unit(entry_path, operation, expected_unit): + uri = f"imas:hdf5?path={entry_path}#core_profiles/time" + response = pytest.test_client.get( + "/data/plot_data", + params={"uri": uri, "signal_operations": [f"{operation}:{uri}"]}, + ) + + assert response.status_code == 200 + assert response.json()["data"]["unit"] == expected_unit + + def test_plot_data_with_signal_operations_same_shape_different_uris(interpolation_entry_path_directory): db_names = [ f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 9768146f..7779fb10 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -6,6 +6,7 @@ apply_savgol_filter, apply_signal_operations, apply_simple_operations, + combine_signal_units, ) @@ -53,6 +54,21 @@ def test_apply_signal_operations_preserves_operand_nans(): assert np.allclose(result, [11.0, np.nan, 33.0], equal_nan=True) +@pytest.mark.parametrize( + ("left_unit", "right_unit", "operation", "expected"), + [ + ("kg", "kg", "add", "kg"), + ("kg", "kg", "sub", "kg"), + ("kg", "kg", "mul", "kg*kg"), + ("kg", "kg", "div", ""), + ("kg", "s", "div", "kg/s"), + ("", "kg", "mul", "kg"), + ], +) +def test_combine_signal_units(left_unit, right_unit, operation, expected): + assert combine_signal_units(left_unit, right_unit, operation) == expected + + def test_apply_gaussian_smoothing(): data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1]) From c218ca2b9f1a865de491def5f514c7c3243a6ecc Mon Sep 17 00:00:00 2001 From: Palak Bartek EXT Date: Tue, 21 Jul 2026 13:12:16 +0200 Subject: [PATCH 24/29] Matching time coordinates for signal operations --- .../ibex/data_source/imas_python_source.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index b14b919c..0a37a4fc 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -1250,9 +1250,40 @@ def convert_to_lists(data): # Fetch it now and verify shape compatibility. request = PlotDataRequestModel(uri=signal_uri) other_signal = self.get_plot_data(request) + + # Comparing time coordinates (if any) + other_coordinates = other_signal["data"]["coordinates"] + + # Looking for time coordinate of the 'current' signal + current_time = None + for coordinate in coordinates_to_be_returned: + if coordinate["name"] == "time": + current_time = coordinate + break + + # Looking for time coordinate of the 'other' signal + other_time = None + for coordinate in other_coordinates: + if coordinate["name"] == "time": + other_time = coordinate + break + + time_coordinates_match = ( # Both time coordinates are None + current_time is None and other_time is None + ) or ( # Both time coordinates are equal + current_time is not None + and other_time is not None + and np.array_equal( + np.asarray(flatten(convert_to_lists(current_time["value"]))), + np.asarray(flatten(convert_to_lists(other_time["value"]))), + equal_nan=True, + ) + ) + if ( other_signal["data"]["shape"] != "irregular" and other_signal["data"]["shape"] == original_data_shape + and time_coordinates_match ): others_signals_data[signal_uri] = { "uri": request.uri, @@ -1265,7 +1296,7 @@ def convert_to_lists(data): "unit": other_signal["data"]["unit"], } else: - msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape does not match. Try interpolating signal onto data's shape." + msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape or time coordinates does not match. Try interpolating signal onto data's shape." raise InvalidParametersException(msg) # Step 4: prepare interpolated_data (resampled or raw) From 405950cc2cbd6f57742563633a7ca06fbacb56d2 Mon Sep 17 00:00:00 2001 From: Palak Bartek EXT Date: Tue, 21 Jul 2026 14:30:54 +0200 Subject: [PATCH 25/29] Code refactoring --- .../ibex/data_source/imas_python_source.py | 89 ++++++++++--------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 0a37a4fc..9e9cf296 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -1251,54 +1251,34 @@ def convert_to_lists(data): request = PlotDataRequestModel(uri=signal_uri) other_signal = self.get_plot_data(request) + # Comparing signal shapes + if ( + other_signal["data"]["shape"] == "irregular" + or other_signal["data"]["shape"] != original_data_shape + ): + msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape does not match. Try interpolating signal onto data's shape." + raise InvalidParametersException(msg) + # Comparing time coordinates (if any) - other_coordinates = other_signal["data"]["coordinates"] - - # Looking for time coordinate of the 'current' signal - current_time = None - for coordinate in coordinates_to_be_returned: - if coordinate["name"] == "time": - current_time = coordinate - break - - # Looking for time coordinate of the 'other' signal - other_time = None - for coordinate in other_coordinates: - if coordinate["name"] == "time": - other_time = coordinate - break - - time_coordinates_match = ( # Both time coordinates are None - current_time is None and other_time is None - ) or ( # Both time coordinates are equal - current_time is not None - and other_time is not None - and np.array_equal( - np.asarray(flatten(convert_to_lists(current_time["value"]))), - np.asarray(flatten(convert_to_lists(other_time["value"]))), - equal_nan=True, - ) + time_coordinates_match = self._time_coordinates_match( + coordinates_to_be_returned, other_signal["data"]["coordinates"] ) - if ( - other_signal["data"]["shape"] != "irregular" - and other_signal["data"]["shape"] == original_data_shape - and time_coordinates_match - ): - others_signals_data[signal_uri] = { - "uri": request.uri, - "data": other_signal["data"]["value"], - "coordinates": [ - sorted(set(flatten(convert_to_lists(c["value"])))) - for c in other_signal["data"]["coordinates"] - ], - "shape": other_signal["data"]["shape"], - "unit": other_signal["data"]["unit"], - } - else: - msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape or time coordinates does not match. Try interpolating signal onto data's shape." + if not time_coordinates_match: + msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Time coordinates does not match. Try interpolating signal onto data's shape." raise InvalidParametersException(msg) + others_signals_data[signal_uri] = { + "uri": request.uri, + "data": other_signal["data"]["value"], + "coordinates": [ + sorted(set(flatten(convert_to_lists(c["value"])))) + for c in other_signal["data"]["coordinates"] + ], + "shape": other_signal["data"]["shape"], + "unit": other_signal["data"]["unit"], + } + # Step 4: prepare interpolated_data (resampled or raw) if "interpolated_data" not in others_signals_data[signal_uri]: if plot_data_query.interpolate_over: @@ -1390,6 +1370,29 @@ def convert_to_lists(data): coordinate["coordinates"] = new_shape_factors_list return result + def _time_coordinates_match(self, coordinates_1, coordinates_2): + # Looking for time coordinate of the 'current' signal + time_1 = None + for coordinate in coordinates_1: + if coordinate["name"] == "time": + time_1 = coordinate + break + # Looking for time coordinate of the 'other' signal + time_2 = None + for coordinate in coordinates_2: + if coordinate["name"] == "time": + time_2 = coordinate + break + if time_1 is None or time_2 is None: + time_coordinates_match = time_1 is None and time_2 is None + else: + time_coordinates_match = np.array_equal( + np.asarray(time_1["value"]), + np.asarray(time_2["value"]), + equal_nan=True, + ) + return time_coordinates_match + def _is_empty(self, seq): """Checks if list is essentially empty (contains only empty lists or empty strings)""" if isinstance(seq, (IDSNumericArray, IDSString0D, IDSString1D, IDSComplex0D, IDSFloat0D, IDSInt0D)): From 556cdb5b84771fc01b54ebcb2f347919264e1077 Mon Sep 17 00:00:00 2001 From: Bartek Palak Date: Wed, 22 Jul 2026 08:40:08 +0200 Subject: [PATCH 26/29] Removing NaN handling section from signal operations documentation. --- .../backend_development/data_manipulation.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index 227e87b2..f8350d2b 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -231,21 +231,6 @@ Multiple signal operations (subtraction then multiplication): '/data/plot_data?uri=&signal_operations=sub:&signal_operations=mul:' \ -H 'accept: application/json' -.. _`Signal operations details`: -Null and NaN handling -~~~~~~~~~~~~~~~~~~~~~~ -Before applying each operation, the backend inspects the operand signal's data for NaN values. -If any NaN values are found, they are replaced with an **operation-specific neutral value** -that will not affect the result of the operation: - -* ``add`` — neutral value ``0`` -* ``sub`` — neutral value ``0`` -* ``mul`` — neutral value ``1`` -* ``div`` — neutral value ``1`` -* ``pow`` — neutral value ``1`` -* ``root`` — neutral value ``1`` - -This prevents NaN propagation through chained operations. From d213449d82fbc6b50f4833374d3a12a6570c8c7a Mon Sep 17 00:00:00 2001 From: Bartek Palak Date: Wed, 22 Jul 2026 15:20:22 +0200 Subject: [PATCH 27/29] Signal operations: checking compatibility of all coordinates of operands --- .../ibex/data_source/imas_python_source.py | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 6b914bf0..37c4ff25 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -1279,13 +1279,13 @@ def convert_to_lists(data): msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape does not match. Try interpolating signal onto data's shape." raise InvalidParametersException(msg) - # Comparing time coordinates (if any) - time_coordinates_match = self._time_coordinates_match( + # Comparing coordinates + coordinates_match = self._coordinates_match( coordinates_to_be_returned, other_signal["data"]["coordinates"] ) - if not time_coordinates_match: - msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Time coordinates does not match. Try interpolating signal onto data's shape." + if not coordinates_match: + msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Coordinates do not match. Try interpolating signal onto data's shape." raise InvalidParametersException(msg) others_signals_data[signal_uri] = { @@ -1390,28 +1390,18 @@ def convert_to_lists(data): coordinate["coordinates"] = new_shape_factors_list return result - def _time_coordinates_match(self, coordinates_1, coordinates_2): - # Looking for time coordinate of the 'current' signal - time_1 = None - for coordinate in coordinates_1: - if coordinate["name"] == "time": - time_1 = coordinate - break - # Looking for time coordinate of the 'other' signal - time_2 = None - for coordinate in coordinates_2: - if coordinate["name"] == "time": - time_2 = coordinate - break - if time_1 is None or time_2 is None: - time_coordinates_match = time_1 is None and time_2 is None - else: - time_coordinates_match = np.array_equal( - np.asarray(time_1["value"]), - np.asarray(time_2["value"]), - equal_nan=True, + def _coordinates_match(self, coordinates_1, coordinates_2): + if len(coordinates_1) != len(coordinates_2): + return False + + coordinates_match = all( + coordinate_1["name"] == coordinate_2["name"] + and np.array_equal( + np.asarray(coordinate_1["value"]), np.asarray(coordinate_2["value"]), equal_nan=True ) - return time_coordinates_match + for coordinate_1, coordinate_2 in zip(coordinates_1, coordinates_2) + ) + return coordinates_match def _is_empty(self, seq): """Checks if list is essentially empty (contains only empty lists or empty strings)""" From e9a9537c3374adabff8efdcdafb9b4245ca412d8 Mon Sep 17 00:00:00 2001 From: Palak Bartek EXT Date: Thu, 23 Jul 2026 13:18:03 +0200 Subject: [PATCH 28/29] refactor: simplify signal operand data preparation --- .../ibex/data_source/imas_python_source.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 37c4ff25..f060eda6 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -1247,9 +1247,9 @@ def convert_to_lists(data): # a. If the signal was already interpolated (stored during # interpolation phase), skip fetching # b. Otherwise fetch the signal and check shape compatibility - # 4. Prepare interpolated_data for each signal: + # 4. Prepare operand data for each signal: # a. If interpolation was requested, resample onto common coords - # b. Otherwise use raw signal data directly (same shape path) + # b. Otherwise normalize raw signal data to a NumPy array # 5. Build a flat uri->array dict and apply all signal operations if plot_data_query.signal_operations: @@ -1276,7 +1276,11 @@ def convert_to_lists(data): other_signal["data"]["shape"] == "irregular" or other_signal["data"]["shape"] != original_data_shape ): - msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Signal shape and data shape does not match. Try interpolating signal onto data's shape." + msg = ( + f"Cannot apply operation on signal {signal_uri} without interpolation. " + "Signal and data shapes do not match. " + "Try interpolating the signal onto the data shape." + ) raise InvalidParametersException(msg) # Comparing coordinates @@ -1285,7 +1289,11 @@ def convert_to_lists(data): ) if not coordinates_match: - msg = f"Cannot apply operation on signal {signal_uri} without interpolation. Coordinates do not match. Try interpolating signal onto data's shape." + msg = ( + f"Cannot apply operation on signal {signal_uri} without interpolation. " + "Coordinates do not match. " + "Try interpolating the signal onto the data coordinates." + ) raise InvalidParametersException(msg) others_signals_data[signal_uri] = { @@ -1299,22 +1307,21 @@ def convert_to_lists(data): "unit": other_signal["data"]["unit"], } - # Step 4: prepare interpolated_data (resampled or raw) - if "interpolated_data" not in others_signals_data[signal_uri]: - if plot_data_query.interpolate_over: - # Resample signal data onto the common coordinate grid - signal_data = resample_data_without_interpolation( - tuple(reversed(others_signals_data[signal_uri]["coordinates"])), - others_signals_data[signal_uri]["data"], - tuple(common_coords_values), - ) - else: - # No interpolation needed — use signal data as-is - signal_data = others_signals_data[signal_uri]["data"] - others_signals_data[signal_uri]["interpolated_data"] = np.array(signal_data) + # Step 4: prepare operand data (resampled or raw) + if plot_data_query.interpolate_over: + # Resample signal data onto the common coordinate grid + signal_data = resample_data_without_interpolation( + tuple(reversed(others_signals_data[signal_uri]["coordinates"])), + others_signals_data[signal_uri]["data"], + tuple(common_coords_values), + ) + else: + signal_data = others_signals_data[signal_uri]["data"] + + others_signals_data[signal_uri]["data"] = np.asarray(signal_data) # Step 5: flatten dict and apply operations in order - signal_data_by_uri = {uri: info["interpolated_data"] for uri, info in others_signals_data.items()} + signal_data_by_uri = {uri: info["data"] for uri, info in others_signals_data.items()} data_to_be_returned = apply_signal_operations( data_to_be_returned, plot_data_query.signal_operations, signal_data_by_uri ) From 353af26d5e029a2b0294cfcdd427377e21789a55 Mon Sep 17 00:00:00 2001 From: Palak Bartek EXT Date: Thu, 23 Jul 2026 13:46:27 +0200 Subject: [PATCH 29/29] Reject add and sub operations with mismatched units --- backend/ibex/data_source/imas_python_source.py | 10 ++++------ backend/ibex/data_source/imas_python_source_utils.py | 4 ++++ backend/tests/test_data_endpoints.py | 8 +++----- backend/tests/test_data_manipulation.py | 9 +++++++++ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index f060eda6..928b9e8e 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -1322,9 +1322,6 @@ def convert_to_lists(data): # Step 5: flatten dict and apply operations in order signal_data_by_uri = {uri: info["data"] for uri, info in others_signals_data.items()} - data_to_be_returned = apply_signal_operations( - data_to_be_returned, plot_data_query.signal_operations, signal_data_by_uri - ) for operation in plot_data_query.signal_operations: operation_type, signal_uri = operation.split(":", 1) result_unit = combine_signal_units( @@ -1332,6 +1329,9 @@ def convert_to_lists(data): others_signals_data[signal_uri]["unit"], operation_type, ) + data_to_be_returned = apply_signal_operations( + data_to_be_returned, plot_data_query.signal_operations, signal_data_by_uri + ) # ============= END signal operations ============= @@ -1403,9 +1403,7 @@ def _coordinates_match(self, coordinates_1, coordinates_2): coordinates_match = all( coordinate_1["name"] == coordinate_2["name"] - and np.array_equal( - np.asarray(coordinate_1["value"]), np.asarray(coordinate_2["value"]), equal_nan=True - ) + and np.array_equal(np.asarray(coordinate_1["value"]), np.asarray(coordinate_2["value"]), equal_nan=True) for coordinate_1, coordinate_2 in zip(coordinates_1, coordinates_2) ) return coordinates_match diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index a84e97d4..3572f715 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -221,6 +221,10 @@ def combine_signal_units(left_unit: str, right_unit: str, operation: str) -> str right_unit = right_unit or "" if operation in {"add", "sub"}: + if left_unit != right_unit: + raise InvalidParametersException( + f"Cannot {operation} signals with different units ({left_unit!r} and {right_unit!r})" + ) return left_unit if operation == "mul": if not left_unit: diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index d7f0b652..ac5ba017 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -373,16 +373,14 @@ def test_plot_data_coordinate_aliases(entry_path, expected_unit): assert dim2_coordinate["unit"].lower() == expected_unit[1] -def test_plot_data_with_signal_operations(entry_path): +def test_plot_data_with_signal_operations_rejects_different_units(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", "signal_operations": [f"add:imas:hdf5?path={entry_path}#core_profiles/global_quantities/ip"], } response = pytest.test_client.get("/data/plot_data", params=parameters) - assert response.status_code == 200 - - response_body = response.json() - assert response_body["data"]["value"] == pytest.approx([2.0, 4.0, 6.0, 8.0, 10.0]) + assert response.status_code == 466 + assert "Cannot add signals with different units" in response.json()["message"] @pytest.mark.parametrize(("operation", "expected_unit"), [("mul", "s*s"), ("div", "")]) diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 7779fb10..b4e47f37 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -69,6 +69,15 @@ def test_combine_signal_units(left_unit, right_unit, operation, expected): assert combine_signal_units(left_unit, right_unit, operation) == expected +@pytest.mark.parametrize("operation", ["add", "sub"]) +def test_combine_signal_units_rejects_different_units(operation): + with pytest.raises( + InvalidParametersException, + match=rf"Cannot {operation} signals with different units", + ): + combine_signal_units("kg", "s", operation) + + def test_apply_gaussian_smoothing(): data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1])