From e4e0fbfa9643f8ef08c8c082b9dc0bbcc9ff3bd9 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 20 Jun 2026 17:34:01 +0200 Subject: [PATCH 001/118] chore(geospatial): add basic gradle configuration --- alchemist-geospatial/build.gradle.kts | 35 ++++++++++++++++++++++++++ alchemist-geospatial/gradle.properties | 10 ++++++++ build.gradle.kts | 1 + gradle/libs.versions.toml | 3 +++ settings.gradle.kts | 1 + 5 files changed, 50 insertions(+) create mode 100644 alchemist-geospatial/build.gradle.kts create mode 100644 alchemist-geospatial/gradle.properties diff --git a/alchemist-geospatial/build.gradle.kts b/alchemist-geospatial/build.gradle.kts new file mode 100644 index 0000000000..bdeb7cfaf8 --- /dev/null +++ b/alchemist-geospatial/build.gradle.kts @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +import Libs.alchemist + +plugins { + id("kotlin-jvm-convention") +} + +dependencies { + api(alchemist("api")) + implementation(libs.cdm.core) + compileOnly(libs.cdm.grib) + implementation(libs.gson) + implementation(libs.slf4j) +} + +publishing.publications { + withType { + pom { + contributors { + contributor { + name.set("Emir Wanes Aouioua") + email.set("emirwanes.aouioua@studio.unibo.it") + } + } + } + } +} diff --git a/alchemist-geospatial/gradle.properties b/alchemist-geospatial/gradle.properties new file mode 100644 index 0000000000..b106d3f66d --- /dev/null +++ b/alchemist-geospatial/gradle.properties @@ -0,0 +1,10 @@ +# +# Copyright (C) 2010-2026, Danilo Pianini and contributors listed in the main project's alchemist/build.gradle file. +# +# This file is part of Alchemist, and is distributed under the terms of the +# GNU General Public License, with a linking exception, +# as described in the file LICENSE in the Alchemist distribution's top directory. +# +artifactId = alchemist-geospatial +projectLongName = Alchemist Geospatial +projectDescription = Integration of Copernicus/ECMWF Temporal Raster Data into the Alchemist Simulator diff --git a/build.gradle.kts b/build.gradle.kts index 06f89bad00..24cce0aa85 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,6 +56,7 @@ allprojects { repositories { google() mavenCentral() + maven("https://artifacts.unidata.ucar.edu/repository/unidata-releases/") } // TEST AND COVERAGE diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7e428c0ded..9d1ea303d0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,6 +4,7 @@ antlr4 = "4.13.2" ais-lib = "2.8.7" apollo = "5.1.0" arrow = "2.2.3" +cdm = "5.6.0.1" compose-multiplatform = "1.11.1" dokka = "2.2.0" graphql = "10.2.1" @@ -38,6 +39,8 @@ appdirs = "net.harawata:appdirs:1.5.0" arrow-core = { module = "io.arrow-kt:arrow-core", version.ref = "arrow" } boilerplate = "org.danilopianini:boilerplate:0.2.2" caffeine = "com.github.ben-manes.caffeine:caffeine:3.2.4" +cdm-core = { module = "edu.ucar:cdm-core", version.ref = "cdm" } +cdm-grib = { module = "edu.ucar:grib", version.ref = "cdm" } classgraph = "io.github.classgraph:classgraph:4.8.192" compose-components-resources = { module = "org.jetbrains.compose.components:components-resources", version.ref = "compose-multiplatform" } compose-foundation = { module = "org.jetbrains.compose.foundation:foundation", version.ref = "compose-multiplatform" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 8e0eee544a..abb34bc3a5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,6 +22,7 @@ include( "alchemist-full", "alchemist-graphql", "alchemist-graphql-surrogates", + "alchemist-geospatial", "alchemist-implementationbase", "alchemist-incarnation-protelis", "alchemist-incarnation-sapere", From 868c1b12dce761705af02e062b02f207fff6230b Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 20 Jun 2026 19:42:14 +0200 Subject: [PATCH 002/118] feat(geospatial): add RasterGrid interface and ArrayRasterGrid implementation --- alchemist-geospatial/build.gradle.kts | 2 +- .../geospatial/reading/ArrayRasterGrid.kt | 30 +++++++++++++ .../model/geospatial/reading/RasterGrid.kt | 44 +++++++++++++++++++ build.gradle.kts | 2 +- gradle/libs.versions.toml | 2 +- 5 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt diff --git a/alchemist-geospatial/build.gradle.kts b/alchemist-geospatial/build.gradle.kts index bdeb7cfaf8..d257cafdcf 100644 --- a/alchemist-geospatial/build.gradle.kts +++ b/alchemist-geospatial/build.gradle.kts @@ -16,7 +16,7 @@ plugins { dependencies { api(alchemist("api")) implementation(libs.cdm.core) - compileOnly(libs.cdm.grib) + runtimeOnly(libs.cdm.grib) implementation(libs.gson) implementation(libs.slf4j) } diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt new file mode 100644 index 0000000000..e4a6a95f49 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.reading + +/** + * In-memory implementation of [RasterGrid], format-agnostic. + * + * Once data has been read from a file (NetCDF, GRIB, or any other source) and loaded here, + * this class has no dependency on the reading library. Values are stored in a single + * flattened row-major array, avoiding the overhead and boxing of Array. + * + * @property latitudes see [RasterGrid.latitudes] (ascending). + * @property longitudes see [RasterGrid.longitudes] (ascending). + * @property values Cell values in row-major order; [Double.NaN] indicates a missing value. + */ +class ArrayRasterGrid( + override val latitudes: DoubleArray, + override val longitudes: DoubleArray, + private val values: DoubleArray, +) : RasterGrid { + + override fun valueAt(latIndex: Int, lonIndex: Int): Double = values[latIndex * longitudes.size + lonIndex] +} diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt new file mode 100644 index 0000000000..f7003928d7 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.reading + +/** + * A single 2D spatial "slice" of data on a regular geographic grid (latitude/longitude). + * + * This interface represents a data container: it exposes axes and cell access, + * but contains no interpolation logic (that lives in the spatial strategies). + * + * Axis contract: [latitudes] and [longitudes] are ALWAYS sorted in ascending order. + * Implementations that read files with descending axes (e.g. GloFAS, whose latitude runs + * from +89.95 to −59.95) must normalize internally, so that strategies never need to + * reason about axis direction. + */ +interface RasterGrid { + /** + * Latitudes of grid nodes, in degrees, sorted in ascending order. + */ + val latitudes: DoubleArray + + /** + * Longitudes of grid nodes, in degrees, sorted in ascending order. + */ + val longitudes: DoubleArray + + /** + * Raw value of the cell at the given index coordinates. + * + * @param latIndex index on the [latitudes] axis + * @param lonIndex index on the [longitudes] axis + * + * @return the value of the cell, or [Double.NaN] if the cell + * contains a missing value or a placeholder + */ + fun valueAt(latIndex: Int, lonIndex: Int): Double +} diff --git a/build.gradle.kts b/build.gradle.kts index 24cce0aa85..77b24b7b76 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,7 +56,7 @@ allprojects { repositories { google() mavenCentral() - maven("https://artifacts.unidata.ucar.edu/repository/unidata-releases/") + maven("https://artifacts.unidata.ucar.edu/repository/unidata-all/") } // TEST AND COVERAGE diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9d1ea303d0..f56378c6ac 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,7 +4,7 @@ antlr4 = "4.13.2" ais-lib = "2.8.7" apollo = "5.1.0" arrow = "2.2.3" -cdm = "5.6.0.1" +cdm = "5.6.0" compose-multiplatform = "1.11.1" dokka = "2.2.0" graphql = "10.2.1" From 62840ec66f7fdc47a911f160662e318ce33d4648 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 20 Jun 2026 23:53:09 +0200 Subject: [PATCH 003/118] feat(geospatial): add TimedGrid and CdmTimedGrid implementation --- .../model/geospatial/reading/CdmTimedGrid.kt | 252 ++++++++++++++++++ .../model/geospatial/reading/TimedGrid.kt | 35 +++ 2 files changed, 287 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt new file mode 100644 index 0000000000..6aee3e5c41 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.reading + +import java.nio.file.Files +import java.nio.file.Path +import java.time.Instant +import java.util.Formatter +import java.util.TreeMap +import ucar.nc2.Variable +import ucar.nc2.constants.AxisType +import ucar.nc2.dataset.CoordinateAxis1D +import ucar.nc2.dataset.CoordinateAxis1DTime +import ucar.nc2.dataset.NetcdfDataset +import ucar.nc2.dataset.NetcdfDatasets + +/** + * Eager [TimedGrid] implementation. This implementation depends on + * [NetCDF-Java](https://docs.unidata.ucar.edu/netcdf-java/current/javadoc/index.html). + * + * Reads a directory of **homogeneous** data files (same variable, same spatial grid, + * disjoint temporal coverage) and exposes them as a single time-ordered slice series. + * All data is loaded into memory at construction time; file handles are closed before + * the constructor returns. + * + * Files are opened in **enhanced mode** (fill values being replaced by [Double.NaN], scale/offset applied). + * Latitude and longitude axes are normalized to **ascending order** regardless of the + * direction stored in the file. Spatial homogeneity across files is validated eagerly. + * The variable to read is selected by [variableName], or auto-detected as the unique + * `(time, lat, lon)` variable in the file. + * + * **SAX note:** `NetcdfDatasets.openDataset` parses its internal XML configuration via SAX. + * If Alchemist's classpath contains a xerces/xml-apis jar that overrides the JDK SAX parser, + * a `SAXNotRecognizedException` will be thrown on first open. Two fixes: exclude the + * offending jar from the dependency tree, or set the system property before any call to this class. + * + * @param directory directory of homogeneous spacial data files (NetCDFs/GRIBs). + * @param variableName name of the variable as it appears in the file (e.g. `"dis24"`), + * not the CDS catalogue name. If `null`, auto-detected from the file. + * @throws IllegalArgumentException if the directory is empty; if the variable is missing + * or ambiguous; if files have mismatched spatial axes; if the dimension order is not + * `(time, lat, lon)`; or if two files share a timestamp (i.e. the temporal coverages are not disjoint). + */ +class CdmTimedGrid(directory: Path, variableName: String? = null) : TimedGrid { + + override val instants: List + private val grids: List + + init { + // lists the files in the directory, which must not be empty + val files = Files.list(directory) + .filter { Files.isRegularFile(it) } + .sorted() + .toList() + require(files.isNotEmpty()) { "No data files in $directory" } + + // maps all file time instances to the corresponding RasterGrid, sorting them by Instant + val map = TreeMap() + + /** + * spatial reference established by the first file. + * All subsequent files must have the same coordinates, same grid, + * same number of points. + */ + var refLats: DoubleArray? = null + var refLons: DoubleArray? = null + + for (file in files) { + /* + * opens the file in "enhanced mode": every fill value get replaced with NaN, + * applies scale/offset, following the CF (Climate and Forecast) convention. + */ + NetcdfDatasets.openDataset(file.toString()).use { ds -> + // ensures that all axis are present (time, lat, lon) + val rawTimeAxis = requireNotNull(ds.findCoordinateAxis(AxisType.Time)) { + "No time axis in $file" + } + val latAxis = requireNotNull(ds.findCoordinateAxis(AxisType.Lat) as? CoordinateAxis1D) { + "No 1D latitude axis in $file" + } + val lonAxis = requireNotNull(ds.findCoordinateAxis(AxisType.Lon) as? CoordinateAxis1D) { + "No 1D longitude axis in $file" + } + val errMsg = Formatter() + + /* + * constructs a CF-aware timeline. + * Interprets units as "hours since 1900-0-01" and non-Gregorian calendars. + */ + val timeAxis = requireNotNull(CoordinateAxis1DTime.factory(ds, rawTimeAxis, errMsg)) { + "Cannot build time axis in $file: $errMsg" + } + + val rawLats: DoubleArray = latAxis.coordValues + val rawLons: DoubleArray = lonAxis.coordValues + + /* + determines whether an axis is descending (e.g. GloFAS lat: +89.95 to -59.95). + RasterGrid contract requires them to be ascending. + */ + val latDesc = rawLats.first() > rawLats.last() + val lonDesc = rawLons.first() > rawLons.last() + val lats = if (latDesc) rawLats.reversedArray() else rawLats + val lons = if (lonDesc) rawLons.reversedArray() else rawLons + + if (refLats == null) { + // the first file sets the reference grid + refLats = lats + refLons = lons + } else { + // subsequent files must have the same spatial coordinates as the first one + require(lats.contentEquals(refLats)) { + "Latitude axes differ in $file vs previous files" + } + require(lons.contentEquals(refLons!!)) { + "Longitude axes differ in $file vs previous files" + } + } + + val nLat = lats.size + val nLon = lons.size + + // derives the dimension names needed to find the variable and to validate the order of the dimensions + val timeDimName = rawTimeAxis.dimensions.first().name + val latDimName = latAxis.dimensions.first().name + val lonDimName = lonAxis.dimensions.first().name + + /* + * finds the variable to read, either by name or by auto-detection + * of the only variable with dimensions (time, lat, lon). + */ + val variable = resolveVariable(ds, variableName, timeDimName, latDimName, lonDimName, file) + + /* + * verifies that the order is (time, lat, lon), as specified by CF convention. + * A different order would produce incorrect values. + */ + val dimNames = variable.dimensions.map { it.name } + require(dimNames == listOf(timeDimName, latDimName, lonDimName)) { + "Unexpected dimension order in $file: $dimNames. " + + "Expected [$timeDimName, $latDimName, $lonDimName] (CF convention)." + } + + val nTime = timeAxis.size.toInt() + for (t in 0 until nTime) { + // converts a CalendarDate CF-Aware to an Instant + val instant = timeAxis.getCalendarDate(t).toDate().toInstant() + + /* + * If there are duplicate timestamps across different files, then there are overlapping + * time ranges in the cdsRequest. This is a configuration error. + */ + require(!map.containsKey(instant)) { + "Duplicate timestamp $instant in $directory | " + + "check that files have disjoint temporal coverage" + } + + val rawData = variable.read( + // temporal origin + intArrayOf(t, 0, 0), + // spatial shape + intArrayOf(1, nLat, nLon), + ) + + /* + constructs the double array in row-major order, with ascending normalized axes. + If the index was descending, then srcLat/srcLon are reversed so that iLat=0 + corresponds to the lowest latitude. + */ + val values = DoubleArray(nLat * nLon) { idx -> + val iLat = idx / nLon + val iLon = idx % nLon + val srcLat = if (latDesc) (nLat - 1 - iLat) else iLat + val srcLon = if (lonDesc) (nLon - 1 - iLon) else iLon + rawData.getDouble(srcLat * nLon + srcLon) + } + map[instant] = ArrayRasterGrid(lats, lons, values) + } + } + } + + instants = map.keys.toList() + grids = map.values.toList() + } + + /** + * Returns the spatial slice at the given index. + * + * @param index 0-based index, aligned with [instants]. + * @return the [RasterGrid] for that instant. + */ + override fun grid(index: Int): RasterGrid = grids[index] + + private companion object { + + /** + * Selects the variable to read from the dataset. + * + * If [name] is provided, looks it up by short name (the name as it appears in the file, + * not the CDS catalogue name). Otherwise, auto-detects the unique 3D variable + * whose dimensions exactly match ([timeDimName], [latDimName], [lonDimName]). + * Coordinate axis variables (latitude, longitude, time themselves) are 1D and are therefore excluded + * automatically. + * + * @param ds the open enhanced dataset. + * @param name explicit variable name, or null for auto-detection. + * @param timeDimName name of the time dimension (from the time axis). + * @param latDimName name of the latitude dimension (from the latitude axis). + * @param lonDimName name of the longitude dimension (from the longitude axis). + * @param file path of the file being read (used for error messages). + * @return the selected [Variable]. + * @throws IllegalArgumentException if the named variable is not found, or if + * auto-detection finds zero or more than one candidate. + */ + fun resolveVariable( + ds: NetcdfDataset, + name: String?, + timeDimName: String, + latDimName: String, + lonDimName: String, + file: Path, + ): Variable { + if (name != null) { + return requireNotNull(ds.findVariable(name)) { + "Variable '$name' not found in $file. " + + "Available: ${ds.variables.map { it.shortName }}" + } + } + val targetDims = setOf(timeDimName, latDimName, lonDimName) + // 3D variables matching {latitude, longitude, time} + val candidates = ds.variables.filter { v -> + v.dimensions.size == 3 && + v.dimensions.map { it.name }.toSet() == targetDims + } + require(candidates.isNotEmpty()) { + "No variable with dimensions $targetDims found in $file" + } + require(candidates.size == 1) { + "Multiple candidate variables with dimensions $targetDims in $file: " + + "${candidates.map { it.shortName }}. Specify variableName explicitly." + } + return candidates.single() + } + } +} diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt new file mode 100644 index 0000000000..642a992f09 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.reading + +import java.time.Instant + +/** + * A time-ordered sequence of [RasterGrid] slices backed by geophysical source data. + * + * [instants] and [grid] are aligned: the i-th instant is the real-world timestamp of the + * slice returned by `grid(i)`. [instants] is guaranteed to be strictly ascending with no + * duplicates. [instants] exposes [java.time.Instant] so that callers can convert + * real-world timestamps to simulation time once at construction; [grid] is index-based + * because temporal interpolation operates on adjacent indices, not timestamps directly. + */ +interface TimedGrid { + /** + * Real-world timestamps of each slice, strictly ascending, aligned with [grid]. + */ + val instants: List + + /** + * Returns the spatial slice at [index]. + * + * @param index 0-based, aligned with [instants]. + */ + fun grid(index: Int): RasterGrid +} From fd364bd4afb7fc26c42a1a4fb4dbb5d8cb86c5c6 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 21 Jun 2026 02:44:22 +0200 Subject: [PATCH 004/118] chore(geospatial): update cdm dependency version to latest (5.9.1) --- alchemist-geospatial/build.gradle.kts | 2 ++ gradle/libs.versions.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/alchemist-geospatial/build.gradle.kts b/alchemist-geospatial/build.gradle.kts index d257cafdcf..afe4b6e9fc 100644 --- a/alchemist-geospatial/build.gradle.kts +++ b/alchemist-geospatial/build.gradle.kts @@ -18,7 +18,9 @@ dependencies { implementation(libs.cdm.core) runtimeOnly(libs.cdm.grib) implementation(libs.gson) + implementation(libs.guava) implementation(libs.slf4j) + testImplementation(alchemist("test")) } publishing.publications { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f56378c6ac..d2a95bfbcd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,7 +4,7 @@ antlr4 = "4.13.2" ais-lib = "2.8.7" apollo = "5.1.0" arrow = "2.2.3" -cdm = "5.6.0" +cdm = "5.9.1" compose-multiplatform = "1.11.1" dokka = "2.2.0" graphql = "10.2.1" From 7489e9318fa9606e6cff78be3591beda30d3d249 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 21 Jun 2026 16:24:26 +0200 Subject: [PATCH 005/118] test(geospatial): add tests for ArrayRasterGrid and TestCdmTimedGrid --- .../geospatial/reading/TestArrayRasterGrid.kt | 66 +++++ .../geospatial/reading/TestCdmTimedGrid.kt | 251 ++++++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestArrayRasterGrid.kt create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestArrayRasterGrid.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestArrayRasterGrid.kt new file mode 100644 index 0000000000..9b65ba259f --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestArrayRasterGrid.kt @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.reading + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe + +class TestArrayRasterGrid : StringSpec({ + + /** + * 3 latitudes by 4 longitudes. + * The cell value at (iLat, iLon) is iLat * 10 + iLon, + * so it is easily verifiable. + */ + val lats = doubleArrayOf(10.0, 20.0, 30.0) + val lons = doubleArrayOf(5.0, 15.0, 25.0, 35.0) + val values = DoubleArray(12) { idx -> (idx / lons.size) * 10.0 + (idx % lons.size) } + val grid = ArrayRasterGrid(lats, lons, values) + + // Value access tests + "valueAt should return the correct value for an interior cell" { + // (iLat=1, iLong=2), then index = 1*4 + 2 = 6, then value = 1*10 + 2 = 12 + grid.valueAt(1, 2) shouldBe 12.0 + } + + "valueAt should return correct values at all four corners" { + grid.valueAt(0, 0) shouldBe 0.0 // bottom left corner + grid.valueAt(0, 3) shouldBe 3.0 // bottom right + grid.valueAt(2, 0) shouldBe 20.0 // top left + grid.valueAt(2, 3) shouldBe 23.0 // top right + } + + // Missing values tests + "valueAt should preserve Double.NaN for missing values" { + // (iLat=1, iLon=1) is a NaN + val nanValues = DoubleArray(12) { idx -> if (idx == 5) Double.NaN else idx.toDouble() } + val nanGrid = ArrayRasterGrid(lats, lons, nanValues) + nanGrid.valueAt(1, 1).shouldBeNaN() + } + + "a grid entirely made of NaN should return NaN everywhere" { + val allNaN = ArrayRasterGrid(lats, lons, DoubleArray(12) { Double.NaN }) + for (iLat in 0..2) { + for (iLon in 0..3) { + allNaN.valueAt(iLat, iLon).shouldBeNaN() + } + } + } + + // Axis tests + "latitudes should be accessible and match the constructor argument" { + grid.latitudes shouldBe lats + } + + "longitudes should be accessible and match the constructor argument" { + grid.longitudes shouldBe lons + } +}) diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt new file mode 100644 index 0000000000..c1c75f86ee --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.reading + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.collections.shouldBeSortedWith +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe +import java.nio.file.Files +import java.nio.file.Path +import ucar.ma2.Array as UcarArray +import ucar.ma2.ArrayDouble +import ucar.ma2.ArrayFloat +import ucar.ma2.DataType +import ucar.nc2.Attribute +import ucar.nc2.write.NetcdfFormatWriter + +class TestCdmTimedGrid : StringSpec({ + + /** + * the directory where the temporary NetCDF files for the tests will be created + */ + val tempDir: Path = Files.createTempDirectory("cdm-timed-grid-test") + + afterSpec { + // empties the tmp directory after each test + tempDir.toFile().deleteRecursively() + } + + // Basic reading tests + "should read a single file and expose the correct number of instants" { + val dir = Files.createTempDirectory(tempDir, "basic") + writeFixedTestNetcdf(dir, "data.nc", doubleArrayOf(0.0, 24.0, 48.0)) + val tg = CdmTimedGrid(dir) + tg.instants.size shouldBe 3 + } + + "instants should be sorted in ascending chronological order regardless of file order" { + val dir = Files.createTempDirectory(tempDir, "sorted") + // b_file.nc is read second (in alphabetical order) but contains the most recent times + writeFixedTestNetcdf(dir, "b_file.nc", doubleArrayOf(48.0, 72.0)) + writeFixedTestNetcdf(dir, "a_file.nc", doubleArrayOf(0.0, 24.0)) + val tg = CdmTimedGrid(dir) + tg.instants.size shouldBe 4 + tg.instants shouldBeSortedWith compareBy { it } + } + + "grid[i] should be accessible for every index in instants" { + val dir = Files.createTempDirectory(tempDir, "align") + writeFixedTestNetcdf(dir, "data.nc", doubleArrayOf(0.0, 24.0, 48.0, 72.0, 96.0)) + val tg = CdmTimedGrid(dir) + // ff instants and grids are misaligned, the grid(s) would throw an IndexOutOfBoundsException + tg.instants.indices.forEach { i -> tg.grid(i).latitudes.size shouldBe 3 } + } + + // Descending latitude normalization tests + "latitudes should be ascending even when the file stores them descending" { + val dir = Files.createTempDirectory(tempDir, "lat-desc") + writeTestNetcdf( + path = dir.resolve("desc.nc"), + lats = floatArrayOf(30f, 20f, 10f), // descending latitudes + lons = floatArrayOf(5f, 15f, 25f, 35f), + timeHours = doubleArrayOf(0.0), + ) + val resultLats = CdmTimedGrid(dir).grid(0).latitudes + resultLats shouldBe doubleArrayOf(10.0, 20.0, 30.0) + } + + "values should be correctly re-mapped after descending latitude normalization" { + val dir = Files.createTempDirectory(tempDir, "lat-remap") + /* + * 2-by-2 grid with descending lat. Row 0: north (20°), row 1: south (10°). + * After normalization: iLat=0 = south (10°), iLat=1 = north (20°). + */ + writeTestNetcdf( + path = dir.resolve("remap.nc"), + lats = floatArrayOf(20f, 10f), + lons = floatArrayOf(5f, 15f), + timeHours = doubleArrayOf(0.0), + // north=[100,101], south=[200,201] + rawValues = floatArrayOf(100f, 101f, 200f, 201f), + ) + // checks if the rows get reversed + val grid = CdmTimedGrid(dir).grid(0) + grid.valueAt(0, 0) shouldBe 200.0 + grid.valueAt(0, 1) shouldBe 201.0 + grid.valueAt(1, 0) shouldBe 100.0 + grid.valueAt(1, 1) shouldBe 101.0 + } + + // "_FillValue" to NaN test + "fill values should be exposed as Double.NaN" { + val dir = Files.createTempDirectory(tempDir, "fillval") + val fill = -9999f + writeTestNetcdf( + path = dir.resolve("fill.nc"), + lats = floatArrayOf(10f, 20f), + lons = floatArrayOf(5f, 15f), + timeHours = doubleArrayOf(0.0), + rawValues = floatArrayOf(fill, 42f, 42f, 42f), + fillValue = fill, + ) + val grid = CdmTimedGrid(dir).grid(0) + grid.valueAt(0, 0).shouldBeNaN() + grid.valueAt(0, 1) shouldBe 42.0 + } + + // Configuration errors tests + "should throw IllegalArgumentException on empty directory" { + val emptyDir = Files.createTempDirectory(tempDir, "empty") + shouldThrow { CdmTimedGrid(emptyDir) } + } + + "should throw IllegalArgumentException on duplicate timestamps across files" { + val dir = Files.createTempDirectory(tempDir, "dup") + // writes multiple files with overlapping time offsets + for (i in 1..3) { + writeFixedTestNetcdf(dir, "f$i.nc", doubleArrayOf(0.0, 24.0)) + } + shouldThrow { CdmTimedGrid(dir) } + } + + "should throw IllegalArgumentException when files have mismatched spatial grids" { + val dir = Files.createTempDirectory(tempDir, "mismatch") + writeTestNetcdf( + dir.resolve("f1.nc"), + floatArrayOf(10f, 20f, 30f), + floatArrayOf(5f, 15f), + doubleArrayOf(0.0), + ) + writeTestNetcdf( + dir.resolve("f2.nc"), + floatArrayOf(40f, 50f, 60f), + floatArrayOf(5f, 15f), + doubleArrayOf(24.0), + ) + shouldThrow { CdmTimedGrid(dir) } + } +}) + +/** + * Writes a minimal CF-compliant NetCDF-3 file to [path] to be used as a [CdmTimedGrid]. + * + * The file contains: + * - a `time` axis with units `"hours since 2024-01-01 00:00"` and values [timeHours]. + * - a `latitude` axis (CF axis Y) with values [lats]. May be descending to test normalization. + * - a `longitude` axis (CF axis X) with values [lons]. + * - a single data variable named [variableName] with `_FillValue` set to [fillValue]. + * + * @param path destination file. The parent (directory must already exist). + * @param lats latitude values in degrees, stored in the file as-is. Pass a descending array + * to test axis normalization in [CdmTimedGrid]. + * @param lons longitude values in degrees, stored in the file as-is. + * @param timeHours time offsets in hours from `2024-01-01 00:00`, one per time step. + * @param rawValues flat `time * lat * lon` data array in row-major order, or `null` to + * use the default pattern `iLat * 10 + iLon` repeated across all time steps. If provided, + * size must equal `nTime * lats.size * lons.size`. + * @param fillValue written as the `_FillValue` attribute on the data variable. When the file + * is opened in CDM enhanced mode, cells matching this value are replaced with [Double.NaN]. + * @param variableName short name of the data variable as it appears in the file (e.g. `"dis24"`). + */ +private fun writeTestNetcdf( + path: Path, + lats: FloatArray, + lons: FloatArray, + timeHours: DoubleArray, + rawValues: FloatArray? = null, + fillValue: Float = -9999f, + variableName: String = "dis24", +) { + // defines the schema on the builder + val builder = NetcdfFormatWriter.createNewNetcdf3(path.toString()) + + builder.addDimension("time", timeHours.size) + builder.addDimension("latitude", lats.size) + builder.addDimension("longitude", lons.size) + + // CF compliant axis + builder.addVariable("time", DataType.DOUBLE, "time").apply { + addAttribute(Attribute("units", "hours since 2024-01-01 00:00")) + addAttribute(Attribute("calendar", "standard")) + addAttribute(Attribute("axis", "T")) + } + builder.addVariable("latitude", DataType.FLOAT, "latitude").apply { + addAttribute(Attribute("units", "degrees_north")) + addAttribute(Attribute("axis", "Y")) + } + builder.addVariable("longitude", DataType.FLOAT, "longitude").apply { + addAttribute(Attribute("units", "degrees_east")) + addAttribute(Attribute("axis", "X")) + } + builder.addVariable(variableName, DataType.FLOAT, "time latitude longitude").apply { + addAttribute(Attribute("_FillValue", fillValue)) + } + + // creates the file and enters write mode + builder.build().use { writer -> + val timeArr = ArrayDouble.D1(timeHours.size) + timeHours.forEachIndexed { i, v -> timeArr.set(i, v) } + writer.write("time", timeArr) + + val latArr = ArrayFloat.D1(lats.size) + lats.forEachIndexed { i, v -> latArr.set(i, v) } + writer.write("latitude", latArr) + + val lonArr = ArrayFloat.D1(lons.size) + lons.forEachIndexed { i, v -> lonArr.set(i, v) } + writer.write("longitude", lonArr) + + val flat = rawValues ?: FloatArray(timeHours.size * lats.size * lons.size) { idx -> + val iLat = (idx / lons.size) % lats.size + val iLon = idx % lons.size + (iLat * 10 + iLon).toFloat() + } + writer.write( + variableName, + UcarArray.factory( + DataType.FLOAT, + intArrayOf(timeHours.size, lats.size, lons.size), + flat, + ), + ) + } +} + +/** + * Creates a NetCDF-3 file with + * - latitudes: (10°, 20°, 30°) + * - longitudes: (5°, 15°, 25°, 35°) + * - the provided hours offset from `2024-01-01 00:00` + * + * @param dir the directory where the file will be created (must exist) + * @param fileName the name of the file + * @param timeHours hours offsets from `2024-01-01 00:00` + */ +private fun writeFixedTestNetcdf(dir: Path, fileName: String, timeHours: DoubleArray) { + writeTestNetcdf( + path = dir.resolve(fileName), + lats = floatArrayOf(10f, 20f, 30f), + lons = floatArrayOf(5f, 15f, 25f, 35f), + timeHours = timeHours, + ) +} From f8c91b7198abc2f258aaefcad70a20ff9101fe47 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 21 Jun 2026 18:34:34 +0200 Subject: [PATCH 006/118] feat(geospatial): add Axes helpers for index finding and their tests --- .../model/geospatial/strategy/Axes.kt | 114 ++++++++++++++++++ .../model/geospatial/strategy/TestAxes.kt | 106 ++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/Axes.kt create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestAxes.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/Axes.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/Axes.kt new file mode 100644 index 0000000000..b254ec11aa --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/Axes.kt @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import java.util.Arrays + +/* + * Index-finding helpers over a regular geographic axis (the latitudes or longitudes of a RasterGrid). + * + * The shared precondition for every function is: axis must be non-empty and sorted in + * STRICTLY ascending order with finite values (no duplicates, no NaN within the axis). + * + * The ascending order and the absence of NaN within the axis are NOT verified: an O(n) scan + * would defeat the purpose of the O(log n) search, so violating them yields undefined results. + * An empty axis and a NaN query coordinate throw an IllegalArgumentException, as those checks + * are O(1). + */ + +/** + * Finds the index of the closest coordinate on the [axis] relative to the query [coordinate]. + * + * If the [coordinate] lies outside the boundaries of the axis, the result is clamped to the edge + * (`0` if below the first node, `axis.lastIndex` if above the last node). This makes the function + * suitable for both nearest-neighbor sampling inside the grid and edge-clamping outside it. + * Ties (when the coordinate is exactly halfway between two nodes) resolve to the lower index. + * + * @param axis grid coordinates along one dimension, strictly ascending. + * @param coordinate the query coordinate in the same unit as [axis]. + * @return a valid index into [axis], guaranteed to be within `0..axis.lastIndex`. + * @throws IllegalArgumentException if [axis] is empty or [coordinate] is NaN. + */ +internal fun nearestIndex(axis: DoubleArray, coordinate: Double): Int { + require(axis.isNotEmpty()) { "Axis cannot be empty." } + require(!coordinate.isNaN()) { "The query coordinate cannot be NaN." } + + val binarySearchResult = Arrays.binarySearch(axis, coordinate) + + // exact match found: the coordinate aligns perfectly with a grid node. + if (binarySearchResult >= 0) return binarySearchResult + + /* + * binarySearch returns -(insertionPoint) - 1 when no match is found. Inverting the formula + * yields the insertion point: the index of the first node strictly greater than the coordinate. + */ + val upperIndex = -binarySearchResult - 1 + + if (upperIndex <= 0) return 0 + if (upperIndex >= axis.size) return axis.lastIndex + + // the coordinate is between axis[lowerIndex] and axis[upperIndex] + val lowerIndex = upperIndex - 1 + + val distanceToLower = coordinate - axis[lowerIndex] + val distanceToUpper = axis[upperIndex] - coordinate + + // return the index with the smaller distance. Ties go to lowerIndex. + return if (distanceToLower <= distanceToUpper) lowerIndex else upperIndex +} + +/** + * Finds the pair of indices `(lowerIndex, upperIndex)` that bracket the given [coordinate] + * on the [axis], such that `axis[lowerIndex] <= coordinate <= axis[upperIndex]`. + * + * The indices will be identical (`lowerIndex == upperIndex`) when the [coordinate] lands exactly + * on a node, or when it is outside the axis boundaries (both indices clamp to the same edge). + * + * @param axis grid coordinates along one dimension, strictly ascending. + * @param coordinate the query coordinate in the same unit as [axis]. + * @return a [Pair] where `first` is the lower index and `second` is the upper index. + * @throws IllegalArgumentException if [axis] is empty or [coordinate] is NaN. + */ +internal fun bracketIndices(axis: DoubleArray, coordinate: Double): Pair { + require(axis.isNotEmpty()) { "Axis cannot be empty." } + require(!coordinate.isNaN()) { "The query coordinate cannot be NaN." } + + val binarySearchResult = Arrays.binarySearch(axis, coordinate) + + // exact match + if (binarySearchResult >= 0) return binarySearchResult to binarySearchResult + + val upperIndex = -binarySearchResult - 1 + + if (upperIndex <= 0) return 0 to 0 + if (upperIndex >= axis.size) return axis.lastIndex to axis.lastIndex + + val lowerIndex = upperIndex - 1 + return lowerIndex to upperIndex +} + +/** + * Computes the normalized position of [coordinate] within the segment + * `[axis[lowerIndex], axis[upperIndex]]`: `0.0` exactly at [lowerIndex], `1.0` exactly at + * [upperIndex]. Returns `0.0` when `lowerIndex == upperIndex`, avoiding a + * division by a zero span. + * + * @param axis grid coordinates along one dimension, strictly ascending. + * @param lowerIndex the lower boundary index, as returned by [bracketIndices]. + * @param upperIndex the upper boundary index, as returned by [bracketIndices]. + * @param coordinate the query coordinate, within `[axis[lowerIndex], axis[upperIndex]]`. + * @return the interpolation weight in `[0.0, 1.0]`, or `0.0` if the indices are equal. + */ +internal fun weight(axis: DoubleArray, lowerIndex: Int, upperIndex: Int, coordinate: Double): Double { + if (lowerIndex == upperIndex) return 0.0 + val totalSpan = axis[upperIndex] - axis[lowerIndex] + val distanceFromLower = coordinate - axis[lowerIndex] + return distanceFromLower / totalSpan +} diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestAxes.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestAxes.kt new file mode 100644 index 0000000000..851f7eeac5 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestAxes.kt @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe + +class TestAxes : StringSpec({ + + // indices 0..3 + val axis = doubleArrayOf(10.0, 20.0, 30.0, 40.0) + val singleNode = doubleArrayOf(42.0) + + // nearestIndex tests + "nearestIndex returns the exact index on a node hit" { + nearestIndex(axis, 10.0) shouldBe 0 + nearestIndex(axis, 20.0) shouldBe 1 + nearestIndex(axis, 40.0) shouldBe 3 + } + + "nearestIndex picks the closer node between two nodes" { + // closer to 20 + nearestIndex(axis, 22.0) shouldBe 1 + // closer to 30 + nearestIndex(axis, 28.0) shouldBe 2 + } + + "nearestIndex resolves an exact tie to the lower index" { + nearestIndex(axis, 25.0) shouldBe 1 + } + + "nearestIndex clamps below the first node" { + nearestIndex(axis, 5.0) shouldBe 0 + nearestIndex(axis, -1000.0) shouldBe 0 + } + + "nearestIndex clamps above the last node" { + nearestIndex(axis, 50.0) shouldBe 3 + nearestIndex(axis, 1000.0) shouldBe 3 + } + + "nearestIndex always returns 0 on a single-node axis" { + nearestIndex(singleNode, 42.0) shouldBe 0 + nearestIndex(singleNode, 0.0) shouldBe 0 + nearestIndex(singleNode, 100.0) shouldBe 0 + } + + "nearestIndex rejects an empty axis" { + shouldThrow { nearestIndex(DoubleArray(0), 5.0) } + } + + "nearestIndex rejects a NaN query" { + shouldThrow { nearestIndex(axis, Double.NaN) } + } + + // bracketIndices tests + "bracketIndices returns a degenerate pair on a node hit" { + bracketIndices(axis, 30.0) shouldBe (2 to 2) + } + + "bracketIndices brackets an interior coordinate" { + bracketIndices(axis, 23.0) shouldBe (1 to 2) + } + + "bracketIndices clamps outside the boundaries" { + bracketIndices(axis, 5.0) shouldBe (0 to 0) + bracketIndices(axis, 50.0) shouldBe (3 to 3) + } + + "bracketIndices is degenerate on a single-node axis" { + bracketIndices(singleNode, 42.0) shouldBe (0 to 0) + bracketIndices(singleNode, 0.0) shouldBe (0 to 0) + bracketIndices(singleNode, 100.0) shouldBe (0 to 0) + } + + "bracketIndices rejects an empty axis and a NaN query" { + shouldThrow { bracketIndices(DoubleArray(0), 5.0) } + shouldThrow { bracketIndices(axis, Double.NaN) } + } + + // weight tests + "weight is 0.0 at the lower node and 1.0 at the upper node" { + weight(axis, 1, 2, 20.0) shouldBe 0.0 + weight(axis, 1, 2, 30.0) shouldBe 1.0 + } + + "weight is the normalized position within the segment" { + // sets a tolerance for the results + val tolerance = 1e-9 + weight(axis, 1, 2, 25.0) shouldBe (0.5 plusOrMinus tolerance) + weight(axis, 1, 2, 23.0) shouldBe (0.3 plusOrMinus tolerance) + } + + "weight is 0.0 on a degenerate bracket" { + weight(axis, 2, 2, 30.0) shouldBe 0.0 + } +}) From eeff0e50a5e2c136d53a6ed07663fd0d67d29d76 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 21 Jun 2026 23:46:43 +0200 Subject: [PATCH 007/118] feat(geospatial): add SpatialInterpolationStrategy and an enum with buil-in implementation --- .../strategy/SpatialInterpolationStrategy.kt | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt new file mode 100644 index 0000000000..0c8c35548d --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import it.unibo.alchemist.model.geospatial.reading.RasterGrid + +/** + * Strategy for **spatial interpolation**: given a grid slice and a position assumed to be *inside* + * its extent, produces a value by combining nearby cells. It is a functional interface, so a custom rule + * can be passed as a lambda. The built-in implementations live in [SpatialInterpolation]. + * + * Bounds checking and out-of-extent handling are NOT this strategy's concern: they belong to + * [SpatialExtrapolationStrategy] instead when the point is outside. + */ +fun interface SpatialInterpolationStrategy { + + /** + * @param grid the slice to sample. + * @param latitude latitude of the point, assumed to be inside of [grid]. + * @param longitude longitude of the point, assumed to be inside of [grid]. + * @return the interpolated value, or [Double.NaN] if the involved cells are missing. + */ + fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double +} + +/** + * Built-in spatial interpolation strategies. + */ +enum class SpatialInterpolation : SpatialInterpolationStrategy { + + /** + * Value of the nearest cell to the point. + */ + NEAREST { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double { + val nearestLatitudeIndex = nearestIndex(grid.latitudes, latitude) + val nearestLongitudeIndex = nearestIndex(grid.longitudes, longitude) + return grid.valueAt(nearestLatitudeIndex, nearestLongitudeIndex) + } + }, + + /** + * Bilinear interpolation over the 4 cells surrounding the point. If any of the 4 corners is + * missing ([Double.NaN]), returns [Double.NaN] rather than blending in a fictitious value. + * Interpolation is performed by applying these calculations: + * [Bilinear Interpolation Wikipedia](https://en.wikipedia.org/wiki/Bilinear_interpolation) + */ + BILINEAR { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double { + val (lowerLatitudeIndex, upperLatitudeIndex) = bracketIndices(grid.latitudes, latitude) + val (lowerLongitudeIndex, upperLongitudeIndex) = bracketIndices(grid.longitudes, longitude) + + /** + * axes are sorted in ascending order, so lower latiduce index = south, + * lower longitude index = west + */ + val southWestValue = grid.valueAt(lowerLatitudeIndex, lowerLongitudeIndex) + val southEastValue = grid.valueAt(lowerLatitudeIndex, upperLongitudeIndex) + val northWestValue = grid.valueAt(upperLatitudeIndex, lowerLongitudeIndex) + val northEastValue = grid.valueAt(upperLatitudeIndex, upperLongitudeIndex) + + val anyCornerMissing = southWestValue.isNaN() || southEastValue.isNaN() || + northWestValue.isNaN() || northEastValue.isNaN() + if (anyCornerMissing) return Double.NaN + + val longitudeWeight = weight( + grid.longitudes, + lowerLongitudeIndex, + upperLongitudeIndex, + longitude, + ) + val latitudeWeight = weight( + grid.latitudes, + lowerLatitudeIndex, + upperLatitudeIndex, + latitude, + ) + + // interpolates along longitude first (one value per latitude row), then along latitude. + val southValue = southWestValue + (southEastValue - southWestValue) * longitudeWeight + val northValue = northWestValue + (northEastValue - northWestValue) * longitudeWeight + return southValue + (northValue - southValue) * latitudeWeight + } + }, +} From f6a027ffe7acc33b5835821ef67ba7f0234a1be1 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 21 Jun 2026 23:47:30 +0200 Subject: [PATCH 008/118] test(geospatial): add test for SpatialInterpolation enum --- .../strategy/TestSpatialInterpolation.kt | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialInterpolation.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialInterpolation.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialInterpolation.kt new file mode 100644 index 0000000000..0517f93262 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialInterpolation.kt @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe +import it.unibo.alchemist.model.geospatial.reading.ArrayRasterGrid + +class TestSpatialInterpolation : StringSpec({ + + val tolerance = 1e-9 + + /* + * 2x2 grid with NON-AFFINE corners (i.e. NE != SE + NW - SW), so that bilinear + * exercises the u*v cross terms. + * It can be represented as follows: + * W(lon 100) E(lon 200) + * N(lat 20) 20(NW) 100(NE) + * S(lat 10) 0(SW) 10(SE) + */ + val southWest = 0.0 + val southEast = 10.0 + val northWest = 20.0 + val northEast = 100.0 + val grid = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0), + longitudes = doubleArrayOf(100.0, 200.0), + // row-major representation + values = doubleArrayOf(southWest, southEast, northWest, northEast), + ) + + /* + * 3x3 grid used to verify correct cell selection in a multi-cell grid: on an affine field + * bilinear reproduces the value exactly, so the expected result is just lat + lon. + */ + val affineGrid = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0, 30.0), + longitudes = doubleArrayOf(100.0, 200.0, 300.0), + values = doubleArrayOf( + 110.0, 210.0, 310.0, // lat = 10 + 120.0, 220.0, 320.0, // lat = 20 + 130.0, 230.0, 330.0, // lat = 30 + ), + ) + + // NEAREST spatial interpolation strategy tests + "NEAREST returns the cell value on an exact node hit" { + SpatialInterpolation.NEAREST.valueAt(grid, 10.0, 100.0) shouldBe southWest + SpatialInterpolation.NEAREST.valueAt(grid, 20.0, 200.0) shouldBe northEast + } + + "NEAREST picks the closest cell" { + // closest to south-west + SpatialInterpolation.NEAREST.valueAt(grid, 12.0, 110.0) shouldBe southWest + // closest to north-east + SpatialInterpolation.NEAREST.valueAt(grid, 18.0, 190.0) shouldBe northEast + } + + "NEAREST resolves an exact tie to the lower index on each axis" { + // lat 15 and lon 150 are both exactly between nodes, so the lower index should be picked + SpatialInterpolation.NEAREST.valueAt(grid, 15.0, 150.0) shouldBe southWest + } + + "NEAREST selects the correct cell in a multi-cell grid" { + // nearest node to (22, 290) is (20, 300) + SpatialInterpolation.NEAREST.valueAt(affineGrid, 22.0, 290.0) shouldBe 320.0 + } + + "NEAREST returns NaN when the nearest cell is missing" { + val gridWithHole = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0), + longitudes = doubleArrayOf(100.0, 200.0), + values = doubleArrayOf(Double.NaN, 10.0, 20.0, 100.0), // south-west is missing + ) + SpatialInterpolation.NEAREST.valueAt(gridWithHole, 11.0, 105.0).shouldBeNaN() + } + + // BILINEAR spatial interpolation strategy tests + "BILINEAR returns the exact corner value on a node hit (no interpolation)" { + SpatialInterpolation.BILINEAR.valueAt(grid, 10.0, 100.0) shouldBe southWest + SpatialInterpolation.BILINEAR.valueAt(grid, 10.0, 200.0) shouldBe southEast + SpatialInterpolation.BILINEAR.valueAt(grid, 20.0, 100.0) shouldBe northWest + SpatialInterpolation.BILINEAR.valueAt(grid, 20.0, 200.0) shouldBe northEast + } + + "BILINEAR at the cell center is the average of the four corners" { + // (0 + 10 + 20 + 100) / 4 = 32.5 + SpatialInterpolation.BILINEAR.valueAt(grid, 15.0, 150.0) shouldBe (32.5 plusOrMinus tolerance) + } + + "BILINEAR along an edge degenerates to linear interpolation on that edge" { + /* + * lat = 10 lies exactly on the south edge: only SW and SE should contribute in the calculation. + * halfway in longitude: (0 + 10) / 2 = 5 + */ + SpatialInterpolation.BILINEAR.valueAt(grid, 10.0, 150.0) shouldBe (5.0 plusOrMinus tolerance) + } + + "BILINEAR weights the corners by their fractional distance" { + /* + * u = (125 - 100)/100 = 0.25, v = (12.5 - 10)/10 = 0.25 + * 0*0.75*0.75 + 10*0.25*0.75 + 20*0.75*0.25 + 100*0.25*0.25 = 1.875 + 3.75 + 6.25 = 11.875 + */ + SpatialInterpolation.BILINEAR.valueAt(grid, 12.5, 125.0) shouldBe (11.875 plusOrMinus tolerance) + } + + "BILINEAR reproduces an affine field exactly and brackets the correct cell" { + // f(lat, lon) = lat + lon, so (15, 250) should be 265 + SpatialInterpolation.BILINEAR.valueAt(affineGrid, 15.0, 250.0) shouldBe (265.0 plusOrMinus tolerance) + } + + "BILINEAR propagates NaN when any of the four corners is missing" { + for (i in 0..3) { + val values = doubleArrayOf(0.0, 10.0, 20.0, 30.0) + values[i] = Double.NaN + val gridWithHole = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0), + longitudes = doubleArrayOf(100.0, 200.0), + values = values, + ) + SpatialInterpolation.BILINEAR.valueAt(gridWithHole, 15.0, 150.0).shouldBeNaN() + } + } +}) From b7e3cc73a175e2bab161ebad9e869445ebc0af78 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 22 Jun 2026 22:59:14 +0200 Subject: [PATCH 009/118] feat(geospatial): add SpatialExtrapolationStrategy interface and built-in strategies --- .../strategy/SpatialExtrapolation.kt | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolation.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolation.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolation.kt new file mode 100644 index 0000000000..8039c1c50b --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolation.kt @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import it.unibo.alchemist.model.geospatial.reading.RasterGrid + +/** + * Strategy for **spatial extrapolation**: what to return when the requested position is *outside* + * the geographic extent of the grid (the data is entirely absent there, as opposed to a missing + * in-grid cell, which is a [MissingValueStrategy] concern). Built-in implementations in + * [SpatialExtrapolation]. + */ +fun interface SpatialExtrapolationStrategy { + + /** + * @param grid the slice from which the point stands out. + * @param latitude latitude of the point, outside the bounds of [grid]. + * @param longitude longitude of the point, outside the bounds of [grid]. + * @return the value to return outside the extent. + */ + fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double +} + +/** + * Built-in spatial extrapolation strategies. + */ +enum class SpatialExtrapolation : SpatialExtrapolationStrategy { + + /** + * Always returns `0.0` outside the grid. + */ + ZERO { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double = 0.0 + }, + + /** + * Always returns `Double.NaN` outside the grid. + */ + NAN { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double = Double.NaN + }, + + /** + * Clamps the point to the nearest edge cell and returns its value. + * + * **Note:** the returned value is the raw edge cell, which may itself be [Double.NaN] if that edge + * cell is a fill value. [SpatialSampler] applies the [MissingValueStrategy] only on the in-extent + * path, so a NaN produced here is *not* substituted: it is returned as is. + */ + NEAREST_EDGE { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double { + // nearestIndex clamps an out-of-bounds coordinate to the boundary index, i.e. the edge cell. + val edgeLatitudeIndex = nearestIndex(grid.latitudes, latitude) + val edgeLongitudeIndex = nearestIndex(grid.longitudes, longitude) + return grid.valueAt(edgeLatitudeIndex, edgeLongitudeIndex) + } + }, +} From 5290a0ae39f9c78414841465acfdfab0d44d91f4 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 22 Jun 2026 23:09:58 +0200 Subject: [PATCH 010/118] refactor(geospatial): modified file name from SpatialExtrapolation to SpatialExtrapolationStrategy --- .../{SpatialExtrapolation.kt => SpatialExtrapolationStrategy.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/{SpatialExtrapolation.kt => SpatialExtrapolationStrategy.kt} (100%) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolation.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolationStrategy.kt similarity index 100% rename from alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolation.kt rename to alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolationStrategy.kt From 365d9e20694a86b66b3f431b341aa5cd85cc758e Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 22 Jun 2026 23:22:44 +0200 Subject: [PATCH 011/118] test(geospatial): add tests for SpatialExtrapolation built-in strategies --- .../strategy/TestSpatialExtrapolation.kt | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialExtrapolation.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialExtrapolation.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialExtrapolation.kt new file mode 100644 index 0000000000..ac8a013098 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialExtrapolation.kt @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe +import it.unibo.alchemist.model.geospatial.reading.ArrayRasterGrid + +class TestSpatialExtrapolation : StringSpec({ + + val grid = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0, 30.0), + longitudes = doubleArrayOf(100.0, 200.0, 300.0), + values = doubleArrayOf( + 110.0, 210.0, 310.0, // lat 10 + 120.0, 220.0, 320.0, // lat 20 + 130.0, 230.0, 330.0, // lat 30 + ), + ) + + /* + * Fixed out-of-bounds points covering the 8 exterior regions (4 corners and 4 edges). The third + * value is the expected NEAREST_EDGE result. The in-bounds axis coordinate always sits + * exactly on a node, so there is no nearest/tie ambiguity to resolve. + */ + val outOfBoundsPoints = listOf( + Triple(5.0, 50.0, 110.0), // SW corner + Triple(5.0, 200.0, 210.0), // S edge + Triple(5.0, 350.0, 310.0), // SE corner + Triple(20.0, 350.0, 320.0), // E edge + Triple(40.0, 350.0, 330.0), // NE corner + Triple(40.0, 200.0, 230.0), // N edge + Triple(40.0, 50.0, 130.0), // NW corner + Triple(20.0, 50.0, 120.0), // W edge + ) + + // ZERO spatial extrapolation strategy test + "ZERO returns 0.0 for every out-of-bounds point" { + for ((latitude, longitude, _) in outOfBoundsPoints) { + withClue("at ($latitude, $longitude)") { + SpatialExtrapolation.ZERO.valueAt(grid, latitude, longitude) shouldBe 0.0 + } + } + } + + // NAN spatial extrapolation strategy test + "NAN returns NaN for every out-of-bounds point" { + for ((latitude, longitude, _) in outOfBoundsPoints) { + withClue("at ($latitude, $longitude)") { + SpatialExtrapolation.NAN.valueAt(grid, latitude, longitude).shouldBeNaN() + } + } + } + + // NEAREST_EDGE spatial extrapolation strategy tests + "NEAREST_EDGE clamps to the nearest edge node for every out-of-bounds region" { + for ((latitude, longitude, expectedEdgeValue) in outOfBoundsPoints) { + withClue("at ($latitude, $longitude)") { + SpatialExtrapolation.NEAREST_EDGE.valueAt(grid, latitude, longitude) shouldBe expectedEdgeValue + } + } + } + + "NEAREST_EDGE returns a raw missing edge cell (NaN) without substitution" { + val gridWithMissingCorner = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0, 30.0), + longitudes = doubleArrayOf(100.0, 200.0, 300.0), + values = doubleArrayOf( + Double.NaN, 210.0, 310.0, // SW corner missing + 120.0, 220.0, 320.0, + 130.0, 230.0, 330.0, + ), + ) + // both points lie beyond the south-west corner, so both should clamp to the (missing) SW node + for ((latitude, longitude) in listOf(5.0 to 50.0, 8.0 to 90.0)) { + withClue("at ($latitude, $longitude)") { + SpatialExtrapolation.NEAREST_EDGE.valueAt(gridWithMissingCorner, latitude, longitude).shouldBeNaN() + } + } + } +}) From 3f1745f36761aba019aa5125a4caa3a9cecac876 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 22 Jun 2026 23:59:31 +0200 Subject: [PATCH 012/118] feat(geospatial): add MissingValueStrategy and built-in implementation --- .../strategy/MissingValueStrategy.kt | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt new file mode 100644 index 0000000000..f1a862ef49 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import it.unibo.alchemist.model.geospatial.reading.RasterGrid + +/** + * Strategy for **missing values**: what to return when the position is *inside* the grid but the + * cell (or the cells involved in an interpolation) are fill values ([Double.NaN]). Distinct from + * [SpatialExtrapolationStrategy] because the cause differs (in-grid missing data, not a position + * outside the extent of the grid). Built-in implementations in [MissingValue]. + */ +fun interface MissingValueStrategy { + + /** + * @param grid the slice on which a missing value was encountered. + * @param latitude latitude of the point. + * @param longitude longitude of the point. + * @return the value to return in place of the missing one. + */ + fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double +} + +/** + * Built-in missing-value strategies. + */ +enum class MissingValue : MissingValueStrategy { + + /** + * Propagates [Double.NaN], i.e. the missing data. + */ + NAN { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double = Double.NaN + }, + + /** + * Returns `0.0` on missing data. + */ + ZERO { + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double = 0.0 + }, +} From ef952662db31b0bdf9801dc6b60573c337cc0c4c Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 00:20:25 +0200 Subject: [PATCH 013/118] test(geospatial): add tests for MissingValue strategies --- .../geospatial/strategy/TestMissingValue.kt | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestMissingValue.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestMissingValue.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestMissingValue.kt new file mode 100644 index 0000000000..247113b23c --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestMissingValue.kt @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe +import it.unibo.alchemist.model.geospatial.reading.ArrayRasterGrid + +class TestMissingValue : StringSpec({ + + // 3x3 grid, the values are irrelevant + val grid = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0, 30.0), + longitudes = doubleArrayOf(100.0, 200.0, 300.0), + values = doubleArrayOf( + 110.0, 210.0, 310.0, + 120.0, 220.0, 320.0, + 130.0, 230.0, 330.0, + ), + ) + + /* + * A fixed set of probe points, deliberately mixing in-grid and out-of-grid coordinates: a + * MissingValueStrategy must ignore position entirely, so the result must be constant across all + * of them. (Whether the strategy is actually reached only on the in-extent path is a + * SpatialSampler concern) + */ + val probePoints = listOf( + 15.0 to 150.0, // inside the grid + 20.0 to 200.0, // exactly on a node + 5.0 to 50.0, // out of bounds, south-west + 40.0 to 350.0, // out of bounds, north-east + ) + + // NAN missing value strategy test + "NAN returns NaN regardless of grid and position" { + for ((latitude, longitude) in probePoints) { + withClue("at ($latitude, $longitude)") { + MissingValue.NAN.valueAt(grid, latitude, longitude).shouldBeNaN() + } + } + } + + // ZERO missing value strategy test + "ZERO returns 0.0 regardless of grid and position" { + for ((latitude, longitude) in probePoints) { + withClue("at ($latitude, $longitude)") { + MissingValue.ZERO.valueAt(grid, latitude, longitude) shouldBe 0.0 + } + } + } +}) From 655b7c2ad623480de387357335ffb55aadc7426e Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 15:47:03 +0200 Subject: [PATCH 014/118] feat(geospatial): add SpatialSampler to orchestrate spatial strategies --- .../geospatial/strategy/SpatialSampler.kt | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt new file mode 100644 index 0000000000..78de26eee3 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import it.unibo.alchemist.model.geospatial.reading.RasterGrid + +/** + * Orchestrates the three spatial strategies (in-extent interpolation, + * out-of-extent extrapolation, missing-value handling) and exposes a single "sample a slice at a + * point" operation. + * + * @property interpolation strategy used when the point is inside the grid extent. + * @property outOfBounds strategy used when the point is outside the grid extent. + * @property missing strategy used when interpolation generates a missing value. + */ +class SpatialSampler( + private val interpolation: SpatialInterpolationStrategy, + private val outOfBounds: SpatialExtrapolationStrategy, + private val missing: MissingValueStrategy, +) { + + /** + * Samples [grid] at the given position, applying in order: bounds check, interpolation and + * missing-value handling. + * + * **Note**: missing-value substitution is applied **only** on the in-extent path. A + * value produced by the out-of-extent strategy is returned as is, even if it is [Double.NaN]. + * + * @param grid the spatial slice to sample. + * @param latitude latitude of the point. + * @param longitude longitude of the point. + * @return the sampled value according to the configured strategies. + */ + fun sample(grid: RasterGrid, latitude: Double, longitude: Double): Double { + val isLatitudeInBounds = latitude in grid.latitudes.first()..grid.latitudes.last() + val isLongitudeInBounds = longitude in grid.longitudes.first()..grid.longitudes.last() + + if (!isLatitudeInBounds || !isLongitudeInBounds) { + return outOfBounds.valueAt(grid, latitude, longitude) + } + + val interpolatedValue = interpolation.valueAt(grid, latitude, longitude) + return if (interpolatedValue.isNaN()) missing.valueAt(grid, latitude, longitude) else interpolatedValue + } +} From 58fd9ab175e96439eb80a759fe17a7fce6ee3e43 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 16:48:35 +0200 Subject: [PATCH 015/118] test(geospatial): add tests for SpatialSampler (spatial strategies orchestrator) --- .../geospatial/strategy/TestSpatialSampler.kt | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialSampler.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialSampler.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialSampler.kt new file mode 100644 index 0000000000..0988e76a7a --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestSpatialSampler.kt @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe +import it.unibo.alchemist.model.geospatial.reading.ArrayRasterGrid +import it.unibo.alchemist.model.geospatial.reading.RasterGrid + +/** + * A recording strategy that satisfies all three spatial interfaces (they share the same signature), + * so that this class can play any role. Each instance counts how many times it has been invoked, + * allowing tests to accurately verify which strategy the SpatialSampler has reached. + * + * @param result the fixed value that this mock strategy must return. + */ +private class SpyStrategy(private val result: Double) : + SpatialInterpolationStrategy, SpatialExtrapolationStrategy, MissingValueStrategy { + var calls = 0 + private set + + override fun valueAt(grid: RasterGrid, latitude: Double, longitude: Double): Double { + calls++ + return result + } +} + +class TestSpatialSampler : StringSpec({ + + val tolerance = 1e-9 + + /* + * Affine 3x3 grid (node value = lat + lon). Bounds: lat 10..30, lon 100..300. + */ + val grid = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0, 30.0), + longitudes = doubleArrayOf(100.0, 200.0, 300.0), + values = doubleArrayOf( + 110.0, 210.0, 310.0, + 120.0, 220.0, 320.0, + 130.0, 230.0, 330.0, + ), + ) + + /* + * 2x2 grid with the south-west corner missing, used to exercise NaN handling and the + * out-of-bounds vs missing asymmetry. Bounds: lat 10..20, lon 100..200. + */ + val holeGrid = ArrayRasterGrid( + latitudes = doubleArrayOf(10.0, 20.0), + longitudes = doubleArrayOf(100.0, 200.0), + values = doubleArrayOf(Double.NaN, 10.0, 20.0, 100.0), + ) + + val outOfBoundsPoints = listOf( + 5.0 to 150.0, + 35.0 to 150.0, + 15.0 to 50.0, + 15.0 to 350.0, + 5.0 to 50.0, + 180.0 to 180.0, + ) + + val boundaryPoints = listOf( + 10.0 to 150.0, + 30.0 to 150.0, + 15.0 to 100.0, + 15.0 to 300.0, + 10.0 to 100.0, + 30.0 to 300.0, + ) + + // tests with mock strategies + "in bounds with a finite interpolation returns it and consults neither extrapolation nor missing" { + val interpolation = SpyStrategy(42.0) + val outOfBounds = SpyStrategy(99.0) + val missing = SpyStrategy(-1.0) + val sampler = SpatialSampler(interpolation, outOfBounds, missing) + + sampler.sample(grid, 15.0, 150.0) shouldBe 42.0 + interpolation.calls shouldBe 1 + outOfBounds.calls shouldBe 0 + missing.calls shouldBe 0 + } + + "in bounds with a NaN interpolation delegates to missing and returns its value" { + val interpolation = SpyStrategy(Double.NaN) + val outOfBounds = SpyStrategy(99.0) + val missing = SpyStrategy(7.0) + val sampler = SpatialSampler(interpolation, outOfBounds, missing) + + sampler.sample(grid, 15.0, 150.0) shouldBe 7.0 + interpolation.calls shouldBe 1 + missing.calls shouldBe 1 + outOfBounds.calls shouldBe 0 + } + + "out of bounds delegates to extrapolation and consults neither interpolation nor missing" { + for ((latitude, longitude) in outOfBoundsPoints) { + val interpolation = SpyStrategy(42.0) + val outOfBounds = SpyStrategy(99.0) + val missing = SpyStrategy(-1.0) + val sampler = SpatialSampler(interpolation, outOfBounds, missing) + + withClue("at ($latitude, $longitude)") { + sampler.sample(grid, latitude, longitude) shouldBe 99.0 + outOfBounds.calls shouldBe 1 + interpolation.calls shouldBe 0 + missing.calls shouldBe 0 + } + } + } + + "points exactly on the grid boundary are treated as IN bounds" { + for ((latitude, longitude) in boundaryPoints) { + val interpolation = SpyStrategy(42.0) + val outOfBounds = SpyStrategy(99.0) + val missing = SpyStrategy(-1.0) + val sampler = SpatialSampler(interpolation, outOfBounds, missing) + + withClue("at ($latitude, $longitude)") { + sampler.sample(grid, latitude, longitude) shouldBe 42.0 + interpolation.calls shouldBe 1 + outOfBounds.calls shouldBe 0 + } + } + } + + // tests with real strategies + "composed NEAREST + ZERO + NAN samples in bounds and zeroes out of bounds" { + val sampler = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.ZERO, + MissingValue.NAN, + ) + sampler.sample(grid, 20.0, 200.0) shouldBe 220.0 + sampler.sample(grid, 5.0, 50.0) shouldBe 0.0 + } + + "composed BILINEAR propagates an in-grid NaN through MissingValue.NAN" { + val sampler = SpatialSampler( + SpatialInterpolation.BILINEAR, + SpatialExtrapolation.ZERO, + MissingValue.NAN, + ) + /* + * (15, 150) is the cell center and SW corner is NaN, so bilinear should return NaN which + * should be propagated by MissingValue + */ + sampler.sample(holeGrid, 15.0, 150.0).shouldBeNaN() + } + + "composed BILINEAR substitutes an in-grid NaN through MissingValue.ZERO" { + val sampler = SpatialSampler( + SpatialInterpolation.BILINEAR, + SpatialExtrapolation.ZERO, + MissingValue.ZERO, + ) + sampler.sample(holeGrid, 15.0, 150.0) shouldBe (0.0 plusOrMinus tolerance) + } + + "an in-extent NaN is substituted by missing, but an out-of-extent NaN is not" { + // in bounds: nearest hits the missing SW node, MissingValue.ZERO substitutes + val inExtent = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.ZERO, + MissingValue.ZERO, + ) + inExtent.sample(holeGrid, 10.0, 100.0) shouldBe 0.0 + + // out of bounds: NEAREST_EDGE clamps to the same missing SW node. NaN returned WITHOUT substitution + val outOfExtent = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.NEAREST_EDGE, + MissingValue.ZERO, + ) + outOfExtent.sample(holeGrid, 5.0, 50.0).shouldBeNaN() + } +}) From 893f1cdddabcb4d3bf3e5ab6ae3042076e6ab0d3 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 17:33:10 +0200 Subject: [PATCH 016/118] feat(geospatial): add TemporalInterpolationStrategy to interpolate between spatially resolved values --- .../strategy/TemporalInterpolationStrategy.kt | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt new file mode 100644 index 0000000000..bee70e94a7 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +/** + * Strategy for **temporal interpolation**: given the two **already** spatially resolved values of the + * slices bracketing the current time, plus a normalized weight, produces the value at the current + * time. Built-in implementations in [TemporalInterpolation]. + */ +fun interface TemporalInterpolationStrategy { + + /** + * @param valueBefore value at the slice at or immediately before the current time. + * @param valueAfter value at the slice immediately after the current time. + * @param weight normalized position in `[0.0, 1.0]`: `0.0` coincides with [valueBefore], `1.0` with [valueAfter]. + * @return the interpolated value. + */ + fun interpolate(valueBefore: Double, valueAfter: Double, weight: Double): Double +} + +/** + * Built-in temporal interpolation strategies. + */ +enum class TemporalInterpolation : TemporalInterpolationStrategy { + + /** + * Linear blend between the two adjacent values. + */ + LINEAR { + override fun interpolate(valueBefore: Double, valueAfter: Double, weight: Double): Double = + valueBefore + (valueAfter - valueBefore) * weight + }, + + /** + * Always the value of the earlier slice. + */ + BEFORE { + override fun interpolate(valueBefore: Double, valueAfter: Double, weight: Double): Double = valueBefore + }, + + /** + * Always the value of the later slice. + */ + AFTER { + override fun interpolate(valueBefore: Double, valueAfter: Double, weight: Double): Double = valueAfter + }, + + /** + * The value of the slice temporally closer to the current time. + * Ties are resolve to `valueAfter`. + */ + NEAREST { + override fun interpolate(valueBefore: Double, valueAfter: Double, weight: Double): Double = + if (weight < 0.5) valueBefore else valueAfter + }, +} From 8c2285fb6fba42038ad4c5a8ad08660620d2dd3d Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 17:52:43 +0200 Subject: [PATCH 017/118] test(geospatial): add tests for TemporalInterpolation strategies --- .../strategy/TestTemporalInterpolation.kt | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalInterpolation.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalInterpolation.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalInterpolation.kt new file mode 100644 index 0000000000..07e930e161 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalInterpolation.kt @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe + +class TestTemporalInterpolation : StringSpec({ + + val tolerance = 1e-9 + + val valueBefore = 10.0 + val valueAfter = 30.0 + + // weights spanning in [0, 1] + val steps = 20 + val weights = (0..steps).map { it.toDouble() / steps } + + // LINEAR temporal interpolation strategy tests + "LINEAR returns the endpoints at weight 0 and 1" { + TemporalInterpolation.LINEAR.interpolate( + valueBefore, + valueAfter, + 0.0, + ) shouldBe (valueBefore plusOrMinus tolerance) + + TemporalInterpolation.LINEAR.interpolate( + valueBefore, + valueAfter, + 1.0, + ) shouldBe (valueAfter plusOrMinus tolerance) + } + + "LINEAR blends proportionally to the weight" { + for (weight in weights) { + val expected = valueBefore + (valueAfter - valueBefore) * weight + withClue("at weight $weight") { + TemporalInterpolation.LINEAR.interpolate( + valueBefore, + valueAfter, + weight, + ) shouldBe (expected plusOrMinus tolerance) + } + } + } + + // BEFORE temporal interpolation strategy test + "BEFORE always returns the earlier value regardless of weight" { + for (weight in weights) { + withClue("at weight $weight") { + TemporalInterpolation.BEFORE.interpolate(valueBefore, valueAfter, weight) shouldBe valueBefore + } + } + } + + // AFTER temporal interpolation strategy test + "AFTER always returns the later value regardless of weight" { + for (weight in weights) { + withClue("at weight $weight") { + TemporalInterpolation.AFTER.interpolate(valueBefore, valueAfter, weight) shouldBe valueAfter + } + } + } + + // NEAREST temporal interpolation strategy tests + "NEAREST returns the earlier value when the weight is below 0.5" { + for (weight in listOf(0.0, 0.25, 0.499)) { + withClue("at weight $weight") { + TemporalInterpolation.NEAREST.interpolate(valueBefore, valueAfter, weight) shouldBe valueBefore + } + } + } + + "NEAREST returns the later value when the weight is at or above 0.5" { + for (weight in listOf(0.5, 0.75, 1.0)) { + withClue("at weight $weight") { + TemporalInterpolation.NEAREST.interpolate(valueBefore, valueAfter, weight) shouldBe valueAfter + } + } + } + + "NEAREST resolves the exact 0.5 tie to the later value" { + TemporalInterpolation.NEAREST.interpolate(valueBefore, valueAfter, 0.5) shouldBe valueAfter + } +}) From e5a74db6a6c9d23153f59df1d685dde3fb7cf8ca Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 18:47:39 +0200 Subject: [PATCH 018/118] feat(geospatial): add TemporalExtrapolation strategies --- .../strategy/TemporalExtrapolationStrategy.kt | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt new file mode 100644 index 0000000000..71cf8fd719 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +/** + * Strategy for **temporal extrapolation**: what to return when the simulation time is *outside* the + * time interval covered by the data. Built-in implementations in [TemporalExtrapolation]. + */ +fun interface TemporalExtrapolationStrategy { + + /** + * @param currentTime current simulation time (as a `Double`), outside the range of [sliceTimes]. + * @param sliceTimes times of the available slices, ascending. For custom strategies that + * need the actual instants. + * @param sampleSlice given a slice index, returns that slice's value **already spatially sampled** + * at the requested position. + * @return the extrapolated value. + */ + fun valueAt(currentTime: Double, sliceTimes: List, sampleSlice: (sliceIndex: Int) -> Double): Double +} + +/** + * Built-in temporal extrapolation strategies. + */ +enum class TemporalExtrapolation : TemporalExtrapolationStrategy { + + /** + * Holds the value of the **last** available slice. + */ + LAST { + override fun valueAt( + currentTime: Double, + sliceTimes: List, + sampleSlice: (sliceIndex: Int) -> Double, + ): Double = sampleSlice(sliceTimes.lastIndex) + }, + + /** + * Holds the value of the **first** available slice. + */ + FIRST { + override fun valueAt( + currentTime: Double, + sliceTimes: List, + sampleSlice: (sliceIndex: Int) -> Double, + ): Double = sampleSlice(0) + }, +} From 445a255b42efba45dff34244a3d5352d701e73d4 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 23 Jun 2026 19:06:23 +0200 Subject: [PATCH 019/118] test(geospatial): add tests for TemporalExtrapolation strategies --- .../strategy/TestTemporalExtrapolation.kt | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalExtrapolation.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalExtrapolation.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalExtrapolation.kt new file mode 100644 index 0000000000..47541bd2d0 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/strategy/TestTemporalExtrapolation.kt @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.strategy + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe + +class TestTemporalExtrapolation : StringSpec({ + + /* + * Simple recording callback: it remembers the index it was asked for, and returns a value + * derived from that index so a single assertion proves both which index was sampled and + * that the strategy returns the callback's result. + */ + class IndexRecorder { + var sampledIndex: Int? = null + private set + + val sampleSlice: (Int) -> Double = { index -> + sampledIndex = index + index * 10.0 + } + } + + val sliceTimes = listOf(100.0, 200.0, 300.0, 400.0) + + // out of range probe times: one before the first slice, one after the last. + val timeBeforeRange = 50.0 + val timeAfterRange = 500.0 + + "LAST samples the last slice index regardless of the current time" { + for (currentTime in listOf(timeBeforeRange, timeAfterRange)) { + val recorder = IndexRecorder() + withClue("at time $currentTime") { + TemporalExtrapolation.LAST.valueAt( + currentTime, + sliceTimes, + recorder.sampleSlice, + ) shouldBe sliceTimes.lastIndex * 10 + recorder.sampledIndex shouldBe sliceTimes.lastIndex + } + } + } + + "FIRST samples the first slice index regardless of the current time" { + for (currentTime in listOf(timeBeforeRange, timeAfterRange)) { + val recorder = IndexRecorder() + withClue("at time $currentTime") { + TemporalExtrapolation.FIRST.valueAt( + currentTime, + sliceTimes, + recorder.sampleSlice, + ) shouldBe 0.0 + recorder.sampledIndex shouldBe 0 + } + } + } + + "LAST and FIRST both sample index 0 when there is a single slice" { + val singleTime = listOf(100.0) + val lastRecorder = IndexRecorder() + val firstRecorder = IndexRecorder() + + TemporalExtrapolation.LAST.valueAt( + timeAfterRange, + singleTime, + lastRecorder.sampleSlice, + ) + TemporalExtrapolation.FIRST.valueAt( + timeBeforeRange, + singleTime, + firstRecorder.sampleSlice, + ) + + lastRecorder.sampledIndex shouldBe 0 + firstRecorder.sampledIndex shouldBe 0 + } +}) From 1c19ebf502cd964e6d5bf9ca077fae6223176bd6 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Wed, 24 Jun 2026 18:44:37 +0200 Subject: [PATCH 020/118] refactor(geospatial): make raster and strategy types Serializable --- .../alchemist/model/geospatial/reading/ArrayRasterGrid.kt | 6 ++++++ .../alchemist/model/geospatial/reading/CdmTimedGrid.kt | 2 ++ .../unibo/alchemist/model/geospatial/reading/RasterGrid.kt | 4 +++- .../unibo/alchemist/model/geospatial/reading/TimedGrid.kt | 3 ++- .../model/geospatial/strategy/MissingValueStrategy.kt | 3 ++- .../geospatial/strategy/SpatialExtrapolationStrategy.kt | 3 ++- .../geospatial/strategy/SpatialInterpolationStrategy.kt | 3 ++- .../alchemist/model/geospatial/strategy/SpatialSampler.kt | 7 ++++++- .../geospatial/strategy/TemporalExtrapolationStrategy.kt | 4 +++- .../geospatial/strategy/TemporalInterpolationStrategy.kt | 4 +++- 10 files changed, 31 insertions(+), 8 deletions(-) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt index e4a6a95f49..d60b21fd4b 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/ArrayRasterGrid.kt @@ -9,6 +9,8 @@ package it.unibo.alchemist.model.geospatial.reading +import java.io.Serializable + /** * In-memory implementation of [RasterGrid], format-agnostic. * @@ -27,4 +29,8 @@ class ArrayRasterGrid( ) : RasterGrid { override fun valueAt(latIndex: Int, lonIndex: Int): Double = values[latIndex * longitudes.size + lonIndex] + + private companion object { + private const val serialVersionUID = 1L + } } diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt index 6aee3e5c41..af91d290e3 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/CdmTimedGrid.kt @@ -200,6 +200,8 @@ class CdmTimedGrid(directory: Path, variableName: String? = null) : TimedGrid { private companion object { + private const val serialVersionUID = 1L + /** * Selects the variable to read from the dataset. * diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt index f7003928d7..428c830ec2 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/RasterGrid.kt @@ -9,6 +9,8 @@ package it.unibo.alchemist.model.geospatial.reading +import java.io.Serializable + /** * A single 2D spatial "slice" of data on a regular geographic grid (latitude/longitude). * @@ -20,7 +22,7 @@ package it.unibo.alchemist.model.geospatial.reading * from +89.95 to −59.95) must normalize internally, so that strategies never need to * reason about axis direction. */ -interface RasterGrid { +interface RasterGrid : Serializable { /** * Latitudes of grid nodes, in degrees, sorted in ascending order. */ diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt index 642a992f09..8f32514ea8 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/reading/TimedGrid.kt @@ -9,6 +9,7 @@ package it.unibo.alchemist.model.geospatial.reading +import java.io.Serializable import java.time.Instant /** @@ -20,7 +21,7 @@ import java.time.Instant * real-world timestamps to simulation time once at construction; [grid] is index-based * because temporal interpolation operates on adjacent indices, not timestamps directly. */ -interface TimedGrid { +interface TimedGrid : Serializable { /** * Real-world timestamps of each slice, strictly ascending, aligned with [grid]. */ diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt index f1a862ef49..e138648d2d 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/MissingValueStrategy.kt @@ -10,6 +10,7 @@ package it.unibo.alchemist.model.geospatial.strategy import it.unibo.alchemist.model.geospatial.reading.RasterGrid +import java.io.Serializable /** * Strategy for **missing values**: what to return when the position is *inside* the grid but the @@ -17,7 +18,7 @@ import it.unibo.alchemist.model.geospatial.reading.RasterGrid * [SpatialExtrapolationStrategy] because the cause differs (in-grid missing data, not a position * outside the extent of the grid). Built-in implementations in [MissingValue]. */ -fun interface MissingValueStrategy { +fun interface MissingValueStrategy : Serializable { /** * @param grid the slice on which a missing value was encountered. diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolationStrategy.kt index 8039c1c50b..276ab32e9d 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolationStrategy.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialExtrapolationStrategy.kt @@ -10,6 +10,7 @@ package it.unibo.alchemist.model.geospatial.strategy import it.unibo.alchemist.model.geospatial.reading.RasterGrid +import java.io.Serializable /** * Strategy for **spatial extrapolation**: what to return when the requested position is *outside* @@ -17,7 +18,7 @@ import it.unibo.alchemist.model.geospatial.reading.RasterGrid * in-grid cell, which is a [MissingValueStrategy] concern). Built-in implementations in * [SpatialExtrapolation]. */ -fun interface SpatialExtrapolationStrategy { +fun interface SpatialExtrapolationStrategy : Serializable { /** * @param grid the slice from which the point stands out. diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt index 0c8c35548d..b6bd62d992 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialInterpolationStrategy.kt @@ -10,6 +10,7 @@ package it.unibo.alchemist.model.geospatial.strategy import it.unibo.alchemist.model.geospatial.reading.RasterGrid +import java.io.Serializable /** * Strategy for **spatial interpolation**: given a grid slice and a position assumed to be *inside* @@ -19,7 +20,7 @@ import it.unibo.alchemist.model.geospatial.reading.RasterGrid * Bounds checking and out-of-extent handling are NOT this strategy's concern: they belong to * [SpatialExtrapolationStrategy] instead when the point is outside. */ -fun interface SpatialInterpolationStrategy { +fun interface SpatialInterpolationStrategy : Serializable { /** * @param grid the slice to sample. diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt index 78de26eee3..11e42f2d12 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/SpatialSampler.kt @@ -10,6 +10,7 @@ package it.unibo.alchemist.model.geospatial.strategy import it.unibo.alchemist.model.geospatial.reading.RasterGrid +import java.io.Serializable /** * Orchestrates the three spatial strategies (in-extent interpolation, @@ -24,7 +25,7 @@ class SpatialSampler( private val interpolation: SpatialInterpolationStrategy, private val outOfBounds: SpatialExtrapolationStrategy, private val missing: MissingValueStrategy, -) { +) : Serializable { /** * Samples [grid] at the given position, applying in order: bounds check, interpolation and @@ -49,4 +50,8 @@ class SpatialSampler( val interpolatedValue = interpolation.valueAt(grid, latitude, longitude) return if (interpolatedValue.isNaN()) missing.valueAt(grid, latitude, longitude) else interpolatedValue } + + private companion object { + private const val serialVersionUID = 1L + } } diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt index 71cf8fd719..570b3500dd 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalExtrapolationStrategy.kt @@ -9,11 +9,13 @@ package it.unibo.alchemist.model.geospatial.strategy +import java.io.Serializable + /** * Strategy for **temporal extrapolation**: what to return when the simulation time is *outside* the * time interval covered by the data. Built-in implementations in [TemporalExtrapolation]. */ -fun interface TemporalExtrapolationStrategy { +fun interface TemporalExtrapolationStrategy : Serializable { /** * @param currentTime current simulation time (as a `Double`), outside the range of [sliceTimes]. diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt index bee70e94a7..14db06b483 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/strategy/TemporalInterpolationStrategy.kt @@ -9,12 +9,14 @@ package it.unibo.alchemist.model.geospatial.strategy +import java.io.Serializable + /** * Strategy for **temporal interpolation**: given the two **already** spatially resolved values of the * slices bracketing the current time, plus a normalized weight, produces the value at the current * time. Built-in implementations in [TemporalInterpolation]. */ -fun interface TemporalInterpolationStrategy { +fun interface TemporalInterpolationStrategy : Serializable { /** * @param valueBefore value at the slice at or immediately before the current time. From 469001bcfc1656133cc434f34bd53403768ff223 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Wed, 24 Jun 2026 19:49:57 +0200 Subject: [PATCH 021/118] feat(geospatial): add not fully implemented GeoRasterLayer (costructor with data download/caching needs to be added) --- .../model/geospatial/GeoRasterLayer.kt | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/GeoRasterLayer.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/GeoRasterLayer.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/GeoRasterLayer.kt new file mode 100644 index 0000000000..f79f02ff24 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/GeoRasterLayer.kt @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial + +import it.unibo.alchemist.model.Environment +import it.unibo.alchemist.model.GeoPosition +import it.unibo.alchemist.model.Layer +import it.unibo.alchemist.model.geospatial.reading.CdmTimedGrid +import it.unibo.alchemist.model.geospatial.reading.TimedGrid +import it.unibo.alchemist.model.geospatial.strategy.MissingValue +import it.unibo.alchemist.model.geospatial.strategy.SpatialExtrapolation +import it.unibo.alchemist.model.geospatial.strategy.SpatialInterpolation +import it.unibo.alchemist.model.geospatial.strategy.SpatialSampler +import it.unibo.alchemist.model.geospatial.strategy.TemporalExtrapolation +import it.unibo.alchemist.model.geospatial.strategy.TemporalExtrapolationStrategy +import it.unibo.alchemist.model.geospatial.strategy.TemporalInterpolation +import it.unibo.alchemist.model.geospatial.strategy.TemporalInterpolationStrategy +import it.unibo.alchemist.model.geospatial.strategy.bracketIndices +import it.unibo.alchemist.model.geospatial.strategy.weight +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +/** + * A [Layer] that exposes a [Double] raster value for any [GeoPosition] as a function + * of the current simulation time, backed by temporal NetCDF/GRIB data. + * On [getValue], the layer knows the perceived simulation time at that moment via + * [Environment.simulationOrNull], so no reaction is needed to "advance" the layer. + * + * When [getValue] is called, the two time slices that enclose the current time are selected, + * their values are evaluated using [spatial], and they are blended using [temporalInterpolation]. + * If the simulation time falls outside the calculated simulation time range, + * [temporalExtrapolation] is applied. + * + * Once the construction is complete, real-world time [Instant] are converted proportionally to + * simulation time using [timeOrigin] and [timeScale]. + * + * Two constructors are currently available (a third one is planned once the + * `acquisition` subpackage is implemented. SEE THE TODO below): + * - **Primary** (this): accepts a ready-built [TimedGrid]; intended for unit tests. + * - **Directory**: accepts a local [Path], builds [CdmTimedGrid] internally (no network I/O). + * + * @property environment simulation environment. Used only to read the current time via [Environment.simulationOrNull]. + * @property data temporal raster series backing this layer. + * @param timeOrigin real-world [Instant] corresponding to simulation `Time.ZERO`. Defaults to the first instant in [data]. + * @param timeScale real-world [Duration] of one simulation time unit. Defaults to one hour. + * @param spatial composer of the three spatial strategies (in-extent interpolation, + * out-of-extent extrapolation, missing-value handling). Defaults to `NEAREST / ZERO / NAN`. + * @param temporalInterpolation strategy for blending adjacent slice values. Defaults to [TemporalInterpolation.LINEAR]. + * @param temporalExtrapolation strategy for values outside the covered time range. Defaults to [TemporalExtrapolation.LAST]. + */ +@Suppress("serial") +class GeoRasterLayer( + private val environment: Environment<*, GeoPosition>, + private val data: TimedGrid, + timeOrigin: Instant? = null, + timeScale: Duration = Duration.ofHours(1), + private val spatial: SpatialSampler = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.ZERO, + MissingValue.NAN, + ), + private val temporalInterpolation: TemporalInterpolationStrategy = TemporalInterpolation.LINEAR, + private val temporalExtrapolation: TemporalExtrapolationStrategy = TemporalExtrapolation.LAST, +) : Layer { + + init { + require(data.instants.isNotEmpty()) { "TimedGrid must have at least one time instant" } + } + + /** + * Real-world instant corresponding to simulation `Time.ZERO`. + * Equals `timeOrigin` if provided, otherwise the first instant in [data]. + */ + private val origin: Instant = timeOrigin ?: data.instants.first() + + /** + * Conversion of real-world time instants to simulation time (as an array of `Double`). + * The conversion occurs only during construction and takes into account the `origin` + * and the `timeScale`. + */ + private val sliceTimes: DoubleArray = data.instants + .map { toSimulationTime(it, origin, timeScale) } + .toDoubleArray() + + /** + * It reads the simulation time, finds the adjacent time slices using [bracketIndices], + * samples each of them using [spatial], and blends the result using [temporalInterpolation]. + * It delegates value retrieval to [temporalExtrapolation] if the current simulation time + * is outside the simulation time range. + * + * **Note**: If the simulation has not yet started, the simulation time taken into account is `0.0`. + * + * @param position the geographic position to query. + * @return the interpolated or extrapolated [Double] value. + */ + override fun getValue(position: GeoPosition): Double { + val t = environment.simulationOrNull?.time?.toDouble() ?: 0.0 + + /* + * a function that retrieves the value resolved by the SpatialSampler for + * the grid at a given index. + */ + val sample = { i: Int -> + spatial.sample( + data.grid(i), + position.latitude, + position.longitude, + ) + } + + // the simulation time is outside the calculated simulation time range. Extrapolates the value. + if (t < sliceTimes.first() || t > sliceTimes.last()) { + return temporalExtrapolation.valueAt( + t, + sliceTimes.toList(), + sample, + ) + } + + // the indices of the spatial slices that enclose time t. + val (sliceIndexBefore, sliceIndexAfter) = bracketIndices(sliceTimes, t) + + // the time t falls exactly on a slice. + if (sliceIndexBefore == sliceIndexAfter) return sample(sliceIndexBefore) + + /* + * the time t lies between two distinct slices. + * Applies the interpolation strategy between the two values measured in the slices. + */ + val blendWeight = weight( + sliceTimes, + sliceIndexBefore, + sliceIndexAfter, + t, + ) + return temporalInterpolation.interpolate( + sample(sliceIndexBefore), + sample(sliceIndexAfter), + blendWeight, + ) + } + + /** + * Constructs a [GeoRasterLayer] from a local directory of data files (no network I/O). + * + * Builds a [CdmTimedGrid] from [dataDirectory] and delegates to the primary constructor. + * (Intended for integration tests with pre downloaded static NetCDF files) + * + * @param environment simulation environment. + * @param dataDirectory directory containing one or more homogeneous data files (same variable + * and spatial grid, disjoint time ranges). + * @param variable variable name inside the file (e.g. `"dis24"`, the GRIB shortName). + * If `null`, it is auto-detected as the unique `(time, lat, lon)` variable. + * @param timeOrigin see primary constructor. + * @param timeScale see primary constructor. + * @param spatial see primary constructor. + * @param temporalInterpolation see primary constructor. + * @param temporalExtrapolation see primary constructor. + */ + constructor( + environment: Environment<*, GeoPosition>, + dataDirectory: Path, + variable: String? = null, + timeOrigin: Instant? = null, + timeScale: Duration = Duration.ofHours(1), + spatial: SpatialSampler = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.ZERO, + MissingValue.NAN, + ), + temporalInterpolation: TemporalInterpolationStrategy = TemporalInterpolation.LINEAR, + temporalExtrapolation: TemporalExtrapolationStrategy = TemporalExtrapolation.LAST, + ) : this( + environment, + CdmTimedGrid(dataDirectory, variable), + timeOrigin, + timeScale, + spatial, + temporalInterpolation, + temporalExtrapolation, + ) + + /* + * TODO!!! YAML constructor (to add after the acquisition subpackage is implemented). + * Reminder: it will accept endpoint, dataset, REQUEST!!, credentials and cache parameters. + * (As strings?) + */ + private companion object { + private const val serialVersionUID = 1L + } +} + +/** + * Converts a real-world [Instant] to a simulation time [Double] using millisecond precision. + * + * @param instant timestamp to convert. + * @param origin the instant that maps to `0.0` in simulation time. + * @param scale duration of one simulation time unit. + */ +private fun toSimulationTime(instant: Instant, origin: Instant, scale: Duration): Double = + Duration.between(origin, instant).toMillis().toDouble() / scale.toMillis().toDouble() From 9f4bfd3e3eca717d7d792cf98e0aebd1d0e5e391 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Wed, 24 Jun 2026 22:41:15 +0200 Subject: [PATCH 022/118] refactor(geospatial): move writeTestNetcdf function in a shared test utils file --- alchemist-geospatial/build.gradle.kts | 1 + .../alchemist/model/geospatial/TestUtils.kt | 108 ++++++++++++++++++ .../geospatial/reading/TestCdmTimedGrid.kt | 92 +-------------- 3 files changed, 110 insertions(+), 91 deletions(-) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestUtils.kt diff --git a/alchemist-geospatial/build.gradle.kts b/alchemist-geospatial/build.gradle.kts index afe4b6e9fc..fb951d6863 100644 --- a/alchemist-geospatial/build.gradle.kts +++ b/alchemist-geospatial/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { runtimeOnly(libs.cdm.grib) implementation(libs.gson) implementation(libs.guava) + implementation(libs.mockk) implementation(libs.slf4j) testImplementation(alchemist("test")) } diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestUtils.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestUtils.kt new file mode 100644 index 0000000000..2e8854eda8 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestUtils.kt @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial + +import it.unibo.alchemist.model.geospatial.reading.CdmTimedGrid +import java.nio.file.Path +import ucar.ma2.Array as UcarArray +import ucar.ma2.ArrayDouble +import ucar.ma2.ArrayFloat +import ucar.ma2.DataType +import ucar.nc2.Attribute +import ucar.nc2.write.NetcdfFormatWriter + +/** + * Writes a minimal CF-compliant NetCDF-3 file to [path] for use in geospatial module tests. + * + * The file contains: + * - a `time` axis with units "hours since [timeEpoch]" and values [timeHours]. + * - a `latitude` axis (CF axis Y) with values [lats]. May be descending to test normalization. + * - a `longitude` axis (CF axis X) with values [lons]. + * - a single data variable named [variableName] with `_FillValue` set to [fillValue]. + * + * @param path destination file. The parent directory must already exist. + * @param lats latitude values in degrees, stored in the file as-is. Pass a descending array + * to test axis normalization in [CdmTimedGrid]. + * @param lons longitude values in degrees, stored in the file as-is. + * @param timeHours time offsets in hours from [timeEpoch], one value per time step. + * @param rawValues flat `time * lat * lon` array in row-major order, or `null` to use the + * default pattern `iLat * 10 + iLon` repeated across all time steps. If provided, its size + * must equal `timeHours.size * lats.size * lons.size`. + * @param fillValue written as the `_FillValue` attribute on the data variable. When the file + * is opened in CDM enhanced mode, cells matching this value are replaced with [Double.NaN]. + * @param variableName short name of the data variable as it appears in the file (e.g. `"dis24"`). + * @param timeEpoch reference date for the CF `units` attribute, in the format + * `"yyyy-MM-dd HH:mm"` (e.g. `"2024-01-01 00:00"`). Values in [timeHours] are interpreted + * as hours elapsed since this date. + */ +internal fun writeTestNetcdf( + path: Path, + lats: FloatArray, + lons: FloatArray, + timeHours: DoubleArray, + rawValues: FloatArray? = null, + fillValue: Float = -9999f, + variableName: String = "dis24", + timeEpoch: String = "2024-01-01 00:00", +) { + // defines the schema on the builder + val builder = NetcdfFormatWriter.createNewNetcdf3(path.toString()) + + builder.addDimension("time", timeHours.size) + builder.addDimension("latitude", lats.size) + builder.addDimension("longitude", lons.size) + + // CF compliant axis + builder.addVariable("time", DataType.DOUBLE, "time").apply { + addAttribute(Attribute("units", "hours since $timeEpoch")) + addAttribute(Attribute("calendar", "standard")) + addAttribute(Attribute("axis", "T")) + } + builder.addVariable("latitude", DataType.FLOAT, "latitude").apply { + addAttribute(Attribute("units", "degrees_north")) + addAttribute(Attribute("axis", "Y")) + } + builder.addVariable("longitude", DataType.FLOAT, "longitude").apply { + addAttribute(Attribute("units", "degrees_east")) + addAttribute(Attribute("axis", "X")) + } + builder.addVariable(variableName, DataType.FLOAT, "time latitude longitude").apply { + addAttribute(Attribute("_FillValue", fillValue)) + } + + // creates the file and enters write mode + builder.build().use { writer -> + val timeArr = ArrayDouble.D1(timeHours.size) + timeHours.forEachIndexed { i, v -> timeArr.set(i, v) } + writer.write("time", timeArr) + + val latArr = ArrayFloat.D1(lats.size) + lats.forEachIndexed { i, v -> latArr.set(i, v) } + writer.write("latitude", latArr) + + val lonArr = ArrayFloat.D1(lons.size) + lons.forEachIndexed { i, v -> lonArr.set(i, v) } + writer.write("longitude", lonArr) + + val flat = rawValues ?: FloatArray(timeHours.size * lats.size * lons.size) { idx -> + val iLat = (idx / lons.size) % lats.size + val iLon = idx % lons.size + (iLat * 10 + iLon).toFloat() + } + writer.write( + variableName, + UcarArray.factory( + DataType.FLOAT, + intArrayOf(timeHours.size, lats.size, lons.size), + flat, + ), + ) + } +} diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt index c1c75f86ee..83362abaf9 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt @@ -14,14 +14,9 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.shouldBeSortedWith import io.kotest.matchers.doubles.shouldBeNaN import io.kotest.matchers.shouldBe +import it.unibo.alchemist.model.geospatial.writeTestNetcdf import java.nio.file.Files import java.nio.file.Path -import ucar.ma2.Array as UcarArray -import ucar.ma2.ArrayDouble -import ucar.ma2.ArrayFloat -import ucar.ma2.DataType -import ucar.nc2.Attribute -import ucar.nc2.write.NetcdfFormatWriter class TestCdmTimedGrid : StringSpec({ @@ -146,91 +141,6 @@ class TestCdmTimedGrid : StringSpec({ } }) -/** - * Writes a minimal CF-compliant NetCDF-3 file to [path] to be used as a [CdmTimedGrid]. - * - * The file contains: - * - a `time` axis with units `"hours since 2024-01-01 00:00"` and values [timeHours]. - * - a `latitude` axis (CF axis Y) with values [lats]. May be descending to test normalization. - * - a `longitude` axis (CF axis X) with values [lons]. - * - a single data variable named [variableName] with `_FillValue` set to [fillValue]. - * - * @param path destination file. The parent (directory must already exist). - * @param lats latitude values in degrees, stored in the file as-is. Pass a descending array - * to test axis normalization in [CdmTimedGrid]. - * @param lons longitude values in degrees, stored in the file as-is. - * @param timeHours time offsets in hours from `2024-01-01 00:00`, one per time step. - * @param rawValues flat `time * lat * lon` data array in row-major order, or `null` to - * use the default pattern `iLat * 10 + iLon` repeated across all time steps. If provided, - * size must equal `nTime * lats.size * lons.size`. - * @param fillValue written as the `_FillValue` attribute on the data variable. When the file - * is opened in CDM enhanced mode, cells matching this value are replaced with [Double.NaN]. - * @param variableName short name of the data variable as it appears in the file (e.g. `"dis24"`). - */ -private fun writeTestNetcdf( - path: Path, - lats: FloatArray, - lons: FloatArray, - timeHours: DoubleArray, - rawValues: FloatArray? = null, - fillValue: Float = -9999f, - variableName: String = "dis24", -) { - // defines the schema on the builder - val builder = NetcdfFormatWriter.createNewNetcdf3(path.toString()) - - builder.addDimension("time", timeHours.size) - builder.addDimension("latitude", lats.size) - builder.addDimension("longitude", lons.size) - - // CF compliant axis - builder.addVariable("time", DataType.DOUBLE, "time").apply { - addAttribute(Attribute("units", "hours since 2024-01-01 00:00")) - addAttribute(Attribute("calendar", "standard")) - addAttribute(Attribute("axis", "T")) - } - builder.addVariable("latitude", DataType.FLOAT, "latitude").apply { - addAttribute(Attribute("units", "degrees_north")) - addAttribute(Attribute("axis", "Y")) - } - builder.addVariable("longitude", DataType.FLOAT, "longitude").apply { - addAttribute(Attribute("units", "degrees_east")) - addAttribute(Attribute("axis", "X")) - } - builder.addVariable(variableName, DataType.FLOAT, "time latitude longitude").apply { - addAttribute(Attribute("_FillValue", fillValue)) - } - - // creates the file and enters write mode - builder.build().use { writer -> - val timeArr = ArrayDouble.D1(timeHours.size) - timeHours.forEachIndexed { i, v -> timeArr.set(i, v) } - writer.write("time", timeArr) - - val latArr = ArrayFloat.D1(lats.size) - lats.forEachIndexed { i, v -> latArr.set(i, v) } - writer.write("latitude", latArr) - - val lonArr = ArrayFloat.D1(lons.size) - lons.forEachIndexed { i, v -> lonArr.set(i, v) } - writer.write("longitude", lonArr) - - val flat = rawValues ?: FloatArray(timeHours.size * lats.size * lons.size) { idx -> - val iLat = (idx / lons.size) % lats.size - val iLon = idx % lons.size - (iLat * 10 + iLon).toFloat() - } - writer.write( - variableName, - UcarArray.factory( - DataType.FLOAT, - intArrayOf(timeHours.size, lats.size, lons.size), - flat, - ), - ) - } -} - /** * Creates a NetCDF-3 file with * - latitudes: (10°, 20°, 30°) From 52be2828858e9eb3c95b93dc91257c9bb1718944 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Wed, 24 Jun 2026 22:55:15 +0200 Subject: [PATCH 023/118] chore(geospatial): remove mockk dependency from build.gradle file --- alchemist-geospatial/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/alchemist-geospatial/build.gradle.kts b/alchemist-geospatial/build.gradle.kts index fb951d6863..afe4b6e9fc 100644 --- a/alchemist-geospatial/build.gradle.kts +++ b/alchemist-geospatial/build.gradle.kts @@ -19,7 +19,6 @@ dependencies { runtimeOnly(libs.cdm.grib) implementation(libs.gson) implementation(libs.guava) - implementation(libs.mockk) implementation(libs.slf4j) testImplementation(alchemist("test")) } From dcd74f012404ecda4040e69aaa930469b28562ca Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Thu, 25 Jun 2026 18:53:55 +0200 Subject: [PATCH 024/118] test(geospatial): add tests for GeoRasterLayer --- .../model/geospatial/TestGeoRasterLayer.kt | 406 ++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt new file mode 100644 index 0000000000..6f1f50d039 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt @@ -0,0 +1,406 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.doubles.shouldBeNaN +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import it.unibo.alchemist.model.Environment +import it.unibo.alchemist.model.GeoPosition +import it.unibo.alchemist.model.geospatial.reading.ArrayRasterGrid +import it.unibo.alchemist.model.geospatial.reading.RasterGrid +import it.unibo.alchemist.model.geospatial.reading.TimedGrid +import it.unibo.alchemist.model.geospatial.strategy.MissingValue +import it.unibo.alchemist.model.geospatial.strategy.SpatialExtrapolation +import it.unibo.alchemist.model.geospatial.strategy.SpatialInterpolation +import it.unibo.alchemist.model.geospatial.strategy.SpatialSampler +import it.unibo.alchemist.model.geospatial.strategy.TemporalExtrapolation +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import kotlin.io.path.createTempDirectory + +private const val TOLERANCE = 1e-9 + +class TestGeoRasterLayer : StringSpec({ + + // fixed spatial grid + val lats = doubleArrayOf(44.0, 45.0, 46.0) + val lons = doubleArrayOf(11.0, 12.0, 13.0) + + val center: GeoPosition = mockk { + every { latitude } returns 45.0 + every { longitude } returns 12.0 + } + + /** + * Environment mock with the simulation fixed at time [t] + */ + fun envAt(t: Double): Environment = mockk { + every { simulationOrNull } returns mockk { + every { time } returns mockk { every { toDouble() } returns t } + } + } + + /** + * Environment mock whose simulation time is read from [currentT]. + */ + fun mutableEnv(currentT: () -> Double): Environment = mockk { + every { simulationOrNull } returns mockk { + every { time } returns mockk { every { toDouble() } answers { currentT() } } + } + } + + /** + * Environment mock where the simulation has not started yet. + */ + val envNoSim: Environment = mockk { + every { simulationOrNull } returns null + } + + /** + * A [RasterGrid] where every cell has a fixed [value]. + */ + fun flatGrid(value: Double): RasterGrid = ArrayRasterGrid( + lats, + lons, + DoubleArray(lats.size * lons.size) { value }, + ) + + /** + * Returns a [TimedGrid] made of flat slices. + * THe i-th slice has all cells equal to `sliceValue[i]`. + * Time steps are spaced [step] apart starting from [base]. + */ + fun syntheticGrid( + vararg sliceValues: Double, + base: Instant = Instant.EPOCH, + step: Duration = Duration.ofHours(1), + ): TimedGrid { + val instants = sliceValues.indices.map { i -> + base.plus(step.multipliedBy(i.toLong())) + } + val grids = sliceValues.map { v -> flatGrid(v) } + return object : TimedGrid { + override val instants = instants + override fun grid(index: Int): RasterGrid = grids[index] + } + } + + val tempDir: Path = createTempDirectory("geo-raster-layer-test") + + // cleans the temp directory after each test + afterSpec { + tempDir.toFile().deleteRecursively() + } + + "require fails on empty TimedGrid" { + val emptyGrid = object : TimedGrid { + override val instants: List = emptyList() + override fun grid(index: Int): RasterGrid = throw UnsupportedOperationException() + } + shouldThrow { + GeoRasterLayer(envAt(0.0), emptyGrid) + } + } + + // timeOrigin tests + "default timeOrigin maps the first instant to t=0.0" { + val timedGrid = syntheticGrid(7.0, 14.0) + val layer = GeoRasterLayer( + envAt(0.0), + timedGrid, + ) + withClue("t=0.0 should hit the first slice exactly") { + for (i in timedGrid.instants.indices) { + layer.getValue(center) shouldBe 7.0 + } + } + } + + "explicit timeOrigin shifts the temporal origin" { + val layer = GeoRasterLayer( + envAt(0.0), + syntheticGrid(10.0, 20.0, 30.0), + // instants: EPOCH, EPOCH+1h, EPOCH+2h + timeOrigin = Instant.EPOCH.plus(Duration.ofHours(1)), + ) + withClue("t=0.0 with shifted origin maps to the second slice = 20.0") { + layer.getValue(center) shouldBe 20.0 + } + } + + "explicit timeOrigin: t before shifted range extrapolates with LAST" { + // computed simulation times: -1.0, 0.0, 1.0, t=-2.0 is out of range + val layer = GeoRasterLayer( + envAt(-2.0), + syntheticGrid(10.0, 20.0, 30.0), + timeOrigin = Instant.EPOCH.plus(Duration.ofHours(1)), + ) + withClue("t=-2.0 < sliceTimes.first()=-1.0, extrapolates to last slice: 30.0") { + layer.getValue(center) shouldBe 30.0 + } + } + + // timescale tests + "timeScale PT30M: one real hour maps to t=2.0" { + /* + * instants in the file: EPOCH, EPOCH+1h + * sliceTimes should be [0.0, 2.0] with scale PT30M + */ + val layer = GeoRasterLayer( + envAt(1.0), + syntheticGrid(0.0, 10.0), + timeScale = Duration.ofMinutes(30), + ) + withClue("t=1.0 is halfway between 0.0 and 2.0, LINEAR blending should return 5.0") { + layer.getValue(center) shouldBe (5.0 plusOrMinus TOLERANCE) + } + } + + "timeScale PT6H: one real six-hour step maps to t=1.0" { + /* + * instants in the file: EPOCH, EPOCH+6h + * sliceTimes should be [0.0, 1.0] with scale PT6H + */ + val layer = GeoRasterLayer( + envAt(0.5), + syntheticGrid(0.0, 12.0, step = Duration.ofHours(6)), + timeScale = Duration.ofHours(6), + ) + withClue("t=0.5 halfway, LINEAR blending should return 6.0") { + layer.getValue(center) shouldBe (6.0 plusOrMinus TOLERANCE) + } + } + + // interpolation and extrapolation tests + "exact hits on slices produce values without temporal interpolation" { + val sliceValues = doubleArrayOf(1.0, 2.0, 3.0) + for (i in sliceValues.indices) { + val time = i.toDouble() + val layer = GeoRasterLayer( + envAt(time), + syntheticGrid(*sliceValues), + ) + withClue("t=$time hits slice $i = ${sliceValues[i]}") { + layer.getValue(center) shouldBe sliceValues[i] + } + } + } + + "temporal extrapolation is applied if simulation time is not in range" { + doubleArrayOf(-100.0, -1.0, 99.0, 999.0).forEach { time -> + val layer = GeoRasterLayer( + envAt(time), + syntheticGrid(5.0, 10.0, 15.0), + temporalExtrapolation = TemporalExtrapolation.LAST, + ) + layer.getValue(center) shouldBe 15.0 + } + } + + "simulationOrNull null falls back to t=0.0 and reads the first slice" { + val layer = GeoRasterLayer( + envNoSim, + syntheticGrid(42.0, 84.0), + ) + withClue("null simulation: t=0.0, first slice = 42.0") { + layer.getValue(center) shouldBe 42.0 + } + } + + "spatial extrapolation is applied if the position is out of bounds" { + val outside: GeoPosition = mockk { + every { latitude } returns 90.0 + every { longitude } returns 180.0 + } + val layer = GeoRasterLayer( + envAt(0.0), + syntheticGrid(99.0), + spatial = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.ZERO, + MissingValue.NAN, + ), + ) + withClue("outside extent with ZERO extrapolation returns 0.0") { + layer.getValue(outside) shouldBe 0.0 + } + } + + "MissingValue.NAN propagates NaN for in-grid missing cells" { + val values = DoubleArray(9) { 5.0 }.also { it[4] = Double.NaN } // center cell missing + val timedGrid = object : TimedGrid { + override val instants = listOf(Instant.EPOCH) + override fun grid(index: Int) = ArrayRasterGrid(lats, lons, values) + } + val layer = GeoRasterLayer( + envAt(0.0), + timedGrid, + spatial = SpatialSampler( + SpatialInterpolation.NEAREST, + SpatialExtrapolation.ZERO, + MissingValue.NAN, + ), + ) + withClue("center cell is NaN; MissingValue.NAN propagates NaN") { + layer.getValue(center).shouldBeNaN() + } + } + + // directory constructor with real NetCDF files tests + "directory constructor: single file, exact hit at t=0 with default timeOrigin" { + val dir = tempDir.resolve("single").also { it.toFile().mkdirs() } + /* + * slice values: + * slice 0: all 10.0 + * slice 1: all 20.0 + * slice 2: all 30.0 + */ + writeTestNetcdf( + path = dir.resolve("data.nc"), + lats = floatArrayOf(44f, 45f, 46f), + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(0.0, 1.0, 2.0), + rawValues = FloatArray(27) { idx -> ((idx / 9) + 1) * 10f }, + ) + + val layer = GeoRasterLayer(envAt(0.0), dir) + withClue("t=0.0 -> first slice -> all cells = 10.0") { + layer.getValue(center) shouldBe (10.0 plusOrMinus TOLERANCE) + } + } + + "directory constructor: linear interpolation between two slices in file" { + val dir = tempDir.resolve("interp").also { it.toFile().mkdirs() } + writeTestNetcdf( + path = dir.resolve("data.nc"), + lats = floatArrayOf(44f, 45f, 46f), + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(0.0, 1.0), + rawValues = FloatArray(18) { idx -> if (idx < 9) 0f else 10f }, + ) + + val layer = GeoRasterLayer(envAt(0.5), dir) + withClue("t=0.5 halfway between 0.0 and 10.0 -> 5.0") { + layer.getValue(center) shouldBe (5.0 plusOrMinus TOLERANCE) + } + } + + "directory constructor: descending latitude axis is normalised to ascending" { + val dir = tempDir.resolve("desc-lat").also { it.toFile().mkdirs() } + writeTestNetcdf( + path = dir.resolve("data.nc"), + lats = floatArrayOf(46f, 45f, 44f), // lats descending + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(0.0), + rawValues = FloatArray(9) { idx -> + when (idx / 3) { + 0 -> 1f + 1 -> 2f + else -> 3f + } + }, + ) + + val layer = GeoRasterLayer(envAt(0.0), dir) + val lat44: GeoPosition = mockk { + every { latitude } returns 44.0 + every { longitude } returns 12.0 + } + val lat46: GeoPosition = mockk { + every { latitude } returns 46.0 + every { longitude } returns 12.0 + } + + withClue("after normalisation, lat=44 (file-row 2) should return 3.0") { + layer.getValue(lat44) shouldBe (3.0 plusOrMinus TOLERANCE) + } + withClue("after normalisation, lat=46 (file-row 0) should return 1.0") { + layer.getValue(lat46) shouldBe (1.0 plusOrMinus TOLERANCE) + } + } + + "directory constructor: variable auto-detected when variableName is null" { + val dir = tempDir.resolve("auto-detect").also { it.toFile().mkdirs() } + writeTestNetcdf( + path = dir.resolve("data.nc"), + lats = floatArrayOf(44f, 45f, 46f), + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(0.0), + rawValues = FloatArray(9) { 7f }, + ) + + val layer = GeoRasterLayer(envAt(0.0), dir, variable = null) + withClue("auto-detected variable; all cells = 7.0") { + layer.getValue(center) shouldBe (7.0 plusOrMinus TOLERANCE) + } + } + + "directory constructor: explicit variable name is used correctly" { + val dir = tempDir.resolve("explicit-var").also { it.toFile().mkdirs() } + writeTestNetcdf( + path = dir.resolve("data.nc"), + lats = floatArrayOf(44f, 45f, 46f), + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(0.0), + rawValues = FloatArray(9) { 42f }, + variableName = "dis24", + ) + + val layer = GeoRasterLayer(envAt(0.0), dir, variable = "dis24") + withClue("explicit variable 'dis24'; all cells = 42.0") { + layer.getValue(center) shouldBe (42.0 plusOrMinus TOLERANCE) + } + } + + "directory constructor: two disjoint files are merged into a single ordered time series" { + val dir = tempDir.resolve("two-files").also { it.toFile().mkdirs() } + // file 1: t=0h (all values=10.0) and t=1h (all values=20.0) + writeTestNetcdf( + path = dir.resolve("part1.nc"), + lats = floatArrayOf(44f, 45f, 46f), + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(0.0, 1.0), + rawValues = FloatArray(18) { idx -> if (idx < 9) 10f else 20f }, + ) + // file 2: t=2h (all values=30.0) and t=3h (all values=40.0) + writeTestNetcdf( + path = dir.resolve("part2.nc"), + lats = floatArrayOf(44f, 45f, 46f), + lons = floatArrayOf(11f, 12f, 13f), + timeHours = doubleArrayOf(2.0, 3.0), + rawValues = FloatArray(18) { idx -> if (idx < 9) 30f else 40f }, + ) + var t = 0.0 + val layer = GeoRasterLayer(mutableEnv { t }, dir) + + (0..3).forEach { + val time = it.toDouble() + val expectedTime = (it + 1) * 10.0 + t = time + + withClue("t=$time. Exact hit on merged series = $expectedTime") { + layer.getValue(center) shouldBe (expectedTime plusOrMinus TOLERANCE) + } + } + + // interpolation across the file boundary: t=1.5 should return the mean with LINEAR + t = 1.5 + withClue("t=1.5 crosses file boundary. LINEAR blend of 20.0 and 30.0 = 25.0") { + layer.getValue(center) shouldBe (25.0 plusOrMinus TOLERANCE) + } + } +}) From 06317cc1ca9333fd73a58d816ec02d89fb188de1 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 00:08:21 +0200 Subject: [PATCH 025/118] feat(geospatial): add CanonicalJson singleton to convert any structure into a key-sorted JSON string --- .../geospatial/acquisition/CanonicalJson.kt | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CanonicalJson.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CanonicalJson.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CanonicalJson.kt new file mode 100644 index 0000000000..003038d2f2 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CanonicalJson.kt @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import java.util.TreeMap + +/** + * Deterministic JSON encoding used to derive stable cache keys and submit bodies. + * + * "Canonical" here means exactly one normalization: **map keys are sorted recursively** at every + * nesting level, so a request encodes identically regardless of the key order it was authored in. + * **List order is preserved**, since in CDS/EWDS it is semantic (e.g. `area: [N, W, S, E]` + * has a different meaning than `[W, N, E, S]`). + */ +internal object CanonicalJson { + + private val gson: Gson = GsonBuilder().serializeNulls().create() + + /** + * Encodes [value] as canonical JSON. + * + * @param value a serializable value, typically a nested [Map]/[List]. + * @return its canonical JSON representation: map keys sorted recursively, list order untouched. + */ + fun encode(value: Any): String = gson.toJson(canonicalize(value)) + + /** + * Recursively rewrites [value] into a form Gson serializes deterministically: every [Map] + * becomes a key-sorted [TreeMap], every [List] keeps its order, + * scalars (strings, booleans, numbers, nulls) are left untouched. + */ + private fun canonicalize(value: Any?): Any? = when (value) { + // maintains the keys sorted + is Map<*, *> -> TreeMap().apply { + value.forEach { (key, mapValue) -> + // recursively evaluates nested structures + put(key.toString(), canonicalize(mapValue)) + } + } + is List<*> -> value.map(::canonicalize) + else -> value + } +} From 74efaff4c96b9fd6f0659b0e4b097164f7ea936a Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 00:21:32 +0200 Subject: [PATCH 026/118] test(geospatial): add tests for CanonicalJson key ordering --- .../acquisition/TestCanonicalJson.kt | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCanonicalJson.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCanonicalJson.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCanonicalJson.kt new file mode 100644 index 0000000000..1202a0f4cb --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCanonicalJson.kt @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe + +class TestCanonicalJson : StringSpec({ + + "map keys are sorted, producing a deterministic result" { + CanonicalJson.encode(mapOf("b" to "1", "a" to "2")) shouldBe """{"a":"2","b":"1"}""" + } + + "the same map written with different key order encodes identically" { + val items = listOf("dataset" to "x", "variable" to listOf("dis24"), "year" to "2024") + val map1 = mapOf(*items.toTypedArray()) + val map2 = mapOf(*items.reversed().toTypedArray()) + CanonicalJson.encode(map1) shouldBe CanonicalJson.encode(map2) + } + + "key sorting recurses into nested maps" { + val a = mapOf("outer" to mapOf("b" to "1", "a" to "2")) + val b = mapOf("outer" to mapOf("a" to "2", "b" to "1")) + CanonicalJson.encode(a) shouldBe CanonicalJson.encode(b) + } + + "list order is significant: reordering elements yields a different encoding" { + val list = listOf(50.0, 5.0, 45.0, 10.0) + val nwse = mapOf("area" to list) + val shuffled = mapOf("area" to list.reversed()) + CanonicalJson.encode(nwse) shouldNotBe CanonicalJson.encode(shuffled) + } + + "list order is preserved verbatim, including zero-padded strings" { + CanonicalJson.encode(mapOf("day" to listOf("02", "01"))) shouldBe """{"day":["02","01"]}""" + } + + "explicit nulls are encoded, distinguishing a present but null-key from an absent one" { + CanonicalJson.encode(mapOf("a" to null)) shouldNotBe CanonicalJson.encode(emptyMap()) + } + + "a realistic GloFAS request is order-stable across its keys" { + val request = mapOf( + "system_version" to listOf("version_3_1"), + "hydrological_model" to listOf("lisflood"), + "variable" to listOf("river_discharge_in_the_last_24_hours"), + "hyear" to listOf("2024"), + "hmonth" to listOf("06"), + "hday" to listOf("10"), + "data_format" to "netcdf", + ) + val reordered = mapOf( + "data_format" to "netcdf", + "variable" to listOf("river_discharge_in_the_last_24_hours"), + "hday" to listOf("10"), + "hmonth" to listOf("06"), + "hyear" to listOf("2024"), + "hydrological_model" to listOf("lisflood"), + "system_version" to listOf("version_3_1"), + ) + CanonicalJson.encode(request) shouldBe CanonicalJson.encode(reordered) + } +}) From ace407d3c33ed5bfd74b27c41bc4ed92bd4be4d2 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 14:51:16 +0200 Subject: [PATCH 027/118] feat(geospatial): add CacheKey interface to get a deterministic cache entry --- .../model/geospatial/acquisition/CacheKey.kt | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheKey.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheKey.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheKey.kt new file mode 100644 index 0000000000..5eb23eade7 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheKey.kt @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +/** + * Deterministic identity of a request, used by [CacheManager] as the directory name for its + * cache entry. Implemented by request types (e.g. [CopernicusRequest], [BBBikeRequest]). + * + * [CacheManager] depends on this interface and nothing else, so it stays unaware + * of HTTP and of the individual provider APIs. + */ +interface CacheKey { + + /** + * @return a **deterministic, file-system-safe** directory name derived *only* from what + * determines the downloaded bytes, so that equal content always maps to the same name. The + * returned name must be a single path segment: no separators (`/`, `\`), and safe on Linux, + * macOS, and Windows. + */ + fun toFileName(): String +} From 40d22081e5dd449c53e22258ee424e34f4522373 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 15:40:04 +0200 Subject: [PATCH 028/118] feat(geospatial): add FileNames utility file; add function to sanitize file/dirs names --- .../model/geospatial/acquisition/FileNames.kt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/FileNames.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/FileNames.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/FileNames.kt new file mode 100644 index 0000000000..93cd6df7c5 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/FileNames.kt @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +/** + * Renders this `String` as a single file-system-safe path segment: every character outside `[A-Za-z0-9._-]` + * is replaced with `_`. Distinct strings may collapse to the same output (e.g. `"a b"` and `"a_b"`). + * + * @return this string sanitized (i.e. all non alphatical/numerical characters replaced by `_`) + */ +internal fun String.toFileSystemSafe(): String = this.replace(Regex("[^A-Za-z0-9._-]"), "_") From 35eecf412203afe277b61017539e5381f05c821d Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 15:50:49 +0200 Subject: [PATCH 029/118] test(geospatial): add tests for String.toFileSystemSafe utility --- .../geospatial/acquisition/TestFileNames.kt | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestFileNames.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestFileNames.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestFileNames.kt new file mode 100644 index 0000000000..487ef37845 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestFileNames.kt @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldMatch +import io.kotest.matchers.string.shouldNotContain + +class TestFileNames : StringSpec({ + + // toFileSystemSafe extension function tests + "a plain name with only safe characters is left unchanged" { + "cems-glofas-historical".toFileSystemSafe() shouldBe "cems-glofas-historical" + "ERA5_2024.v3".toFileSystemSafe() shouldBe "ERA5_2024.v3" + } + + "spaces are replaced" { + "New York".toFileSystemSafe() shouldBe "New_York" + } + + "path separators are replaced" { + "a/b".toFileSystemSafe() shouldBe "a_b" + "a\\b".toFileSystemSafe() shouldBe "a_b" + } + + "Windows-illegal characters are replaced" { + // < > : " | ? * are illegal in a Windows filename + """a:bd"e|f?g*h""".toFileSystemSafe() shouldBe "a_b_c_d_e_f_g_h" + } + + "non-ASCII letters are replaced" { + "Forlì-Cesena".toFileSystemSafe() shouldBe "Forl_-Cesena" + } + + "output never contains a path separator, for any input" { + listOf("a/b", "a\\b", "/", "\\", "C:/x", "../../etc").forEach { raw -> + raw.toFileSystemSafe().shouldNotContain("/") + raw.toFileSystemSafe().shouldNotContain("\\") + } + } + + "output contains only allowlisted characters" { + """!"£$%&/()=?^@#°""".toFileSystemSafe() shouldMatch Regex("^[A-Za-z0-9._-]*$") + } + + "sanitizing a sanitized name changes nothing" { + val once = "a b/c:d".toFileSystemSafe() + once.toFileSystemSafe() shouldBe once + } +}) From be128f0c86d4e77dec47cfc9e7ecdbd0c85686b5 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 16:30:31 +0200 Subject: [PATCH 030/118] feat(geospatial): add CopernicusRequest cache key for ECMWF requests --- .../acquisition/CopernicusRequest.kt | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt new file mode 100644 index 0000000000..745cadcd37 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import java.security.MessageDigest + +/** + * A request to an ECMWF data store (CDS or EWDS): carries the request **identity** and **what to + * download**. + * + * @property dataset dataset identifier (e.g. `"cems-glofas-historical"`). + * @property inputs the **opaque** request map: the selection (variables, dates, area, format, ...) + * of parameters used for the request to the datastore. It is intentionally untyped because the fields + * vary widely per dataset (ERA5 uses `year/month/day`, GloFAS uses `hyear/hmonth/hday/system_version/...`); + * modeling them as fixed fields would be brittle. + */ +data class CopernicusRequest(val dataset: String, val inputs: Map) : CacheKey { + + /** + * @return a deterministic directory name: a human-readable sanitized prefix from [dataset], + * followed by a truncation of the SHA-256 hash of the canonical `(dataset, inputs)` pair. The + * prefix aids the human reading the cache directory; the hash provides collision resistance. + */ + override fun toFileName(): String { + val canonical = CanonicalJson.encode( + mapOf( + "dataset" to dataset, + "inputs" to inputs, + ), + ) + return "${dataset.toFileSystemSafe()}_${sha256Hex(canonical).take(HASH_PREFIX_LENGTH)}" + } + + private companion object { + + /** + * Number of leading hex characters of the SHA-256 digest kept in the folder name. + * Used to provide collision-safety for a personal cache while keeping the name short. + */ + private const val HASH_PREFIX_LENGTH = 16 + + /** + * Returns the SHA-256 digest of [str] (UTF-8) as a lowercase hex string. + * + * @param str the UTF-8 string to hash. + * @return the digest of SHA-256 over [str]. + */ + private fun sha256Hex(str: String): String = MessageDigest.getInstance("SHA-256") + .digest(str.toByteArray(Charsets.UTF_8)) + /* + * 0 = padding with zeros instead of spaces. + * ("05" and "5" are different values in ECMWF API) + * 2 = at least two digits. + * x = all hex are represented in lowercase. + */ + .joinToString("") { "%02x".format(it) } + } +} From cc830399cced7ca63d8435615d7308c0a2a5edd8 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 17:13:03 +0200 Subject: [PATCH 031/118] test(geospatial): add tests for CopernicusRequest --- .../acquisition/TestCopernicusRequest.kt | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusRequest.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusRequest.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusRequest.kt new file mode 100644 index 0000000000..dd1fce00a8 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusRequest.kt @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.string.shouldMatch +import io.kotest.matchers.string.shouldStartWith + +class TestCopernicusRequest : StringSpec({ + + // a realistic request to the EWDS data store + val glofas = CopernicusRequest( + dataset = "cems-glofas-historical", + inputs = mapOf( + "variable" to listOf("river_discharge_in_the_last_24_hours"), + "hyear" to listOf("2024"), + "hmonth" to listOf("06"), + "hday" to listOf("10"), + "data_format" to "netcdf", + ), + ) + + "toFileName is deterministic: the same request yields the same name" { + glofas.toFileName() shouldBe glofas.toFileName() + } + + "toFileName is stable under key reordering (the reason CanonicalJson exists)" { + // same logical request, just reordered + val reordered = CopernicusRequest( + dataset = "cems-glofas-historical", + inputs = mapOf( + "data_format" to "netcdf", + "hday" to listOf("10"), + "hmonth" to listOf("06"), + "hyear" to listOf("2024"), + "variable" to listOf("river_discharge_in_the_last_24_hours"), + ), + ) + reordered.toFileName() shouldBe glofas.toFileName() + } + + "a different dataset yields a different name" { + glofas.copy(dataset = "reanalysis-era5-single-levels").toFileName() shouldNotBe glofas.toFileName() + } + + "a different input value yields a different name (the bytes change)" { + val otherDay = glofas.copy(inputs = glofas.inputs + ("hday" to listOf("11"))) + otherDay.toFileName() shouldNotBe glofas.toFileName() + } + + "reordering a list inside inputs yields a different name (list order is semantic)" { + // area corners reordered = a different request, must not collide + val area = listOf(50.0, 5.0, 45.0, 10.0) + val nwse = glofas.copy(inputs = glofas.inputs + ("area" to area)) + val swapped = glofas.copy(inputs = glofas.inputs + ("area" to area.reversed())) + nwse.toFileName() shouldNotBe swapped.toFileName() + } + + "the readable prefix comes from the dataset, sanitized" { + glofas.toFileName() shouldStartWith "cems-glofas-historical_" + } + + "a dataset id with unsafe characters is sanitized in the prefix" { + val weird = glofas.copy(dataset = "weird/name with:chars") + weird.toFileName() shouldStartWith "weird_name_with_chars_" + } + + "the name complies with the CacheKey contract: a single file-system-safe segment" { + glofas.toFileName() shouldMatch Regex("^[A-Za-z0-9._-]+$") + } + + "the name is a readable prefix followed by a lowercase-hex hash suffix" { + glofas.toFileName() shouldMatch Regex("^cems-glofas-historical_[0-9a-f]+$") + } +}) From 51824267c6e52443aaebd3a1f11ff232c45b7987 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 21:25:38 +0200 Subject: [PATCH 032/118] feat(geospatial): add CdsApiRc to retrieve the user's token to use in the Copernicus requests --- .../model/geospatial/acquisition/CdsApiRc.kt | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt new file mode 100644 index 0000000000..ab7ede5da7 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Reads **only the token** from a `.cdsapirc` file (the format used by ECMWF's official client, + * with `url:` and `key:` lines) See [how-to-api](https://cds.climate.copernicus.eu/how-to-api). + * + * Only `key` is read, because the base URL is a layer parameter + * (`endpoint`): a single `.cdsapirc` has one `url` and cannot serve two data stores (CDS and EWDS), + * whereas the **same token** (unified ECMWF identity) is valid on both. + */ +internal object CdsApiRc { + + private const val KEY_FIELD = "key" + + /** + * Extracts the token from the `key:` line of the file. Each line is split on its **first** `:`, + * so any `:` inside the token itself is preserved. + * + * @param path path to a `.cdsapirc`-formatted file (already resolved). + * @return the token to send in the `PRIVATE-TOKEN` header. + * @throws IllegalStateException if the file has no `key:` line. + */ + fun readToken(path: Path): String = Files.readAllLines(path).firstNotNullOfOrNull { line -> + val separator = line.indexOf(':') + if (separator > 0 && line.take(separator).trim() == KEY_FIELD) { + line.substring(separator + 1).trim() + } else { + null + } + } ?: error("No 'key:' line in $path") +} From 9388b7e8560f660135aff03f3f944d1e78066b11 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 21:45:41 +0200 Subject: [PATCH 033/118] test(geospatial): add tests for CdsApiRc token retrival --- .../geospatial/acquisition/TestCdsApiRc.kt | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt new file mode 100644 index 0000000000..284562cadc --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import java.nio.file.Files +import java.nio.file.Path + +class TestCdsApiRc : StringSpec({ + + val tempDir: Path = Files.createTempDirectory("cdsapirc-test") + + // a realistic ECMWF unified access token + val realisticKey = "5b65k8c5-fr34-81dc-82b3-88e1hib45559" + + // deletes the test file after every unit test + afterSpec { + tempDir.toFile().deleteRecursively() + } + + /** + * Writes [content] to a fresh `.cdsapirc`-style file and returns its path. + */ + fun rcFile(content: String): Path = Files.createTempFile( + tempDir, + "rc", + null, + ).apply { + Files.writeString(this, content) + } + + "reads the token from a well-formed file" { + val path = rcFile( + """ + url: https://cds.climate.copernicus.eu/api + key: $realisticKey + """.trimIndent(), + ) + CdsApiRc.readToken(path) shouldBe realisticKey + } + + "splits on the FIRST colon, preserving colons inside the token" { + /* + * the token exact format may change in the future: + * non-UUID token with internal ':' must survive intact. + */ + val futureKey = "aaa:bbb:ccc" + val path = rcFile("key: $futureKey") + CdsApiRc.readToken(path) shouldBe futureKey + } + + "trims surrounding whitespace around the token value" { + // ECMWF writes a space after the colon in the user-guide + val path = rcFile("key: $realisticKey ") + CdsApiRc.readToken(path) shouldBe realisticKey + } + + "ignores the url line and any other keys" { + val path = rcFile( + """ + url: https://ewds.climate.copernicus.eu/api + verify: 0 + key: $realisticKey + other: another property + """.trimIndent(), + ) + CdsApiRc.readToken(path) shouldBe realisticKey + } + + "ignores comment lines" { + val path = rcFile( + """ + # personal access token below + key: $realisticKey + # personal access token above + """.trimIndent(), + ) + CdsApiRc.readToken(path) shouldBe realisticKey + } + + "throws IllegalStateException when no key line is present" { + val path = rcFile("url: https://cds.climate.copernicus.eu/api") + shouldThrow { CdsApiRc.readToken(path) } + } +}) From 6ecb1d4f005fde770bbbcec0213fc56b5fae29af Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Fri, 26 Jun 2026 21:48:55 +0200 Subject: [PATCH 034/118] docs(geospatial): clarified that the cdsapirc token format is not checked in CdsApiRc --- .../it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt index ab7ede5da7..575e58ea75 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CdsApiRc.kt @@ -19,6 +19,8 @@ import java.nio.file.Path * Only `key` is read, because the base URL is a layer parameter * (`endpoint`): a single `.cdsapirc` has one `url` and cannot serve two data stores (CDS and EWDS), * whereas the **same token** (unified ECMWF identity) is valid on both. + * + * **Note**: The key format is not checked, as there is no guarantee that it will not change in the future. */ internal object CdsApiRc { From b54efb36df0044aeb37bdc9aff005b4fdd1121d7 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 27 Jun 2026 15:29:24 +0200 Subject: [PATCH 035/118] feat(geospatial): add CacheManager to centralize cache entries creation --- .../geospatial/acquisition/CacheManager.kt | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheManager.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheManager.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheManager.kt new file mode 100644 index 0000000000..cd86ad8499 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CacheManager.kt @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import java.nio.file.FileSystemException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption + +/** + * Filesystem cache of the directories produced by data providers. This is the **single** place + * where cache atomicity lives, once for every provider. It knows nothing about HTTP or the + * individual APIs, depending only on [CacheKey]. + * + * This class considers **atomicity but non locking**: if two simultaneous cache-miss runs + * for the production of data; one wins the atomic rename, the other discards its own work. + * Wasteful in that rare case, but it never corrupts the cache. + * + * **Note!** The entries are **trust-based**: a present directory is assumed complete and valid: the content + * of a cache-hit is not re-verified. Manual alteration of a cache entry is out of contract. + * + * @param root cache root directory (e.g. `~/.alchemist/cache/geospatial`). Created on demand. + */ +class CacheManager(private val root: Path) { + + /** + * Temporary subdirectory, on the same filesystem as [root]. + * (required for `ATOMIC_MOVE`). + */ + private val tmpRoot: Path = root.resolve(TEMP_SUBDIR) + + /** + * Returns the directory associated with [key], producing it if absent. + * + * On **cache hit** the existing directory is returned immediately. + * On **cache miss**, [produce] runs into a temporary directory, its non-emptiness is + * validated, and it is then promoted to the final location with an atomic rename. + * If a concurrent process produced the same entry in the meantime, that one is used + * and the local work is discarded. If, for any reason, [produce] fails, the themporary + * directory is removed (no "poisoned" entry is left in cache). + * + * @param key request identity (to determine the directory name). + * @param produce action that fills the provided temporary directory with data. + * @return the [Path] of the final cache directory, filled with data from [produce]. + * + * @throws IllegalStateException if [produce] writes no file in the temporary directory. + */ + fun getOrProduce(key: CacheKey, produce: (Path) -> Unit): Path { + val finalDir = root.resolve(key.toFileName()) + // cache hit: the directory already exists + if (Files.isDirectory(finalDir)) return finalDir + + // cache miss (also creates root if it does not exist) + Files.createDirectories(tmpRoot) + val temp = Files.createTempDirectory(tmpRoot, key.toFileName()) + var moved = false // becomes true if the directory gets promoted + try { + // tries to fill the directory with data + produce(temp) + check(hasData(temp)) { "Provider produced no files for '${key.toFileName()}'" } + + moved = promote(temp, finalDir) + return finalDir + } finally { + // deletes the temp directory if any accident occurs + if (!moved) temp.toFile().deleteRecursively() + } + } + + /** + * Atomically promotes [temp] to [finalDir]. + * + * @param temp the path of the temporary directory. + * @param finalDir the path of the final directory after [temp] gets promoted. + * + * @return true if this call performed the move, false if a concurrent peer had already + * produced [finalDir]. + */ + private fun promote(temp: Path, finalDir: Path): Boolean = try { + Files.move(temp, finalDir, StandardCopyOption.ATOMIC_MOVE) + true + } catch (raceLost: FileSystemException) { + if (!Files.isDirectory(finalDir)) throw raceLost + // peer won; temp dir still exists + false + } + + /** + * Checks if [dir] directory holds at least one regular file. + * + * @return true if [dir] holds any regular file, false otherwise. + */ + private fun hasData(dir: Path): Boolean = + Files.list(dir).use { entries -> entries.anyMatch { Files.isRegularFile(it) } } + + private companion object { + private const val TEMP_SUBDIR = ".tmp" + } +} From 9a7cc80469d7acc6e862fb249357687580064b1f Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 27 Jun 2026 16:44:26 +0200 Subject: [PATCH 036/118] test(geospatial): add tests for CacheManager --- .../acquisition/TestCacheManager.kt | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt new file mode 100644 index 0000000000..0ffcad52f3 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.paths.shouldExist +import io.kotest.matchers.paths.shouldNotExist +import io.kotest.matchers.shouldBe +import java.nio.file.Files +import java.nio.file.Path + +class TestCacheManager : StringSpec({ + + val tempDir: Path = Files.createTempDirectory("cache-manager-test") + + // the name of the created data file + val dataFileName = "data.nc" + + // a lambda that writes a single file in the directory + val writeOneFile: (Path) -> Unit = { dir -> + Files.writeString(dir.resolve(dataFileName), "payload") + } + + // resets the temp dir after every unit test + afterSpec { + tempDir.toFile().deleteRecursively() + } + + /** + * Create a fresh new unique root. + */ + fun freshRoot(): Path = Files.createTempDirectory(tempDir, "root") + + /** + * A [CacheKey] that returns a fixed directory name. + */ + fun key(name: String): CacheKey = object : CacheKey { + override fun toFileName(): String = name + } + + "miss: produce runs exactly once and its files are promoted" { + val cache = CacheManager(freshRoot()) + var calls = 0 + val result = cache.getOrProduce(key("entry_a")) { dir -> + calls++ + writeOneFile(dir) + } + calls shouldBe 1 + result.shouldExist() + result.resolve(dataFileName).shouldExist() + } + + "hit: the second call reuses the first entry without re-producing" { + val cache = CacheManager(freshRoot()) + val first = cache.getOrProduce(key("entry_b")) { dir -> + Files.writeString(dir.resolve(dataFileName), "first") + } + // if the cache wrongly re-ran produce, the file would contain "SECOND" + val second = cache.getOrProduce(key("entry_b")) { dir -> + Files.writeString(dir.resolve(dataFileName), "SECOND") + } + second shouldBe first + Files.readString(second.resolve(dataFileName)) shouldBe "first" + } + + "the returned directory name is exactly key.toFileName() under root" { + val root = freshRoot() + val cache = CacheManager(root) + val dirName = "cems-glofas_abc123" + val result = cache.getOrProduce(key(dirName), writeOneFile) + result shouldBe root.resolve(dirName) + } + + "produce failure: the exception propagates and no entry is promoted" { + val root = freshRoot() + val cache = CacheManager(root) + shouldThrow { + cache.getOrProduce(key("entry_fail")) { error("download blew up") } + } + root.resolve("entry_fail").shouldNotExist() // no poisoned entry + } + + "produce failure: the temporary directory is cleaned up, leaving .tmp empty" { + val root = freshRoot() + val cache = CacheManager(root) + // ignores the exception + runCatching { + cache.getOrProduce(key("entry_fail2")) { dir -> + Files.writeString(dir.resolve("partial.nc"), "half") + // emulates something gone wrong on file writing + error("error after writing") + } + } + // .tmp must hold no leftover temp dirs + val tmpRoot = root.resolve(".tmp") + val leftovers = Files.list(tmpRoot).use { it.toList() } + leftovers.size shouldBe 0 + } + + "empty result: produce leaves no file and throws IllegalStateException, nothing promoted" { + val root = freshRoot() + val cache = CacheManager(root) + val entryName = "entry_empy" + shouldThrow { + cache.getOrProduce(key(entryName)) { /* writes nothing */ } + } + root.resolve(entryName).shouldNotExist() + } + + "validate before promoting: a dir with only subdirs (no regular file) is rejected" { + val root = freshRoot() + val cache = CacheManager(root) + val entryName = "entry_subdir" + shouldThrow { + cache.getOrProduce(key(entryName)) { dir -> + // a directory, but no regular file + Files.createDirectory(dir.resolve("nested")) + } + } + root.resolve(entryName).shouldNotExist() + } + + "race lost in promote: a peer wins after the fast-path check; peer reused, local discarded, temp cleaned" { + val root = freshRoot() + val cache = CacheManager(root) + val k = key("entry_race_real") + val finalDir = root.resolve(k.toFileName()) + + val result = cache.getOrProduce(k) { dir -> + Files.writeString(dir.resolve("local.nc"), "loser") + // a peer produces the same entry + Files.createDirectories(finalDir) + Files.writeString(finalDir.resolve("peer.nc"), "winner") + } + + result shouldBe finalDir + result.resolve("peer.nc").shouldExist() + result.resolve("local.nc").shouldNotExist() + // local temp should not be considered + Files.list(root.resolve(".tmp")).use { it.toList() } shouldBe emptyList() + } + + "hit is detected even across a fresh CacheManager over the same root" { + val root = freshRoot() + CacheManager(root).getOrProduce(key("entry_persist")) { dir -> + Files.writeString(dir.resolve(dataFileName), "first") + } + // a brand-new manager instance over the same root must reuse the existing entry + val result = CacheManager(root).getOrProduce(key("entry_persist")) { dir -> + Files.writeString(dir.resolve(dataFileName), "SECOND") + } + Files.readString(result.resolve(dataFileName)) shouldBe "first" + } +}) From 113f7dfa39215ead772b3f7bfea950ae2dd64e59 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 27 Jun 2026 17:53:02 +0200 Subject: [PATCH 037/118] feat(geospatial): add Archives utility to extract zip archives into a directory --- .../model/geospatial/acquisition/Archives.kt | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Archives.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Archives.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Archives.kt new file mode 100644 index 0000000000..7d9af6693d --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Archives.kt @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipException +import java.util.zip.ZipFile + +/** + * Extract all ZIP files inside of [dir] and deletes them. + * + * Each regular file in [dir] that can be opened as a valid ZIP archive is **extracted flat** (each + * entry by its basename) into [dir], and the archive itself is then deleted. Files that are not ZIP + * archives are left untouched. + * + * **Note:** it is the caller's responsibility to check, if necessary, whether [dir] is empty after this + * operation, as empty ZIPs are extracted too. + * + * @param dir the directory that contains the ZIP archives to extract. + * + * @throws IllegalStateException if two entries in an archive collapse to the same basename. + */ +internal fun flattenArchives(dir: Path) { + val files = Files.list(dir).use { + it.filter(Files::isRegularFile).toList() + } + for (file in files) { + // extract the ZIP file (if it is one) and deletes it. + if (extractArchive(file, dir)) { + Files.delete(file) + } + } +} + +/** + * Extracts [archive] flat into [dir] if it is a ZIP. + * + * @return `true` if [archive] was a valid ZIP (its entries were extracted), `false` if it is not a + * ZIP and was left untouched. + * @throws IllegalStateException on a basename collision between two entries. + */ +private fun extractArchive(archive: Path, dir: Path): Boolean = try { + ZipFile(archive.toFile()).use { zip -> + for (entry in zip.entries()) { + // nothing to do if the entry is directory. + if (entry.isDirectory) continue + + // it's a file. Resolves the new path. + val target = dir.resolve(Path.of(entry.name).fileName.toString()) + // a file with the exact same path already exists. + check(Files.notExists(target)) { + "Flatten collision on '${target.fileName}' from '${archive.fileName}'" + } + + // copies the content in the target path + zip.getInputStream(entry).use { source -> + Files.copy(source, target) + } + } + } + true +} catch (_: ZipException) { + // the file was not a zip archive. + false +} From 3689dce71c3d4cf4dcaa2fa2e351435f4e527c62 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 27 Jun 2026 18:14:48 +0200 Subject: [PATCH 038/118] test(geospatial): add tests for archives extraction utility --- .../geospatial/acquisition/TestArchives.kt | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestArchives.kt diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestArchives.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestArchives.kt new file mode 100644 index 0000000000..355d6f8806 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestArchives.kt @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.paths.shouldExist +import io.kotest.matchers.paths.shouldNotExist +import io.kotest.matchers.shouldBe +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class TestArchives : StringSpec({ + + val tempDir: Path = Files.createTempDirectory("archives-test") + + // deletes the directory and its files after the tests + afterSpec { + tempDir.toFile().deleteRecursively() + } + + // the subdir for each test + lateinit var dir: Path + beforeTest { + dir = Files.createTempDirectory(tempDir, "dir") + } + + /** + * Writes a real zip named [zipName] into [dir] with the given entry path -> content. + */ + fun writeZip(dir: Path, zipName: String, entries: Map): Path { + val zipPath = dir.resolve(zipName) + ZipOutputStream(Files.newOutputStream(zipPath)).use { zos -> + entries.forEach { (name, content) -> + zos.putNextEntry(ZipEntry(name)) + zos.write(content.toByteArray()) + zos.closeEntry() + } + } + return zipPath + } + + /** + * Returns the set of regular file basenames in [dir]. + */ + fun fileNames(dir: Path): Set = Files.list(dir).use { s -> + s.filter { Files.isRegularFile(it) } + .map { it.fileName.toString() }.toList() + }.toSet() + + "a non-zip file is left untouched" { + val data = dir.resolve("dis24.nc") + Files.writeString(data, "I'm not a ZIP!") + val before = Files.readString(data) + + flattenArchives(dir) + + data.shouldExist() + Files.readString(data) shouldBe before + fileNames(dir) shouldBe setOf("dis24.nc") + } + + "a single-entry zip is extracted flat and the archive is deleted" { + val zip = writeZip(dir, "result.zip", mapOf("dis24.nc" to "payload")) + + flattenArchives(dir) + + zip.shouldNotExist() + dir.resolve("dis24.nc").shouldExist() + Files.readString(dir.resolve("dis24.nc")) shouldBe "payload" + fileNames(dir) shouldBe setOf("dis24.nc") + } + + "multiple archives in the same dir are all extracted and deleted" { + writeZip(dir, "part1.zip", mapOf("a.nc" to "A")) + writeZip(dir, "part2.zip", mapOf("b.nc" to "B")) + + flattenArchives(dir) + + fileNames(dir) shouldBe setOf("a.nc", "b.nc") + } + + "a data file alongside an archive: the file stays, the archive is flattened" { + Files.writeString(dir.resolve("already.nc"), "plain") + writeZip(dir, "result.zip", mapOf("fromzip.nc" to "Z")) + + flattenArchives(dir) + + fileNames(dir) shouldBe setOf("already.nc", "fromzip.nc") + } + + "a multi-entry zip extracts every entry" { + writeZip(dir, "result.zip", mapOf("a.nc" to "A", "b.nc" to "B")) + + flattenArchives(dir) + + fileNames(dir) shouldBe setOf("a.nc", "b.nc") + Files.readString(dir.resolve("a.nc")) shouldBe "A" + Files.readString(dir.resolve("b.nc")) shouldBe "B" + } + + "nested entry paths are flattened to their basename" { + writeZip(dir, "result.zip", mapOf("data/2024/06/dis24.nc" to "deep")) + + flattenArchives(dir) + + dir.resolve("dis24.nc").shouldExist() + Files.readString(dir.resolve("dis24.nc")) shouldBe "deep" + fileNames(dir) shouldBe setOf("dis24.nc") + } + + "detection is by content, not extension: a zip named '.nc' is still extracted" { + // the archive itself is named like a data file + val zip = writeZip(dir, "payload.nc", mapOf("dis24.nc" to "inner")) + + flattenArchives(dir) + + zip.shouldNotExist() + dir.resolve("dis24.nc").shouldExist() + Files.readString(dir.resolve("dis24.nc")) shouldBe "inner" + } + + "detection is by content, not extension: a non-zip named '.zip' is left untouched" { + val fake = dir.resolve("archive.zip") + Files.writeString(fake, "this is plain text, not a zip") + + flattenArchives(dir) + + fake.shouldExist() + Files.readString(fake) shouldBe "this is plain text, not a zip" + } + + "a flatten collision (two entries, same basename) throws IllegalStateException" { + writeZip(dir, "result.zip", mapOf("regionA/dis24.nc" to "A", "regionB/dis24.nc" to "B")) + + shouldThrow { flattenArchives(dir) } + } + + "an empty zip (with no entries) extracts nothing, is deleted, and leaves the dir empty" { + val zip = writeZip(dir, "result.zip", emptyMap()) + + flattenArchives(dir) + + zip.shouldNotExist() + fileNames(dir) shouldBe emptySet() + } +}) From 326f3c5b17674827a34ffa7362ca0ea4839a16af Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sat, 27 Jun 2026 18:19:09 +0200 Subject: [PATCH 039/118] docs(geospatial): fixed clarification on AfterSpec callback in test classes --- .../it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt | 2 +- .../alchemist/model/geospatial/acquisition/TestCacheManager.kt | 2 +- .../alchemist/model/geospatial/acquisition/TestCdsApiRc.kt | 2 +- .../alchemist/model/geospatial/reading/TestCdmTimedGrid.kt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt index 6f1f50d039..1f5aedb729 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/TestGeoRasterLayer.kt @@ -101,7 +101,7 @@ class TestGeoRasterLayer : StringSpec({ val tempDir: Path = createTempDirectory("geo-raster-layer-test") - // cleans the temp directory after each test + // deletes the directory and its files after the tests afterSpec { tempDir.toFile().deleteRecursively() } diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt index 0ffcad52f3..20e9372b82 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCacheManager.kt @@ -29,7 +29,7 @@ class TestCacheManager : StringSpec({ Files.writeString(dir.resolve(dataFileName), "payload") } - // resets the temp dir after every unit test + // deletes the directory and its files after the tests afterSpec { tempDir.toFile().deleteRecursively() } diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt index 284562cadc..169d888b8c 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCdsApiRc.kt @@ -22,7 +22,7 @@ class TestCdsApiRc : StringSpec({ // a realistic ECMWF unified access token val realisticKey = "5b65k8c5-fr34-81dc-82b3-88e1hib45559" - // deletes the test file after every unit test + // deletes the directory and its files after the tests afterSpec { tempDir.toFile().deleteRecursively() } diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt index 83362abaf9..efc4dfc9c5 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/reading/TestCdmTimedGrid.kt @@ -25,8 +25,8 @@ class TestCdmTimedGrid : StringSpec({ */ val tempDir: Path = Files.createTempDirectory("cdm-timed-grid-test") + // deletes the directory and its files after the tests afterSpec { - // empties the tmp directory after each test tempDir.toFile().deleteRecursively() } From 7ddb8bbb00fe7b4ad40978497789bb57886ea492 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 28 Jun 2026 17:18:02 +0200 Subject: [PATCH 040/118] feat(geospatial): add ExternalDataProvider interface to abstract the data generation method --- .../acquisition/ExternalDataProvider.kt | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/ExternalDataProvider.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/ExternalDataProvider.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/ExternalDataProvider.kt new file mode 100644 index 0000000000..0d4a18afa1 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/ExternalDataProvider.kt @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import java.nio.file.Path + +/** + * Common contract for an external data source: given a typed request, it generates a + * local directory of files that are ready to be opened. Ensures that the rest of the system + * is unaware of the specifics of each individual provider API. + * + * [R] is associated with [CacheKey], so caching is part of the contract; it is not + * an implementation detail: every request must be cacheable. + * + * The type parameter [R] binds each provider to its own family of requests. For + * example, an `ExternalDataProvider` rejects a `BBBikeRequest` at + * compile time. + * + * @param R the consumed request type, bound to [CacheKey] so that its result + * is always cacheable. + */ +interface ExternalDataProvider { + + /** + * Obtains the local directory (cache entry) corresponding to [request], + * generating it if it does not exist (e.g., by downloading resources). + * + * @param request the request identifying the data to obtain. + * @return the [Path] of a **directory** containing the ready-to-open files. + */ + fun fetch(request: R): Path +} From 5211f1703aec8a2788c0ef12c9b8efc7d16bb1ec Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 28 Jun 2026 18:07:42 +0200 Subject: [PATCH 041/118] feat(geospatial): add Integrity functions to check download resources integrity --- .../model/geospatial/acquisition/Integrity.kt | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt new file mode 100644 index 0000000000..0ebe68ae11 --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import java.nio.file.Files +import java.nio.file.Path +import java.security.DigestInputStream +import java.security.MessageDigest + +/** + * Size of the buffer used to stream a file through the MD5 digest, in bytes. + */ +private const val DIGEST_BUFFER_BYTES = 8 * 1024 // 8 KB + +/** + * Verifies the integrity of a downloaded [file] against the metadata the data store + * advertised for it: the byte count must equal [expectedSizeBytes] and the MD5 digest must equal + * [expectedMd5] (compared case-insensitively). + * + * **Note for archives**: if the downloaded file is an archive (e.g., ZIP), the byte count + * and MD5 hash typically refer to the archive itself and not to its extracted files. + * + * @param file the downloaded file to check. + * @param expectedSizeBytes the expected size in bytes. + * @param expectedMd5 the expected MD5 digest as a hex string. + * + * @throws IllegalStateException if the actual size or MD5 does not match the expected value. + */ +internal fun verify(file: Path, expectedSizeBytes: Long, expectedMd5: String) { + val actualSize = Files.size(file) + check(actualSize == expectedSizeBytes) { + "Size mismatch for '${file.fileName}': expected $expectedSizeBytes bytes, got $actualSize" + } + val actualMd5 = md5Hex(file) + check(actualMd5.equals(expectedMd5, ignoreCase = true)) { + "MD5 mismatch for '${file.fileName}': expected $expectedMd5, got $actualMd5" + } +} + +/** + * Computes the MD5 digest of [file] as a lowercase hex string, + * streaming the file through the digest so arbitrarily large files + * are never fully held in memory. + * + * @param file the file to digest. + * @return the MD5 digest, as a lowercase hex string. + */ +internal fun md5Hex(file: Path): String { + val digest = MessageDigest.getInstance("MD5") + DigestInputStream(Files.newInputStream(file), digest).use { stream -> + // at any given time, a maximum of DIGEST_BUFFER_BYTES bytes are allocated in memory. + val buffer = ByteArray(DIGEST_BUFFER_BYTES) + // fills the buffer on every call until there are no more bytes available. + while (stream.read(buffer) != -1) { + // reading feeds the digest. The bytes themselves are discarded. + } + } + return digest.digest().joinToString("") { + /* + * 0 = padding with zeros instead of spaces. + * 2 = at least two digits. + * x = all hex are represented in lowercase. + */ + "%02x".format(it) + } +} From 028048d408cb09330617814adbc5305640eaf9d0 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 28 Jun 2026 23:25:49 +0200 Subject: [PATCH 042/118] test(geospatial): add file md5 check utility function --- .../model/geospatial/acquisition/Integrity.kt | 16 ++-- .../geospatial/acquisition/TestIntegrity.kt | 78 +++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestIntegrity.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt index 0ebe68ae11..22a7003d7c 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/Integrity.kt @@ -22,25 +22,29 @@ private const val DIGEST_BUFFER_BYTES = 8 * 1024 // 8 KB /** * Verifies the integrity of a downloaded [file] against the metadata the data store * advertised for it: the byte count must equal [expectedSizeBytes] and the MD5 digest must equal - * [expectedMd5] (compared case-insensitively). + * [expectedMd5], if provided. (compared case-insensitively). * * **Note for archives**: if the downloaded file is an archive (e.g., ZIP), the byte count * and MD5 hash typically refer to the archive itself and not to its extracted files. * * @param file the downloaded file to check. * @param expectedSizeBytes the expected size in bytes. - * @param expectedMd5 the expected MD5 digest as a hex string. + * @param expectedMd5 the expected MD5 digest as a hex string, if provided. * * @throws IllegalStateException if the actual size or MD5 does not match the expected value. */ -internal fun verify(file: Path, expectedSizeBytes: Long, expectedMd5: String) { +internal fun verify(file: Path, expectedSizeBytes: Long, expectedMd5: String? = null) { val actualSize = Files.size(file) check(actualSize == expectedSizeBytes) { "Size mismatch for '${file.fileName}': expected $expectedSizeBytes bytes, got $actualSize" } - val actualMd5 = md5Hex(file) - check(actualMd5.equals(expectedMd5, ignoreCase = true)) { - "MD5 mismatch for '${file.fileName}': expected $expectedMd5, got $actualMd5" + + // only if md5 is specified + expectedMd5?.let { + val actualMd5 = md5Hex(file) + check(actualMd5.equals(expectedMd5, ignoreCase = true)) { + "MD5 mismatch for '${file.fileName}': expected $expectedMd5, got $actualMd5" + } } } diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestIntegrity.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestIntegrity.kt new file mode 100644 index 0000000000..8e9eaaa764 --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestIntegrity.kt @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import java.nio.file.Files +import java.nio.file.Path + +class TestIntegrity : StringSpec({ + + val tempDir: Path = Files.createTempDirectory("integrity-test") + afterSpec { + tempDir.toFile().deleteRecursively() + } + + lateinit var file: Path + beforeTest { + file = Files.createTempFile(tempDir, "asset", ".bin") + } + + // some exact MD5 digests + val knownMD5 = mapOf( + "" to "d41d8cd98f00b204e9800998ecf8427e", + "abc" to "900150983cd24fb0d6963f7d28e17f72", + ) + + "md5Hex of empty file matches the known digest" { + // file is created empty by beforeTest + md5Hex(file) shouldBe knownMD5[""] + } + + "md5Hex of 'abc' matches the known digest" { + Files.writeString(file, "abc") + md5Hex(file) shouldBe knownMD5["abc"] + } + + "md5Hex is always 32 lowercase hex chars" { + Files.writeString(file, "whatever content") + val hex = md5Hex(file) + hex.length shouldBe 32 + hex shouldBe hex.lowercase() + } + + "verify passes when size and MD5 both match" { + Files.writeString(file, "abc") + // 3 bytes, known MD5 + verify(file, 3, knownMD5["abc"]!!) + } + + "verify accepts uppercase expected MD5 (case-insensitive)" { + Files.writeString(file, "abc") + verify(file, 3, knownMD5["abc"]!!.uppercase()) + } + + "verify throws on size mismatch" { + Files.writeString(file, "abc") + shouldThrow { + verify(file, 999) + } + } + + "verify throws on MD5 mismatch even when size is right" { + Files.writeString(file, "abc") + shouldThrow { + // correct size (3) but wrong checksum + verify(file, 3, "00000000000000000000000000000000") + } + } +}) From c4c9064968c8deedfff30f2f303f2132891c3127 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Sun, 28 Jun 2026 23:37:13 +0200 Subject: [PATCH 043/118] docs(geospatial): updated hex doc in CopernicusRequest --- .../alchemist/model/geospatial/acquisition/CopernicusRequest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt index 745cadcd37..17673276b4 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusRequest.kt @@ -56,7 +56,6 @@ data class CopernicusRequest(val dataset: String, val inputs: Map) .digest(str.toByteArray(Charsets.UTF_8)) /* * 0 = padding with zeros instead of spaces. - * ("05" and "5" are different values in ECMWF API) * 2 = at least two digits. * x = all hex are represented in lowercase. */ From e9e29021e9dad977fcc24055efaa96b7f4cb9828 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 29 Jun 2026 02:52:52 +0200 Subject: [PATCH 044/118] feat(geospatial): add parsers for Copernicus json responses in CopernicusResponses --- .../acquisition/CopernicusResponses.kt | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt new file mode 100644 index 0000000000..0f34ceb50f --- /dev/null +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import com.google.gson.JsonParser + +/* + * Pure parsers for the JSON bodies of the ECMWF data store REST API. + * + * The job envelope (submit/status/results and its `links`) is an instance of "OGC API - Processes, + * Part 1: Core (OGC 18-062r2)"; the asset metadata field names (`file:size`, `file:checksum`) are + * from the STAC File Info Extension. ECMWF conforms to neither fully: it deviates locally (e.g. an + * `asset.value` envelope that is not a STAC document, a bare MD5 instead of a multihash), so these + * parsers extract only the few fields needed and tolerate the rest. + * + * Each function takes a raw response body string and is fully testable offline against captured + * real responses. + * + * For refence: + * Open Geospatial Consortium API: https://docs.ogc.org/is/18-062r2/18-062r2.html#toc0 + * Spatio Temporal Asset Catalogs: https://github.com/stac-extensions/file + */ + +/** + * Search for and returns the `href` property of the first link in the `links` + * array within the [json] whose `rel` property is equal to [rel]. + * + * ```json + * An example of JSON body is: + * { "links": [ { "rel": "self", "href": "..." }, + * { "rel": "monitor", "href": "URL_TO_RETURN" } ] } + * ``` + * In this example, with `rel = "monitor"` the return value would be `URL_TO_RETURN` + * + * @param json the JSON body as a string. + * @param rel the `rel` property associated with the desired `href`. + * @return the `href` associated with [rel] in the links array of [json], or `null` if not present. + */ +private fun linkHref(json: String, rel: String): String? = JsonParser.parseString(json) + .asJsonObject.getAsJsonArray("links") + ?.map { it.asJsonObject } + ?.firstOrNull { it.get("rel")?.asString == rel } + ?.get("href")?.asString + +/** + * Extracts the job-monitoring URL from a submit response [json] (`POST .../execute`). + * + * Returns the absolute `href` of the link whose `rel` is `"monitor"`. + * The same job is also identifiable via the top-level `jobID` field and the `Location` header. + * The `monitor` link is preferred because it keeps polling decoupled from + * the URL path layout while remaining a pure JSON parser. + * + * @param json the JSON string to parse. + * @return the `href` associated with `rel="monitor"` in the links array of [json]. + * + * @throws IllegalStateException if no `rel="monitor"` link is present in [json]. + */ +internal fun parseMonitorUrl(json: String): String = + linkHref(json, "monitor") ?: error("No link with rel='monitor' in the submit response") + +/** + * Extracts the job status from a status (`GET .../jobs/{id}`) response [json]. + * + * Known values include `accepted`, `running`, `successful`, `failed`, `dismissed`. Callers should + * treat any value other than `successful`/`failed` as "keep polling", rather than enumerating the + * intermediate states, so an unforeseen status does not break the wait loop. + * + * @param json the JSON string to parse. + * @return the request's processing status. + * + * @throws IllegalStateException if the `status` field is absent. + */ +internal fun parseStatus(json: String): String = JsonParser.parseString(json) + .asJsonObject.get("status") + ?.asString + ?: error("No 'status' field in the status response") + +/** + * Extracts the results URL from a status (`GET .../jobs/{id}`) response [json], or `null` if the + * job exposes no results link yet. + * + * The `rel="results"` link appears only once the status is + * `successful`; an `accepted`/`running` job exposes only `rel="self"`. When this returns `null` + * on an already-successful job, the caller may fall back to `"{statusUrl}/results"`. + * + * @param json the JSON string to parse. + * @return the results URL or `null` if no result link is present. + */ +internal fun parseResultsUrl(json: String): String? = linkHref(json, "results") + +/** + * Extracts the downloadable asset metadata from a results (`GET .../jobs/{id}/results`) [json]. + * + * Reads `asset.value.href` (the download URL, served by the object store on a different host and + * without authentication; absolute as served by ECMWF), `file:size`, and the optional + * `file:checksum`. + * + * **On the shape.** The field names `file:size`/`file:checksum` are STAC File Info Extension + * naming, but the `asset.value` envelope is an ECMWF convention, NOT STAC structure: a canonical + * STAC asset lives in an `assets` map with these fields directly on it (no `value` wrapper), and + * this body is not a STAC document. Hence, the two nested lookups (`asset` then `value`). + * + * **On the checksum.** ECMWF emits `file:checksum` as a bare lowercase MD5 hex string, despite the + * STAC file extension nominally prescribing a self-identifying multihash, so it is captured + * verbatim. It is nullable because some datasets/stores may omit it. + * + * @param json the JSON string to parse. + * @return asset metadata as a [RemoteAsset]. + * + * @throws IllegalStateException if `asset.value`, its `href`, or its `file:size` is absent. + */ +internal fun parseAsset(json: String): RemoteAsset { + val value = JsonParser.parseString(json).asJsonObject + .getAsJsonObject("asset") + ?.getAsJsonObject("value") + ?: error("No 'asset.value' object in the results response") + return RemoteAsset( + href = value.get("href")?.asString ?: error("No 'href' in asset.value"), + sizeBytes = value.get("file:size")?.asLong ?: error("No 'file:size' in asset.value"), + md5 = value.get("file:checksum")?.asString, + ) +} + +/** + * Metadata of a result file ready for download. + * + * @property href absolute, unauthenticated download URL. + * @property sizeBytes expected size in bytes; must always be verified after download. + * @property md5 expected MD5, if advertised, else `null`. + */ +internal data class RemoteAsset(val href: String, val sizeBytes: Long, val md5: String?) From ab4121c1dd0860f46855d48adc8c71671e0632e4 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 29 Jun 2026 16:01:17 +0200 Subject: [PATCH 045/118] test(geospatial): add tests for CopernicusResponses parsers along with real json response bodies --- .../acquisition/TestCopernicusResponses.kt | 106 ++++++++++++++++++ .../copernicus-responses/accepted-status.json | 21 ++++ .../copernicus-responses/ads-submit.json | 32 ++++++ .../copernicus-responses/cds-submit.json | 20 ++++ .../copernicus-responses/ewds-submit.json | 32 ++++++ .../copernicus-responses/results.json | 11 ++ .../successful-status.json | 27 +++++ 7 files changed, 249 insertions(+) create mode 100644 alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/accepted-status.json create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/ads-submit.json create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/cds-submit.json create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/ewds-submit.json create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/results.json create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/successful-status.json diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt new file mode 100644 index 0000000000..75c65011aa --- /dev/null +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2010-2026, Danilo Pianini and contributors + * listed, for each module, in the respective subproject's build.gradle.kts file. + * + * This file is part of Alchemist, and is distributed under the terms of the + * GNU General Public License, with a linking exception, + * as described in the file LICENSE in the Alchemist distribution's top directory. + */ + +package it.unibo.alchemist.model.geospatial.acquisition + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe + +/** + * The test data, the ACTUAL response bodies, were captured on 2026-06-28 (yyyy-mm-dd) + * from several ECMWF stores (CDS / EWDS / ADS). The EWDS and ADS submit bodies are verbatim; + * the CDS submit body is truncated. The expected values are read manually from each body. + */ +class TestCopernicusResponses : StringSpec({ + + // a captured submit response paired with the monitor URL expected from it. + data class SubmitCase(val store: String, val body: String, val expectedMonitorUrl: String) + + // some ACTUAL responses received after submit requests + val submitCases = listOf( + SubmitCase( + store = "CDS", + body = loadBody("cds-submit.json"), + expectedMonitorUrl = "https://cds.climate.copernicus.eu/api/retrieve" + + "/v1/jobs/98644c83-07f4-44ff-bc6b-2969c0342a32", + ), + SubmitCase( + store = "EWDS", + body = loadBody("ewds-submit.json"), + expectedMonitorUrl = "https://ewds.climate.copernicus.eu/api/retrieve" + + "/v1/jobs/ca0c3ecc-9c02-48ad-b781-73ee0510e653", + ), + SubmitCase( + store = "ADS", + body = loadBody("ads-submit.json"), + expectedMonitorUrl = "https://ads.atmosphere.copernicus.eu/api/retrieve" + + "/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68", + ), + ) + + // status bodies. One successful (adds a results link), one accepted (only a self link) + val successfulStatus = loadBody("successful-status.json") + val acceptedStatus = loadBody("accepted-status.json") + + // results body (from a GET .../jobs/{id}/results request) + val resultsBody = loadBody("results.json") + + // link extraction + "parseMonitorUrl extracts the monitor link from a submit response" { + submitCases.forEach { case -> + withClue("store: ${case.store}") { + parseMonitorUrl(case.body) shouldBe case.expectedMonitorUrl + } + } + } + + // status extraction + "parseStatus reads 'successful' from a finished job" { + parseStatus(successfulStatus) shouldBe "successful" + } + + "parseStatus reads 'accepted' from a pending job" { + parseStatus(acceptedStatus) shouldBe "accepted" + } + + // download link extraction + "parseResultsUrl returns the results link once the job is successful" { + parseResultsUrl(successfulStatus) shouldBe + "https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/98644c83-07f4-44ff-bc6b-2969c0342a32/results" + } + + "parseResultsUrl returns null while the job is not yet ready" { + parseResultsUrl(acceptedStatus) shouldBe null + } + + // asset metadata extraction + "parseAsset extracts href, size and checksum from a results response" { + parseAsset(resultsBody) shouldBe RemoteAsset( + href = "https://object-store.os-api.cci2.ecmwf.int:443/" + + "cci2-prod-cache-1/2026-06-28/55f861b61cf925b229030a1faf838e93.nc", + sizeBytes = 2_331_970L, + md5 = "b7b990dc67d490e0360c41b47fc616a6", + ) + } +}) + +/** + * Loads the JSON bodies from the .../resources/copernicus-responses/ directory. + * + * @param fileName the name of the JSON file to load. + * @return the JSON body of [fileName] as a string. + * + * @throws IllegalStateException if no file with name [fileName] is found. + */ +private fun loadBody(fileName: String): String = checkNotNull( + TestCopernicusResponses::class.java.getResourceAsStream("/copernicus-responses/$fileName"), +) { + "Missing test fixture: $fileName" +}.bufferedReader().use { it.readText() } diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/accepted-status.json b/alchemist-geospatial/src/test/resources/copernicus-responses/accepted-status.json new file mode 100644 index 0000000000..bce926a6ae --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/accepted-status.json @@ -0,0 +1,21 @@ +{ + "processID": "cams-global-greenhouse-gas-forecasts", + "type": "process", + "jobID": "61ebb7be-650e-4aa5-9039-6030eb01bb68", + "status": "accepted", + "created": "2026-06-28T17:29:06.745241", + "updated": "2026-06-28T17:29:06.745241", + "links": [ + { + "href": "https://ads.atmosphere.copernicus.eu/api/retrieve/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68", + "rel": "self", + "type": "application/json" + } + ], + "metadata": { + "datasetMetadata": { + "catalogue": "cams" + }, + "origin": "api" + } +} \ No newline at end of file diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/ads-submit.json b/alchemist-geospatial/src/test/resources/copernicus-responses/ads-submit.json new file mode 100644 index 0000000000..fc10b7a40c --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/ads-submit.json @@ -0,0 +1,32 @@ +{ + "processID": "cams-global-greenhouse-gas-forecasts", + "type": "process", + "jobID": "61ebb7be-650e-4aa5-9039-6030eb01bb68", + "status": "accepted", + "created": "2026-06-28T17:29:06.745241", + "updated": "2026-06-28T17:29:06.745241", + "links": [ + { + "href": "https://ads.atmosphere.copernicus.eu/api/retrieve/v1/processes/cams-global-greenhouse-gas-forecasts/execute", + "rel": "self" + }, + { + "href": "https://ads.atmosphere.copernicus.eu/api/retrieve/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68", + "rel": "monitor", + "type": "application/json", + "title": "job status info" + } + ], + "metadata": { + "datasetMetadata": { + "messages": [ + { + "date": "2026-06-28T17:29:06.758695", + "severity": "WARNING", + "content": "You are using a deprecated API endpoint. If you are using cdsapi, please upgrade to the latest version." + } + ] + }, + "origin": "api" + } +} \ No newline at end of file diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/cds-submit.json b/alchemist-geospatial/src/test/resources/copernicus-responses/cds-submit.json new file mode 100644 index 0000000000..da9943be1a --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/cds-submit.json @@ -0,0 +1,20 @@ +{ + "processID": "derived-era5-pressure-levels-daily-statistics", + "type": "process", + "jobID": "98644c83-07f4-44ff-bc6b-2969c0342a32", + "status": "accepted", + "created": "2026-06-28T17:19:54.843588", + "updated": "2026-06-28T17:19:54.843588", + "links": [ + { + "href": "https://cds.climate.copernicus.eu/api/retrieve/v1/processes/derived-era5-pressure-levels-daily-statistics/execute", + "rel": "self" + }, + { + "href": "https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/98644c83-07f4-44ff-bc6b-2969c0342a32", + "rel": "monitor", + "type": "application/json", + "title": "job status info" + } + ] +} \ No newline at end of file diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/ewds-submit.json b/alchemist-geospatial/src/test/resources/copernicus-responses/ewds-submit.json new file mode 100644 index 0000000000..1c8a1bcc85 --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/ewds-submit.json @@ -0,0 +1,32 @@ +{ + "processID": "cems-fire-historical-v1", + "type": "process", + "jobID": "ca0c3ecc-9c02-48ad-b781-73ee0510e653", + "status": "accepted", + "created": "2026-06-28T17:24:02.116929", + "updated": "2026-06-28T17:24:02.116929", + "links": [ + { + "href": "https://ewds.climate.copernicus.eu/api/retrieve/v1/processes/cems-fire-historical-v1/execute", + "rel": "self" + }, + { + "href": "https://ewds.climate.copernicus.eu/api/retrieve/v1/jobs/ca0c3ecc-9c02-48ad-b781-73ee0510e653", + "rel": "monitor", + "type": "application/json", + "title": "job status info" + } + ], + "metadata": { + "datasetMetadata": { + "messages": [ + { + "date": "2026-06-28T17:24:02.135635", + "severity": "WARNING", + "content": "You are using a deprecated API endpoint. If you are using cdsapi, please upgrade to the latest version." + } + ] + }, + "origin": "api" + } +} \ No newline at end of file diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/results.json b/alchemist-geospatial/src/test/resources/copernicus-responses/results.json new file mode 100644 index 0000000000..953175033b --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/results.json @@ -0,0 +1,11 @@ +{ + "asset": { + "value": { + "type": "application/netcdf", + "href": "https://object-store.os-api.cci2.ecmwf.int:443/cci2-prod-cache-1/2026-06-28/55f861b61cf925b229030a1faf838e93.nc", + "file:checksum": "b7b990dc67d490e0360c41b47fc616a6", + "file:size": 2331970, + "file:local_path": "s3://cci2-prod-cache-1/2026-06-28/55f861b61cf925b229030a1faf838e93.nc" + } + } +} \ No newline at end of file diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/successful-status.json b/alchemist-geospatial/src/test/resources/copernicus-responses/successful-status.json new file mode 100644 index 0000000000..5a59190e1f --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/successful-status.json @@ -0,0 +1,27 @@ +{ + "processID": "derived-era5-pressure-levels-daily-statistics", + "type": "process", + "jobID": "98644c83-07f4-44ff-bc6b-2969c0342a32", + "status": "successful", + "created": "2026-06-28T17:19:54.843588", + "started": "2026-06-28T17:20:07.894336", + "finished": "2026-06-28T17:20:17.935423", + "updated": "2026-06-28T17:20:17.935423", + "links": [ + { + "href": "https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/98644c83-07f4-44ff-bc6b-2969c0342a32", + "rel": "self", + "type": "application/json" + }, + { + "href": "https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/98644c83-07f4-44ff-bc6b-2969c0342a32/results", + "rel": "results" + } + ], + "metadata": { + "datasetMetadata": { + "catalogue": "c3s" + }, + "origin": "api" + } +} \ No newline at end of file From 2882c41536f886ab65cca01d3412dee4dc23298d Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 29 Jun 2026 17:34:45 +0200 Subject: [PATCH 046/118] feat(geospatial): add a response error parser in CopernicusResponses --- .../acquisition/CopernicusResponses.kt | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt index 0f34ceb50f..64cd0c318f 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt @@ -28,6 +28,34 @@ import com.google.gson.JsonParser * Spatio Temporal Asset Catalogs: https://github.com/stac-extensions/file */ +/** + * Metadata of a result file ready for download. + * + * @property href absolute, unauthenticated download URL. + * @property sizeBytes expected size in bytes; must always be verified after download. + * @property md5 expected MD5, if advertised, else `null`. + */ +internal data class RemoteAsset(val href: String, val sizeBytes: Long, val md5: String?) + +/** + * An RFC 7807 "Problem Details" error report from the data store. + * + * @property type a URI (or, for ECMWF, sometimes an opaque label) identifying the problem type. + * @property title a short, human-readable summary of the problem type. + * @property status the HTTP status code echoed in the body, if present. + * @property detail a human-readable explanation specific to this occurrence, if present. + * @property instance a URI identifying the specific occurrence of the problem, if present. + * @property traceId the data store's trace identifier (`trace_id`), useful when reporting an issue. + */ +internal data class ProblemDetail( + val type: String, + val title: String?, + val status: Int?, + val detail: String?, + val instance: String?, + val traceId: String?, +) + /** * Search for and returns the `href` property of the first link in the `links` * array within the [json] whose `rel` property is equal to [rel]. @@ -129,10 +157,30 @@ internal fun parseAsset(json: String): RemoteAsset { } /** - * Metadata of a result file ready for download. + * Parses an RFC 7807 "Problem Details" error body [json], as returned by the data store on a + * failed request (e.g. a `404` result-not-ready, a `401` authentication required, a `403` + * dataset-license not accepted, a `400` invalid request). * - * @property href absolute, unauthenticated download URL. - * @property sizeBytes expected size in bytes; must always be verified after download. - * @property md5 expected MD5, if advertised, else `null`. + * All fields are optional per RFC 7807 except `type` (which defaults to `"about:blank"`), so every + * field but [ProblemDetail.type] is nullable. Note that ECMWF does not always honor the spec's + * recommendation that `type` be a URI (it sometimes repeats the human-readable title, e.g. + * `"permission denied"`), so `type` is treated as an opaque string, never parsed as a URI. + * + * @param json the JSON error body as a string. + * @return the extracted [ProblemDetail]. */ -internal data class RemoteAsset(val href: String, val sizeBytes: Long, val md5: String?) +internal fun parseProblemDetail(json: String): ProblemDetail { + val obj = JsonParser.parseString(json).asJsonObject + + // a field extractor by name + fun field(name: String): String? = obj.get(name)?.takeUnless { it.isJsonNull }?.asString + + return ProblemDetail( + type = field("type") ?: "about:blank", + title = field("title"), + status = obj.get("status")?.takeUnless { it.isJsonNull }?.asInt, + detail = field("detail"), + instance = field("instance"), + traceId = field("trace_id"), + ) +} From 6984fec24ad225ce6ef9b40af3339b7536f5416a Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Mon, 29 Jun 2026 17:47:26 +0200 Subject: [PATCH 047/118] test(geospatial): add tests for Copernicus error parser --- .../acquisition/TestCopernicusResponses.kt | 36 +++++++++++++++++++ .../error-401-permission-denied.json | 8 +++++ .../error-404-result-not-ready.json | 8 +++++ 3 files changed, 52 insertions(+) create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/error-401-permission-denied.json create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/error-404-result-not-ready.json diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt index 75c65011aa..bc0034e8b8 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt @@ -89,6 +89,42 @@ class TestCopernicusResponses : StringSpec({ md5 = "b7b990dc67d490e0360c41b47fc616a6", ) } + + // errors extraction + "parseProblemDetail extracts all fields from a 404 result-not-ready body" { + parseProblemDetail(loadBody("error-404-result-not-ready.json")) shouldBe ProblemDetail( + type = "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/result-not-ready", + title = "job results not ready", + status = 404, + detail = "status of 61ebb7be-650e-4aa5-9039-6030eb01bb68 is 'accepted'", + instance = "https://ads.atmosphere.copernicus.eu/api/retrieve" + + "/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68/results", + traceId = "e7ba3606-9816-43cc-ab6a-4f0642388701", + ) + } + + "parseProblemDetail handles a 401 whose type is a free-string label, not a URI" { + parseProblemDetail(loadBody("error-401-permission-denied.json")) shouldBe ProblemDetail( + type = "permission denied", + title = "permission denied", + status = 401, + detail = "authentication required", + instance = "https://ads.atmosphere.copernicus.eu/api/retrieve" + + "/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68", + traceId = "b63a2882-2510-4ced-935a-b2faec13eead", + ) + } + + "parseProblemDetail defaults type to 'about:blank' and nulls absent fields" { + parseProblemDetail("""{"detail":"something went wrong"}""") shouldBe ProblemDetail( + type = "about:blank", + title = null, + status = null, + detail = "something went wrong", + instance = null, + traceId = null, + ) + } }) /** diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/error-401-permission-denied.json b/alchemist-geospatial/src/test/resources/copernicus-responses/error-401-permission-denied.json new file mode 100644 index 0000000000..c31214377a --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/error-401-permission-denied.json @@ -0,0 +1,8 @@ +{ + "type": "permission denied", + "title": "permission denied", + "status": 401, + "detail": "authentication required", + "instance": "https://ads.atmosphere.copernicus.eu/api/retrieve/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68", + "trace_id": "b63a2882-2510-4ced-935a-b2faec13eead" +} \ No newline at end of file diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/error-404-result-not-ready.json b/alchemist-geospatial/src/test/resources/copernicus-responses/error-404-result-not-ready.json new file mode 100644 index 0000000000..3ba12a152c --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/error-404-result-not-ready.json @@ -0,0 +1,8 @@ +{ + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/result-not-ready", + "title": "job results not ready", + "status": 404, + "detail": "status of 61ebb7be-650e-4aa5-9039-6030eb01bb68 is 'accepted'", + "instance": "https://ads.atmosphere.copernicus.eu/api/retrieve/v1/jobs/61ebb7be-650e-4aa5-9039-6030eb01bb68/results", + "trace_id": "e7ba3606-9816-43cc-ab6a-4f0642388701" +} \ No newline at end of file From b9df06331f786cb0131f13d943748638e056509d Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 30 Jun 2026 01:14:57 +0200 Subject: [PATCH 048/118] feat(geospatial): add parseFailureMessage to parse jobs failure messages; fixed doc on parsers --- .../acquisition/CopernicusResponses.kt | 61 ++++++++++++++----- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt index 64cd0c318f..bd5425840c 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt @@ -20,10 +20,16 @@ import com.google.gson.JsonParser * `asset.value` envelope that is not a STAC document, a bare MD5 instead of a multihash), so these * parsers extract only the few fields needed and tolerate the rest. * + * Error bodies come in three distinct shapes, each parsed (or not) accordingly: + * - RFC 7807 problem-details on 4xx application errors (400/401/403/404) -> see parseProblemDetail(...). + * - FastAPI/Pydantic validation errors on 422, whose `detail` is an array of objects, NOT a + * string -> deliberately not parsed here (the caller falls back to the raw body); + * - the OGC job's top-level `message` string on a failed/rejected/dismissed job -> see parseFailureMessage(...). + * * Each function takes a raw response body string and is fully testable offline against captured * real responses. * - * For refence: + * For reference: * Open Geospatial Consortium API: https://docs.ogc.org/is/18-062r2/18-062r2.html#toc0 * Spatio Temporal Asset Catalogs: https://github.com/stac-extensions/file */ @@ -58,14 +64,15 @@ internal data class ProblemDetail( /** * Search for and returns the `href` property of the first link in the `links` - * array within the [json] whose `rel` property is equal to [rel]. + * array within the [json] whose `rel` property is equal to [rel]. Returns `null` + * if the body has no `links` array at all. * * ```json * An example of JSON body is: * { "links": [ { "rel": "self", "href": "..." }, * { "rel": "monitor", "href": "URL_TO_RETURN" } ] } * ``` - * In this example, with `rel = "monitor"` the return value would be `URL_TO_RETURN` + * In this example, with `rel = "monitor"` the return value would be `URL_TO_RETURN`. * * @param json the JSON body as a string. * @param rel the `rel` property associated with the desired `href`. @@ -81,9 +88,9 @@ private fun linkHref(json: String, rel: String): String? = JsonParser.parseStrin * Extracts the job-monitoring URL from a submit response [json] (`POST .../execute`). * * Returns the absolute `href` of the link whose `rel` is `"monitor"`. - * The same job is also identifiable via the top-level `jobID` field and the `Location` header. - * The `monitor` link is preferred because it keeps polling decoupled from - * the URL path layout while remaining a pure JSON parser. + * The same job is also identifiable via the top-level `jobID` field and the `Location` response header. + * The `monitor` link is preferred because it keeps polling decoupled from the URL path + * layout while remaining a pure JSON parser. * * @param json the JSON string to parse. * @return the `href` associated with `rel="monitor"` in the links array of [json]. @@ -96,12 +103,15 @@ internal fun parseMonitorUrl(json: String): String = /** * Extracts the job status from a status (`GET .../jobs/{id}`) response [json]. * - * Known values include `accepted`, `running`, `successful`, `failed`, `dismissed`. Callers should - * treat any value other than `successful`/`failed` as "keep polling", rather than enumerating the - * intermediate states, so an unforeseen status does not break the wait loop. + * The values defined by the data store's OpenAPI schema are `accepted`, `running`, `successful`, + * `failed`, `rejected`, and `dismissed`. Of these, `successful` is the sole success and `failed`, + * `rejected`, `dismissed` are terminal failures; `accepted`/`running` are transient. The status is + * returned as a **raw string**, not an enum, because the API is marked unsupported/evolving: an + * unforeseen value must not crash callers, which should treat any unrecognized status as transient + * ("keep polling") rather than enumerating it. * * @param json the JSON string to parse. - * @return the request's processing status. + * @return the job's processing status, verbatim. * * @throws IllegalStateException if the `status` field is absent. */ @@ -114,12 +124,13 @@ internal fun parseStatus(json: String): String = JsonParser.parseString(json) * Extracts the results URL from a status (`GET .../jobs/{id}`) response [json], or `null` if the * job exposes no results link yet. * - * The `rel="results"` link appears only once the status is - * `successful`; an `accepted`/`running` job exposes only `rel="self"`. When this returns `null` - * on an already-successful job, the caller may fall back to `"{statusUrl}/results"`. + * The `rel="results"` link appears only once the status is `successful`; an `accepted`/`running` + * job exposes only `rel="self"`. A `null` on an already-`successful` job indicates an inconsistent + * server response and the caller should fail loudly rather than reconstructing a `.../results` + * path by hand. * * @param json the JSON string to parse. - * @return the results URL or `null` if no result link is present. + * @return the results URL, or `null` if no `rel="results"` link is present. */ internal fun parseResultsUrl(json: String): String? = linkHref(json, "results") @@ -130,12 +141,12 @@ internal fun parseResultsUrl(json: String): String? = linkHref(json, "results") * without authentication; absolute as served by ECMWF), `file:size`, and the optional * `file:checksum`. * - * **On the shape.** The field names `file:size`/`file:checksum` are STAC File Info Extension + * **Note on the shape**: the field names `file:size`/`file:checksum` are STAC File Info Extension * naming, but the `asset.value` envelope is an ECMWF convention, NOT STAC structure: a canonical * STAC asset lives in an `assets` map with these fields directly on it (no `value` wrapper), and * this body is not a STAC document. Hence, the two nested lookups (`asset` then `value`). * - * **On the checksum.** ECMWF emits `file:checksum` as a bare lowercase MD5 hex string, despite the + * **Note on the checksum**: ECMWF emits `file:checksum` as a bare lowercase MD5 hex string, despite the * STAC file extension nominally prescribing a self-identifying multihash, so it is captured * verbatim. It is nullable because some datasets/stores may omit it. * @@ -161,6 +172,10 @@ internal fun parseAsset(json: String): RemoteAsset { * failed request (e.g. a `404` result-not-ready, a `401` authentication required, a `403` * dataset-license not accepted, a `400` invalid request). * + * This parser targets the RFC 7807 shape only. It is **not** suitable for `422` + * validation bodies, whose `detail` is an array of error objects (not a string): calling it on a + * 422 will throw on the `detail` lookup. Callers that may encounter a 422 must guard the call. + * * All fields are optional per RFC 7807 except `type` (which defaults to `"about:blank"`), so every * field but [ProblemDetail.type] is nullable. Note that ECMWF does not always honor the spec's * recommendation that `type` be a URI (it sometimes repeats the human-readable title, e.g. @@ -184,3 +199,17 @@ internal fun parseProblemDetail(json: String): ProblemDetail { traceId = field("trace_id"), ) } + +/** + * Extracts the failure message from a failed/rejected/dismissed job status body, or `null` if absent. + * + * The detail lives in a top-level `message` string (OGC job schema), not in an RFC 7807 body, so + * this does NOT go through [parseProblemDetail]. The schema declares `message` as nullable, and this + * runs on the error path: hence it returns `null` instead of throwing, leaving the caller to fall + * back to the raw JSON body. + * + * @param json the JSON status body as a string. + * @return the `message` string, or `null` if missing/null. + */ +internal fun parseFailureMessage(json: String): String? = JsonParser.parseString(json) + .asJsonObject.get("message")?.takeUnless { it.isJsonNull }?.asString From c75ca00fba42f03b2a93a5e990227da3cab8e2ea Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 30 Jun 2026 01:25:12 +0200 Subject: [PATCH 049/118] test(geospatial): add tests for failed job response parsing --- .../acquisition/TestCopernicusResponses.kt | 35 +++++++++++++++++++ .../error-400-invalid-request.json | 8 +++++ 2 files changed, 43 insertions(+) create mode 100644 alchemist-geospatial/src/test/resources/copernicus-responses/error-400-invalid-request.json diff --git a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt index bc0034e8b8..13a8f10202 100644 --- a/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt +++ b/alchemist-geospatial/src/test/kotlin/it/unibo/alchemist/model/geospatial/acquisition/TestCopernicusResponses.kt @@ -125,6 +125,41 @@ class TestCopernicusResponses : StringSpec({ traceId = null, ) } + + "parseProblemDetail extracts all fields from a 400 invalid-request body" { + parseProblemDetail(loadBody("error-400-invalid-request.json")) shouldBe ProblemDetail( + type = "invalid request", + title = "invalid request", + status = 400, + detail = "Request has not produced a valid combination of values, " + + "please check your selection.\n" + + "{'data_format': ['netcdf'], 'day': ['10'], 'hydrological_model': ['lisflood'], " + + "'leadtime_hour': ['26'], 'month': ['02'], 'product_type': ['control_forecast'], " + + "'system_version': ['operational'], 'variable': ['river_discharge_in_the_last_24_hours'], " + + "'year': ['2024']}", + instance = "https://ewds.climate.copernicus.eu/api/retrieve" + + "/v1/processes/cems-glofas-forecast/execute", + traceId = "cec329b8-cb55-4b84-a3a8-86b85facdbb4", + ) + } + + /* + * no real failed-job body has been captured (failed jobs are rare on stable datasets). + * A failed job would occur after a successful submit (all fields in the request are validated + * correctly in the ECMWF backend) but somehow the requested data can't be processed/returned. + * The following are SYNTHETIC bodies, NOT ECMWF's failed-jobs responses. + */ + "parseFailureMessage reads a top-level message string" { + parseFailureMessage("""{"status":"failed","message":"the job blew up"}""") shouldBe "the job blew up" + } + + "parseFailureMessage returns null when no message is present" { + parseFailureMessage("""{"status":"failed"}""") shouldBe null + } + + "parseFailureMessage returns null when message is JSON null" { + parseFailureMessage("""{"status":"failed","message":null}""") shouldBe null + } }) /** diff --git a/alchemist-geospatial/src/test/resources/copernicus-responses/error-400-invalid-request.json b/alchemist-geospatial/src/test/resources/copernicus-responses/error-400-invalid-request.json new file mode 100644 index 0000000000..6d7adeaa47 --- /dev/null +++ b/alchemist-geospatial/src/test/resources/copernicus-responses/error-400-invalid-request.json @@ -0,0 +1,8 @@ +{ + "type": "invalid request", + "title": "invalid request", + "status": 400, + "detail": "Request has not produced a valid combination of values, please check your selection.\n{'data_format': ['netcdf'], 'day': ['10'], 'hydrological_model': ['lisflood'], 'leadtime_hour': ['26'], 'month': ['02'], 'product_type': ['control_forecast'], 'system_version': ['operational'], 'variable': ['river_discharge_in_the_last_24_hours'], 'year': ['2024']}", + "instance": "https://ewds.climate.copernicus.eu/api/retrieve/v1/processes/cems-glofas-forecast/execute", + "trace_id": "cec329b8-cb55-4b84-a3a8-86b85facdbb4" +} \ No newline at end of file From 7db3dc76da7153ab846c664898805a2bc1cb2fd8 Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 30 Jun 2026 16:22:03 +0200 Subject: [PATCH 050/118] feat(geospatial): add describe function to ProblemDetail class --- .../acquisition/CopernicusResponses.kt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt index bd5425840c..d1fee8a661 100644 --- a/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt +++ b/alchemist-geospatial/src/main/kotlin/it/unibo/alchemist/model/geospatial/acquisition/CopernicusResponses.kt @@ -60,7 +60,23 @@ internal data class ProblemDetail( val detail: String?, val instance: String?, val traceId: String?, -) +) { + /** + * Extracts a human-readable summary of this problem-detail, or an + * empty string if no content is extractable. + * + * @return a human-readable string describing this problem. + */ + fun describe(): String { + val core = detail ?: title ?: return "" + return buildString { + append(core) + append(" [type=$type") + traceId?.let { append(", trace=$it") } + append("]") + } + } +} /** * Search for and returns the `href` property of the first link in the `links` From c11ce96dd433cb3c182a0c98d3f2f1929a591dad Mon Sep 17 00:00:00 2001 From: Emir Wanes Aouioua Date: Tue, 30 Jun 2026 16:25:00 +0200 Subject: [PATCH 051/118] chore(geospatial): add PRIVATE-TOKEN option in .idea Project_Default --- .idea/inspectionProfiles/Project_Default.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 7bee31df84..dc22d00cb3 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -5,6 +5,13 @@