Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
119 commits
Select commit Hold shift + click to select a range
e4e0fbf
chore(geospatial): add basic gradle configuration
Wanes01 Jun 20, 2026
868c1b1
feat(geospatial): add RasterGrid interface and ArrayRasterGrid implem…
Wanes01 Jun 20, 2026
62840ec
feat(geospatial): add TimedGrid and CdmTimedGrid implementation
Wanes01 Jun 20, 2026
fd364bd
chore(geospatial): update cdm dependency version to latest (5.9.1)
Wanes01 Jun 21, 2026
7489e93
test(geospatial): add tests for ArrayRasterGrid and TestCdmTimedGrid
Wanes01 Jun 21, 2026
f8c91b7
feat(geospatial): add Axes helpers for index finding and their tests
Wanes01 Jun 21, 2026
eeff0e5
feat(geospatial): add SpatialInterpolationStrategy and an enum with b…
Wanes01 Jun 21, 2026
f6a027f
test(geospatial): add test for SpatialInterpolation enum
Wanes01 Jun 21, 2026
b7e3cc7
feat(geospatial): add SpatialExtrapolationStrategy interface and buil…
Wanes01 Jun 22, 2026
5290a0a
refactor(geospatial): modified file name from SpatialExtrapolation to…
Wanes01 Jun 22, 2026
365d9e2
test(geospatial): add tests for SpatialExtrapolation built-in strategies
Wanes01 Jun 22, 2026
3f1745f
feat(geospatial): add MissingValueStrategy and built-in implementation
Wanes01 Jun 22, 2026
ef95266
test(geospatial): add tests for MissingValue strategies
Wanes01 Jun 22, 2026
655b7c2
feat(geospatial): add SpatialSampler to orchestrate spatial strategies
Wanes01 Jun 23, 2026
58fd9ab
test(geospatial): add tests for SpatialSampler (spatial strategies or…
Wanes01 Jun 23, 2026
893f1cd
feat(geospatial): add TemporalInterpolationStrategy to interpolate be…
Wanes01 Jun 23, 2026
8c2285f
test(geospatial): add tests for TemporalInterpolation strategies
Wanes01 Jun 23, 2026
e5a74db
feat(geospatial): add TemporalExtrapolation strategies
Wanes01 Jun 23, 2026
445a255
test(geospatial): add tests for TemporalExtrapolation strategies
Wanes01 Jun 23, 2026
1c19ebf
refactor(geospatial): make raster and strategy types Serializable
Wanes01 Jun 24, 2026
469001b
feat(geospatial): add not fully implemented GeoRasterLayer (costructo…
Wanes01 Jun 24, 2026
9f4bfd3
refactor(geospatial): move writeTestNetcdf function in a shared test …
Wanes01 Jun 24, 2026
52be282
chore(geospatial): remove mockk dependency from build.gradle file
Wanes01 Jun 24, 2026
dcd74f0
test(geospatial): add tests for GeoRasterLayer
Wanes01 Jun 25, 2026
06317cc
feat(geospatial): add CanonicalJson singleton to convert any structur…
Wanes01 Jun 25, 2026
74efaff
test(geospatial): add tests for CanonicalJson key ordering
Wanes01 Jun 25, 2026
ace407d
feat(geospatial): add CacheKey interface to get a deterministic cache…
Wanes01 Jun 26, 2026
40d2208
feat(geospatial): add FileNames utility file; add function to sanitiz…
Wanes01 Jun 26, 2026
35eecf4
test(geospatial): add tests for String.toFileSystemSafe utility
Wanes01 Jun 26, 2026
be128f0
feat(geospatial): add CopernicusRequest cache key for ECMWF requests
Wanes01 Jun 26, 2026
cc83039
test(geospatial): add tests for CopernicusRequest
Wanes01 Jun 26, 2026
5182426
feat(geospatial): add CdsApiRc to retrieve the user's token to use in…
Wanes01 Jun 26, 2026
9388b7e
test(geospatial): add tests for CdsApiRc token retrival
Wanes01 Jun 26, 2026
6ecb1d4
docs(geospatial): clarified that the cdsapirc token format is not che…
Wanes01 Jun 26, 2026
b54efb3
feat(geospatial): add CacheManager to centralize cache entries creation
Wanes01 Jun 27, 2026
9a7cc80
test(geospatial): add tests for CacheManager
Wanes01 Jun 27, 2026
113f7df
feat(geospatial): add Archives utility to extract zip archives into a…
Wanes01 Jun 27, 2026
3689dce
test(geospatial): add tests for archives extraction utility
Wanes01 Jun 27, 2026
326f3c5
docs(geospatial): fixed clarification on AfterSpec callback in test c…
Wanes01 Jun 27, 2026
7ddb8bb
feat(geospatial): add ExternalDataProvider interface to abstract the …
Wanes01 Jun 28, 2026
5211f17
feat(geospatial): add Integrity functions to check download resources…
Wanes01 Jun 28, 2026
028048d
test(geospatial): add file md5 check utility function
Wanes01 Jun 28, 2026
c4c9064
docs(geospatial): updated hex doc in CopernicusRequest
Wanes01 Jun 28, 2026
e9e2902
feat(geospatial): add parsers for Copernicus json responses in Copern…
Wanes01 Jun 29, 2026
ab4121c
test(geospatial): add tests for CopernicusResponses parsers along wit…
Wanes01 Jun 29, 2026
2882c41
feat(geospatial): add a response error parser in CopernicusResponses
Wanes01 Jun 29, 2026
6984fec
test(geospatial): add tests for Copernicus error parser
Wanes01 Jun 29, 2026
b9df063
feat(geospatial): add parseFailureMessage to parse jobs failure messa…
Wanes01 Jun 29, 2026
c75ca00
test(geospatial): add tests for failed job response parsing
Wanes01 Jun 29, 2026
7db3dc7
feat(geospatial): add describe function to ProblemDetail class
Wanes01 Jun 30, 2026
c11ce96
chore(geospatial): add PRIVATE-TOKEN option in .idea Project_Default
Wanes01 Jun 30, 2026
3985952
feat(geospatial): add CopernicusDataStoreProvider to retrieve coperni…
Wanes01 Jun 30, 2026
aeca7b6
chore(geospatial): modified RasterGrid to be generic on the hold value
Wanes01 Jul 22, 2026
50f67cd
feat(geospatial): rename TimedGrid into GridSnapshots and make it gen…
Wanes01 Jul 22, 2026
e480971
feat(geospatial): remove extrapolation strategies
Wanes01 Jul 23, 2026
2f19dff
feat(geospatial): trasform missing value strategy into a generic valu…
Wanes01 Jul 23, 2026
9821d94
feat(geospatial): make SpatialInterpolationStrategy generic; convert …
Wanes01 Jul 23, 2026
ed13400
feat(geospatial): make TemporalInterpolationStrategy generic; convert…
Wanes01 Jul 24, 2026
cb72f97
refactor(geospatial): delete SpatialSampler; behavior to incorporate …
Wanes01 Jul 24, 2026
02ea846
docs(geospatial): updated class references in TestUtils
Wanes01 Jul 24, 2026
33280fc
feat(geospatial): make ArrayRasterGrid generic; add measurement conve…
Wanes01 Jul 24, 2026
59bf97e
feat(geospatial): change GeoRasterLayer structure to be generic on ge…
Wanes01 Jul 25, 2026
9b4057c
feat(geospatial): add MapRasterGrid variant for sparse grids
Wanes01 Jul 25, 2026
2d42e13
refactor(geospatial): use MapRasterGrid in CdmGridSnapshots when grid…
Wanes01 Jul 25, 2026
9a11c90
refactor(geospatial): move acquisition's utility classes into a dedic…
Wanes01 Jul 25, 2026
8b88caa
feat(geospatial): bind requests and provider in CacheManager
Wanes01 Jul 26, 2026
5b9c443
feat(geospatial): add FakeHttpServer, a in-JVM fake rest webserver fo…
Wanes01 Jul 26, 2026
edf2416
test(geospatial): add tests for CopernicusDataStoreProvider using a l…
Wanes01 Jul 26, 2026
845bd53
test(geospatial): add tests for CopernicusCacheManager
Wanes01 Jul 27, 2026
628e83f
refactor(geospatial): rename CopernicusResponses to CopernicusParsers
Wanes01 Jul 29, 2026
a4d2a96
feat(geospatial): modify RasterGrid and GridSnapshots to hold Double …
Wanes01 Aug 6, 2026
415cd8d
feat(geospatial): replace missing value strategy with a raw Double co…
Wanes01 Aug 6, 2026
d42a62c
feat(geospatial): remove geric from strategies; add SpatioTemporalInt…
Wanes01 Aug 6, 2026
f57dfa7
feat(geospatial): adapt CopernicusLayer to SpatioTemporalInterpolatio…
Wanes01 Aug 6, 2026
01459f3
feat(geospatial): convert java Instants and Duration to their Kotlin …
Wanes01 Aug 6, 2026
431a535
refactor(geospatial): centralize GeoPosition mock into an internal fu…
Wanes01 Aug 6, 2026
0f97be3
feat(geospatial): add lazy initialization for ECMWF token
Wanes01 Aug 7, 2026
024ffa3
feat(geospatial): add function to expand a tilde into the user home dir
Wanes01 Aug 8, 2026
36dfa58
feat(geospatial): add CopernicusLayer's constructors to be used with …
Wanes01 Aug 8, 2026
67063c1
test(geospatial): re-captured ECMWF response against /execution
Wanes01 Aug 9, 2026
50f3598
fix(geospatial): pad advertised MD5 checksums back to 32 digits
Wanes01 Aug 9, 2026
6082f5c
fix(geospatial): adapt Copernicus parsers and provider to previews ch…
Wanes01 Aug 9, 2026
7fb5fec
style(geospatial): fix detekt warnings
Wanes01 Aug 10, 2026
9dd1a1e
fix(geospatial): swap RasterGrid implementation use in CdmGridSnapshots
Wanes01 Aug 10, 2026
9e66bb0
refactor(geospatial): refactor geospatial module packages
Wanes01 Aug 10, 2026
a5a4749
fix(geospatial): implement equals/hashCode on strategy implementations
Wanes01 Aug 11, 2026
e914445
build(maps): pin protobuf-java to 3.12.2 to avoid a runtime conflict …
Wanes01 Aug 11, 2026
b04b2f6
docs(geospatial): remove SAX note from CdmGridSnaphots
Wanes01 Aug 11, 2026
cd6a227
feat(geospatial): enhance Copernicus' provider and cache manager logs
Wanes01 Aug 12, 2026
e447ce1
fix(geospatial): fix type key not usable in yml simulation config fil…
Wanes01 Aug 12, 2026
ccc4cc5
feat(geospatial): add latitudes and longitudes ascending order check …
Wanes01 Aug 13, 2026
aea9c08
feat(geospatial): make CdmGridSnaphots accept any order of {time, lat…
Wanes01 Aug 13, 2026
05b9b4d
refactor(geospatial): turn RasteGrid into an abstract class to centra…
Wanes01 Aug 13, 2026
467ba6d
docs(geospatial): correct typos and reduce duplication of documentation
Wanes01 Aug 14, 2026
f36134c
refactor(geospatial): rename CdmGridSnaphots into EagerGridSnaphots; …
Wanes01 Aug 14, 2026
558de81
build(full): pin protobuf-java to 3.12.2 to avoid a runtime conflict …
Wanes01 Aug 14, 2026
456be2a
fix(geospatial): use netcdf runtime attribute as a fallback if 'time'…
Wanes01 Aug 16, 2026
d49346e
feat(geospatial): EagerGridSnaphots do not fails on a duplicate insta…
Wanes01 Aug 16, 2026
d09112e
docs(geospatial): fix typo in GridReading
Wanes01 Aug 16, 2026
6d56095
refactor(geospatial): replace residual TimedGrid references in tests …
Wanes01 Aug 17, 2026
e707f3d
fix(geospatial): address SonarCloud findings
Wanes01 Aug 17, 2026
6b50014
docs(geospatial): remove NetCDF-Java link from EagerGridSnaphots
Wanes01 Aug 17, 2026
b7d5a27
fix(geospatial): fix RasterGrid exposing gridValues as a property
Wanes01 Aug 18, 2026
34aba38
docs(geospatial): replace param with property on latitudes and longit…
Wanes01 Aug 18, 2026
de0f68a
refactor(geospatial): avoid inconsistent vertical spacing
Wanes01 Aug 18, 2026
e590b6f
feat(geospatial): remove expandUser function to read absoltue path only
Wanes01 Aug 18, 2026
1047640
refactor(geospatial): move utility functions and classes into utils s…
Wanes01 Aug 18, 2026
c91b0e1
fix(geospatial): fix EagerGridSnaphots failing on GRIB files
Wanes01 Aug 18, 2026
d25c08c
build: use a custom version of CDM that can be fetched from Maven Cen…
DanySK Aug 19, 2026
a318315
test(geospatial): tests for GRIB file reading
Wanes01 Aug 19, 2026
86b39d5
build(maps): stop using a protobuf fixed version
Wanes01 Aug 19, 2026
200c4ad
build(full): stop using a protobuf fixed version
Wanes01 Aug 19, 2026
09fc973
fix(geospatial): swap sax parser for NecdfDatasets.openDataset
Wanes01 Aug 19, 2026
8264879
fix(full): merge plugin.properties if there are multiple entries
Wanes01 Aug 19, 2026
d20a0cb
build: add AppendingTransformer for merging plugin.properties in Shad…
DanySK Aug 20, 2026
8337419
docs(geospatial): align module doc to its latest changes; various doc…
Wanes01 Aug 20, 2026
cd42cd6
Merge branch 'master' into alchemist-geospatial
Wanes01 Aug 21, 2026
0e9df63
test(geospatial): add test to detect compatibility issues between OSM…
Wanes01 Aug 21, 2026
33e418d
build(full): remove the duplicate AppendingTransformer under the Shad…
Wanes01 Aug 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .idea/inspectionProfiles/Project_Default.xml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions alchemist-full/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
*/
import Libs.alchemist
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer
import com.google.common.hash.Hashing
import it.unibo.alchemist.build.commandExists
import it.unibo.alchemist.build.isMac
Expand Down Expand Up @@ -81,6 +82,8 @@ tasks.withType<ShadowJar>().configureEach {
isZip64 = true
mergeServiceFiles()
duplicatesStrategy = DuplicatesStrategy.INCLUDE
// EMF's plugin.properties must be merged if there are multiple entries
transform<AppendingTransformer> { resource = "plugin.properties" }
destinationDirectory.set(rootProject.layout.buildDirectory.map { it.dir("shadow") })
}

Expand Down
40 changes: 40 additions & 0 deletions alchemist-geospatial/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2010-2026, Danilo Pianini and contributors
* listed, for each module, in the respective subproject's build.gradle.kts file.
*
* This file is part of Alchemist, and is distributed under the terms of the
* GNU General Public License, with a linking exception,
* as described in the file LICENSE in the Alchemist distribution's top directory.
*/

import Libs.alchemist

plugins {
id("kotlin-jvm-convention")
}

dependencies {
api(alchemist("api"))

implementation(libs.cdm.core)
implementation(libs.cdm.grib)
implementation(libs.gson)
implementation(libs.guava)
implementation(libs.slf4j)

testImplementation(alchemist("maps"))
testImplementation(alchemist("test"))
}

publishing.publications {
withType<MavenPublication> {
pom {
contributors {
contributor {
name.set("Emir Wanes Aouioua")
email.set("emirwanes.aouioua@studio.unibo.it")
}
}
}
}
}
10 changes: 10 additions & 0 deletions alchemist-geospatial/gradle.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#
# Copyright (C) 2010-2026, Danilo Pianini and contributors listed in the main project's alchemist/build.gradle file.
#
# This file is part of Alchemist, and is distributed under the terms of the
# GNU General Public License, with a linking exception,
# as described in the file LICENSE in the Alchemist distribution's top directory.
#
artifactId = alchemist-geospatial
projectLongName = Alchemist Geospatial
projectDescription = Integration of Copernicus/ECMWF Temporal Raster Data into the Alchemist Simulator
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright (C) 2010-2026, Danilo Pianini and contributors
* listed, for each module, in the respective subproject's build.gradle.kts file.
*
* This file is part of Alchemist, and is distributed under the terms of the
* GNU General Public License, with a linking exception,
* as described in the file LICENSE in the Alchemist distribution's top directory.
*/

package it.unibo.alchemist.boundary.acquisition

/**
* Deterministic identity of a request.
*/
fun interface CacheKey {

/**
* @return a **deterministic, file-system-safe** directory name derived *only* from what
* determines the downloaded bytes, so that equal content always maps to the same name. The
* returned name must be a single path segment: no separators (`/`, `\`), and safe on different OS.
*/
fun toFileName(): String
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright (C) 2010-2026, Danilo Pianini and contributors
* listed, for each module, in the respective subproject's build.gradle.kts file.
*
* This file is part of Alchemist, and is distributed under the terms of the
* GNU General Public License, with a linking exception,
* as described in the file LICENSE in the Alchemist distribution's top directory.
*/

package it.unibo.alchemist.boundary.acquisition

import java.nio.file.Path

/**
* Manages disk-based cache directories identified by deterministic [CacheKey] requests.
*
* When a directory is requested via [getOrProduce], the manager resolves its path.
* If the directory is missing or empty, it automatically fetches and populates the
* data using the configured [provider].
*
* @param R the type of [CacheKey] accepted by this manager.
*
* @see ExternalDataProvider
* @see CacheKey
*/
interface CacheManager<in R : CacheKey> {

/**
* The provider responsible for populating the directory associated with the cache entry in
* [CacheManager.getOrProduce], in the event that it does not exist or is empty.
*/
val provider: ExternalDataProvider<R>

/**
* Calculates and returns the directory where the data was cached via the [request].
*
* @param request the request used to calculate the deterministic directory path.
* @return the path to the directory containing the data.
*/
fun getOrProduce(request: R): Path
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Copyright (C) 2010-2026, Danilo Pianini and contributors
* listed, for each module, in the respective subproject's build.gradle.kts file.
*
* This file is part of Alchemist, and is distributed under the terms of the
* GNU General Public License, with a linking exception,
* as described in the file LICENSE in the Alchemist distribution's top directory.
*/

package it.unibo.alchemist.boundary.acquisition

import java.nio.file.FileSystemException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import org.slf4j.LoggerFactory

/**
* Filesystem cache of the directories produced by data providers. This is the **single** place
* where cache atomicity lives, once for every provider. It knows nothing about HTTP or the
* individual APIs, depending only on [CacheKey].
*
* This class considers **atomicity but non locking**: if two simultaneous cache-miss runs
* for the production of data, one wins the atomic rename, the other discards its own work.
* Wasteful in that rare case, but it never corrupts the cache.
*
* **Note!** The entries are **trust-based**: a present directory is assumed complete and valid: the content
* of a cache-hit is not re-verified. Manual alteration of a cache entry is out of contract.
*
* @param root cache root directory (e.g. `~/.alchemist/cache/geospatial`). Created on demand.
*/
class CopernicusCacheManager(override val provider: ExternalDataProvider<CopernicusRequest>, private val root: Path) :
CacheManager<CopernicusRequest> {

/**
* Temporary subdirectory, on the same filesystem as [root].
* (required for `ATOMIC_MOVE`).
*/
private val tmpRoot: Path = root.resolve(TEMP_SUBDIR)

/**
* Returns the directory associated with [request], producing it if absent.
*
* On **cache hit** the existing directory is returned immediately.
* On **cache miss**, the [provider] runs into a temporary directory, its non-emptiness is
* validated, and it is then promoted to the final location with an atomic rename.
* If a concurrent process produced the same entry in the meantime, that one is used
* and the local work is discarded. If, for any reason, [provider] fails, the temporary
* directory is removed (no "poisoned" entry is left in cache).
*
* @param request request identity (to determine the directory name).
* @return the [Path] of the final cache directory, filled with data.
* @throws IllegalStateException if [provider] writes no file in the temporary directory.
*/
override fun getOrProduce(request: CopernicusRequest): Path {
val finalDir = root.resolve(request.toFileName())
// cache hit: the directory already exists
if (Files.isDirectory(finalDir)) {
logger.info("Cache hit for '${request.toFileName()}': using $finalDir")
return finalDir
}
logger.info("Cache miss for '${request.toFileName()}': fetching data")
// cache miss (also creates root if it does not exist)
Files.createDirectories(tmpRoot)
val temp = Files.createTempDirectory(tmpRoot, request.toFileName())
var moved = false // becomes true if the directory gets promoted
try {
// tries to fill the directory with data
provider.fetch(request, temp)
check(hasData(temp)) { "Provider produced no files for '${request.toFileName()}'" }
moved = promote(temp, finalDir)
logger.info("Asset(s) cached in $finalDir")
return finalDir
} finally {
// deletes the temp directory if any accident occurs
if (!moved) temp.toFile().deleteRecursively()
}
}

/**
* Atomically promotes [temp] to [finalDir].
*
* @param temp the path of the temporary directory.
* @param finalDir the path of the final directory after [temp] gets promoted.
* @return true if this call performed the move, false if a concurrent peer had already
* produced [finalDir].
*/
private fun promote(temp: Path, finalDir: Path): Boolean = try {
Files.move(temp, finalDir, StandardCopyOption.ATOMIC_MOVE)
true
} catch (raceLost: FileSystemException) {
if (!Files.isDirectory(finalDir)) throw raceLost
// peer won but temp dir still exists
false
}

/**
* Checks if [dir] directory holds at least one regular file.
*
* @return true if [dir] holds any regular file, false otherwise.
*/
private fun hasData(dir: Path): Boolean =
Files.list(dir).use { entries -> entries.anyMatch { Files.isRegularFile(it) } }

private companion object {
private val logger = LoggerFactory.getLogger(CopernicusCacheManager::class.java)
private const val TEMP_SUBDIR = ".tmp"
}
}
Loading
Loading