From b11f9c99d885ba9ec599e6d19e9cb3fd96a6e1fc Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Thu, 5 Feb 2026 11:33:28 +0100 Subject: [PATCH 01/74] Modify POM --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6f082cebf..1bdf4fc63 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ 28.0 - 0.10.13 + 0.11.0-SNAPSHOT 1.20.0 1.5.0 11 From 1a5a11244f1663657553d769d357d3f30ab22fb5 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 10 Feb 2026 15:43:07 +0100 Subject: [PATCH 02/74] Correct previous HM version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1bdf4fc63..6f082cebf 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ 28.0 - 0.11.0-SNAPSHOT + 0.10.13 1.20.0 1.5.0 11 From 1f6d6bbb71980e0d1ae7d82dfb4160702f4a4c42 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 10 Feb 2026 17:27:19 +0100 Subject: [PATCH 03/74] Remove apache.common.login --- kactors/org.integratedmodelling.kactors/build.properties | 1 - kdl/org.integratedmodelling.kdl/build.properties | 1 - kim/org.integratedmodelling.kim/build.properties | 1 - 3 files changed, 3 deletions(-) diff --git a/kactors/org.integratedmodelling.kactors/build.properties b/kactors/org.integratedmodelling.kactors/build.properties index aa338a94e..c324a919e 100644 --- a/kactors/org.integratedmodelling.kactors/build.properties +++ b/kactors/org.integratedmodelling.kactors/build.properties @@ -15,6 +15,5 @@ additional.bundles = org.eclipse.xtext.xbase,\ org.eclipse.emf.mwe2.launch,\ org.eclipse.emf.mwe2.lib,\ org.objectweb.asm,\ - org.apache.commons.logging,\ org.apache.log4j,\ com.ibm.icu diff --git a/kdl/org.integratedmodelling.kdl/build.properties b/kdl/org.integratedmodelling.kdl/build.properties index 18d540bf6..a7b32375b 100644 --- a/kdl/org.integratedmodelling.kdl/build.properties +++ b/kdl/org.integratedmodelling.kdl/build.properties @@ -15,6 +15,5 @@ additional.bundles = org.eclipse.xtext.xbase,\ org.eclipse.emf.mwe2.launch,\ org.eclipse.emf.mwe2.lib,\ org.objectweb.asm,\ - org.apache.commons.logging,\ org.apache.log4j,\ com.ibm.icu diff --git a/kim/org.integratedmodelling.kim/build.properties b/kim/org.integratedmodelling.kim/build.properties index 2072401d9..05568542e 100644 --- a/kim/org.integratedmodelling.kim/build.properties +++ b/kim/org.integratedmodelling.kim/build.properties @@ -13,6 +13,5 @@ additional.bundles = org.eclipse.xtext.xbase,\ org.eclipse.emf.mwe2.launch,\ org.eclipse.emf.mwe2.lib,\ org.objectweb.asm,\ - org.apache.commons.logging,\ org.apache.log4j,\ com.ibm.icu \ No newline at end of file From c3c3ac2a9eaf208a50de8fd8cdc5a055e892a0b3 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 10 Feb 2026 17:36:35 +0100 Subject: [PATCH 04/74] Move to 17 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 6f082cebf..1eccc584d 100644 --- a/pom.xml +++ b/pom.xml @@ -108,15 +108,15 @@ utf-8 utf-8 - 11 + 17 28.0 0.10.13 1.20.0 1.5.0 - 11 - 11 + 17 + 17 4.0.9 2.36.0 2.19.0 From 6ab0d4fd60ad0925d0e601ca6f2b4b9cd0dc60ff Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 10 Feb 2026 17:46:25 +0100 Subject: [PATCH 05/74] First step running ant file to upgrade GT, still not compiling --- .../klab/ogc/integration/Geoserver.java | 2 +- .../klab/ogc/integration/Postgis.java | 16 ++++---- .../klab/ogc/vector/files/VectorEncoder.java | 22 +++++----- .../klab/ogc/vector/files/VectorImporter.java | 10 ++--- .../ogc/vector/files/VectorValidator.java | 18 ++++----- .../klab/ogc/vector/wfs/WfsEncoder.java | 6 +-- .../klab/ogc/vector/wfs/WfsValidator.java | 6 +-- .../klab/raster/files/RasterEncoder.java | 4 +- .../klab/raster/files/RasterImporter.java | 10 ++--- .../klab/raster/files/RasterValidator.java | 8 ++-- .../klab/raster/wcs/WcsEncoder.java | 12 +++--- .../klab/stac/STACEncoder.java | 12 +++--- .../klab/stac/STACUtils.java | 2 +- .../stac/extensions/STACIIASAExtension.java | 6 +-- .../geoprocessing/adapters/GridAdapter.java | 4 +- .../core/BarnesSurfaceResolver.java | 14 +++---- .../core/CustomBarnesSurfaceInterpolator.java | 4 +- .../geoprocessing/core/KrigingResolver.java | 2 +- .../morphology/MaximaFinderInstantiator.java | 2 +- .../remotesensing/SentinelResolver.java | 2 +- .../viewshed/ViewshedResolver.java | 2 +- .../cdm/utils/NetCDFUtils.java | 8 ++-- .../components/geospace/extents/Envelope.java | 40 +++++++++---------- .../components/geospace/extents/Grid.java | 2 +- .../geospace/extents/Projection.java | 12 +++--- .../processing/PolygonInstantiatorJAI.java | 4 +- .../geospace/utils/GeotoolsUtils.java | 28 ++++++------- .../geospace/utils/SpatialDisplay.java | 28 ++++++------- .../geospace/visualization/Renderer.java | 12 +++--- .../network/algorithms/Project.java | 16 ++++---- .../OpenBuildingsInstantiator.java | 4 +- pom.xml | 4 +- .../klab/test/cli/WFSWithPreemptiveAuth.java | 2 +- 33 files changed, 162 insertions(+), 162 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java index cef07a298..4fed06170 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java @@ -39,7 +39,7 @@ import org.integratedmodelling.klab.utils.FileUtils; import org.integratedmodelling.klab.utils.MiscUtilities; import org.integratedmodelling.klab.utils.NumberUtils; -import org.opengis.coverage.grid.GridCoverage; +import org.geotools.api.coverage.grid.GridCoverage; import kong.unirest.GetRequest; import kong.unirest.HttpRequestWithBody; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Postgis.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Postgis.java index 93e39ed8a..a60a393d4 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Postgis.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Postgis.java @@ -12,16 +12,16 @@ import java.util.List; import java.util.Map; -import org.geotools.data.DataStore; -import org.geotools.data.DataStoreFinder; +import org.geotools.api.data.DataStore; +import org.geotools.api.data.DataStoreFinder; import org.geotools.data.DataUtilities; -import org.geotools.data.FeatureWriter; -import org.geotools.data.Transaction; +import org.geotools.api.data.FeatureWriter; +import org.geotools.api.data.Transaction; import org.geotools.data.postgis.PostgisNGDataStoreFactory; import org.geotools.data.shapefile.ShapefileDataStore; import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.data.simple.SimpleFeatureIterator; -import org.geotools.data.simple.SimpleFeatureSource; +import org.geotools.api.data.SimpleFeatureSource; import org.geotools.jdbc.JDBCDataStore; import org.geotools.util.factory.Hints; import org.integratedmodelling.klab.Configuration; @@ -46,9 +46,9 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.WKBReader; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.feature.type.AttributeDescriptor; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.type.AttributeDescriptor; import org.postgresql.util.PGobject; public class Postgis extends org.integratedmodelling.klab.persistence.postgis.Postgis { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorEncoder.java index 82cb37213..3533d3b82 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorEncoder.java @@ -19,11 +19,11 @@ import java.util.HashMap; import java.util.Map; -import org.geotools.data.DataStore; -import org.geotools.data.DataStoreFinder; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.DataStore; +import org.geotools.api.data.DataStoreFinder; +import org.geotools.api.data.FeatureSource; import org.geotools.data.shapefile.ShapefileDataStoreFactory; -import org.geotools.data.simple.SimpleFeatureSource; +import org.geotools.api.data.SimpleFeatureSource; import org.geotools.factory.CommonFactoryFinder; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; @@ -68,12 +68,12 @@ import org.locationtech.jts.geom.MultiPoint; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.feature.type.AttributeDescriptor; -import org.opengis.filter.Filter; -import org.opengis.filter.FilterFactory2; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.type.AttributeDescriptor; +import org.geotools.api.filter.Filter; +import org.geotools.api.filter.FilterFactory; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; /** * The Class RasterEncoder. @@ -139,7 +139,7 @@ public void encodeFromFeatures(FeatureSource s Map urnParameters, IGeometry geometry, Builder builder, IContextualizationScope scope) { Filter filter = null; - FilterFactory2 ff = CommonFactoryFinder.getFilterFactory2(GeoTools.getDefaultHints()); + FilterFactory ff = CommonFactoryFinder.getFilterFactory(GeoTools.getDefaultHints()); Scale requestScale = geometry instanceof Scale ? (Scale) geometry : Scale.create(geometry); /* diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorImporter.java index b36b53b19..65cd49c55 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorImporter.java @@ -13,11 +13,11 @@ import java.util.Map; import org.geotools.data.DefaultTransaction; -import org.geotools.data.Transaction; +import org.geotools.api.data.Transaction; import org.geotools.data.shapefile.ShapefileDataStore; import org.geotools.data.shapefile.ShapefileDataStoreFactory; -import org.geotools.data.simple.SimpleFeatureSource; -import org.geotools.data.simple.SimpleFeatureStore; +import org.geotools.api.data.SimpleFeatureSource; +import org.geotools.api.data.SimpleFeatureStore; import org.geotools.feature.DefaultFeatureCollection; import org.geotools.feature.FeatureCollection; import org.geotools.feature.simple.SimpleFeatureBuilder; @@ -48,8 +48,8 @@ import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; public class VectorImporter extends AbstractFilesetImporter { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorValidator.java index 2b14fbe78..07b792e2c 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorValidator.java @@ -26,10 +26,10 @@ import java.util.Map; import org.eclipse.xtext.util.Arrays; -import org.geotools.data.DataAccessFactory; -import org.geotools.data.DataStore; -import org.geotools.data.DataStoreFinder; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.DataAccessFactory; +import org.geotools.api.data.DataStore; +import org.geotools.api.data.DataStoreFinder; +import org.geotools.api.data.FeatureSource; import org.geotools.data.shapefile.ShapefileDataStoreFactory; import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; @@ -60,11 +60,11 @@ import org.integratedmodelling.klab.utils.MiscUtilities; import org.integratedmodelling.klab.utils.StringUtils; import org.integratedmodelling.klab.utils.Utils; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.feature.type.AttributeDescriptor; -import org.opengis.filter.Filter; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.type.AttributeDescriptor; +import org.geotools.api.filter.Filter; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.locationtech.jts.geom.Lineal; import org.locationtech.jts.geom.Polygonal; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsEncoder.java index 414f43031..27b2ef577 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsEncoder.java @@ -17,7 +17,7 @@ import java.io.IOException; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.FeatureSource; import org.geotools.data.wfs.WFSDataStore; import org.integratedmodelling.klab.Version; import org.integratedmodelling.klab.api.data.IGeometry; @@ -26,8 +26,8 @@ import org.integratedmodelling.klab.exceptions.KlabIOException; import org.integratedmodelling.klab.ogc.WfsAdapter; import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; /** * The WFS encoder simply redefines the getFeatureSource method within a vector diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsValidator.java index 94f5509e6..fc6413f68 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/wfs/WfsValidator.java @@ -19,7 +19,7 @@ import java.net.URL; import java.util.Collection; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.FeatureSource; import org.geotools.data.wfs.WFSDataStore; import org.geotools.filter.text.cql2.CQLException; import org.geotools.filter.text.ecql.ECQL; @@ -30,8 +30,8 @@ import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor; import org.integratedmodelling.klab.ogc.WfsAdapter; import org.integratedmodelling.klab.ogc.vector.files.VectorValidator; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; /** * The Class WcsValidator. diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java index fe33871bb..a8489a7ad 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java @@ -67,8 +67,8 @@ import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.utils.MiscUtilities; import org.integratedmodelling.klab.utils.NumberUtils; -import org.opengis.coverage.grid.GridCoverage; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.coverage.grid.GridCoverage; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.geotools.coverage.util.CoverageUtilities; import it.geosolutions.jaiext.range.NoDataContainer; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterImporter.java index ca60bbe97..b979bab9b 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterImporter.java @@ -24,9 +24,9 @@ import org.geotools.data.shapefile.dbf.DbaseFileWriter; import org.geotools.gce.geotiff.GeoTiffWriteParams; import org.geotools.gce.geotiff.GeoTiffWriter; -import org.geotools.styling.ColorMap; -import org.geotools.styling.ColorMapEntry; -import org.geotools.styling.RasterSymbolizer; +import org.geotools.api.style.ColorMap; +import org.geotools.api.style.ColorMapEntry; +import org.geotools.api.style.RasterSymbolizer; import org.integratedmodelling.kim.api.IParameters; import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Observations; @@ -51,8 +51,8 @@ import org.integratedmodelling.klab.utils.Parameters; import org.integratedmodelling.klab.utils.Triple; import org.integratedmodelling.klab.utils.ZipUtils; -import org.opengis.parameter.GeneralParameterValue; -import org.opengis.parameter.ParameterValue; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValue; import it.geosolutions.imageio.plugins.tiff.BaselineTIFFTagSet; public class RasterImporter extends AbstractFilesetImporter { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterValidator.java index 637fb13f9..d8e63f7a7 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterValidator.java @@ -45,9 +45,9 @@ import org.integratedmodelling.klab.utils.FileUtils; import org.integratedmodelling.klab.utils.MiscUtilities; import org.integratedmodelling.klab.utils.URLUtils; -import org.opengis.coverage.SampleDimension; -import org.opengis.geometry.Envelope; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.coverage.SampleDimension; +import org.geotools.api.geometry.Bounds; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; /** * The Class RasterValidator. @@ -72,7 +72,7 @@ public IResource.Builder validate(String urn, URL url, IParameters userD AbstractGridFormat format = GridFormatFinder.findFormat(file); AbstractGridCoverage2DReader reader = format.getReader(file); GridCoverage2D coverage = reader.read(null); - Envelope envelope = coverage.getEnvelope(); + Bounds envelope = coverage.getEnvelope(); CoordinateReferenceSystem crs = coverage.getCoordinateReferenceSystem(); GridGeometry2D grid = coverage.getGridGeometry(); org.integratedmodelling.klab.components.geospace.extents.Envelope refenv = org.integratedmodelling.klab.components.geospace.extents.Envelope diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/wcs/WcsEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/wcs/WcsEncoder.java index b4e4fb3ca..ba55e940f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/wcs/WcsEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/wcs/WcsEncoder.java @@ -44,9 +44,9 @@ import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.raster.wcs.WCSService.WCSLayer; import org.integratedmodelling.klab.utils.FileUtils; -import org.opengis.coverage.grid.GridCoverage; -import org.opengis.geometry.DirectPosition; -import org.opengis.geometry.Envelope; +import org.geotools.api.coverage.grid.GridCoverage; +import org.geotools.api.geometry.Position; +import org.geotools.api.geometry.Bounds; /** * The Class WcsEncoder. @@ -136,10 +136,10 @@ public static File getAdjustedCoverage(InputStream input, IGeometry geometry) { double[] extent = space.getParameters().get(Geometry.PARAMETER_SPACE_BOUNDINGBOX, double[].class); GridCoverage2D coverage = OmsRasterReader.readRaster(coverageFile.getAbsolutePath()); - Envelope envelope = coverage.getEnvelope(); - DirectPosition lowerCorner = envelope.getLowerCorner(); + Bounds envelope = coverage.getEnvelope(); + Position lowerCorner = envelope.getLowerCorner(); double[] westSouth = lowerCorner.getCoordinate(); - DirectPosition upperCorner = envelope.getUpperCorner(); + Position upperCorner = envelope.getUpperCorner(); double[] eastNorth = upperCorner.getCoordinate(); org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(extent[0], diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 5586387d7..8378e6d0a 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import org.geotools.coverage.grid.GridCoverage2D; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.FeatureSource; import org.geotools.geometry.jts.ReferencedEnvelope; import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacItem; @@ -60,12 +60,12 @@ import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.geometry.DirectPosition; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.geometry.Position; import org.geotools.referencing.CRS; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java index f0856fa00..99302419d 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java @@ -11,7 +11,7 @@ import org.integratedmodelling.klab.api.provenance.IArtifact.Type; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; import org.integratedmodelling.klab.utils.DOIReader; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeature; import com.fasterxml.jackson.core.JsonParseException; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java index 345f52f22..8ba031526 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java @@ -5,12 +5,12 @@ import java.util.Iterator; import java.util.List; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.FeatureSource; import org.geotools.data.geojson.GeoJSONReader; import org.geotools.data.memory.MemoryDataStore; import org.integratedmodelling.klab.stac.STACUtils; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; import kong.unirest.HttpResponse; import kong.unirest.JsonNode; diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java index 79e4281ea..7d41fa927 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.Set; -import org.geotools.data.simple.SimpleFeatureSource; +import org.geotools.api.data.SimpleFeatureSource; import org.geotools.feature.FeatureIterator; import org.geotools.grid.Grids; import org.integratedmodelling.klab.Configuration; @@ -44,7 +44,7 @@ import org.mapdb.DB; import org.mapdb.DBMaker; import org.mapdb.Serializer; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeature; @UrnAdapter(type = GridAdapter.NAME, version = Version.CURRENT) public class GridAdapter implements IUrnAdapter { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java index 7717a3d9e..e2a1bb207 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java @@ -48,12 +48,12 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.geometry.Envelope; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.crs.GeographicCRS; -import org.opengis.referencing.crs.ProjectedCRS; -import org.opengis.referencing.datum.Ellipsoid; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.geometry.Bounds; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.GeographicCRS; +import org.geotools.api.referencing.crs.ProjectedCRS; +import org.geotools.api.referencing.datum.Ellipsoid; public class BarnesSurfaceResolver extends AbstractContextualizer implements IResolver, IExpression { @@ -87,7 +87,7 @@ public IState resolve(IState target, IContextualizationScope context) throws Kla grid.getEast(), grid.getWest(), (int) grid.getXCells(), (int) grid.getYCells(), grid.getProjection().getCoordinateReferenceSystem()); - Envelope env = inInterpolationGrid.getEnvelope(); + Bounds env = inInterpolationGrid.getEnvelope(); CoordinateReferenceSystem Crs = grid.getProjection().getCoordinateReferenceSystem(); diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java index aef6fb597..6c100f27a 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java @@ -22,8 +22,8 @@ import org.geotools.referencing.GeodeticCalculator; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.datum.Ellipsoid; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.datum.Ellipsoid; /** * Interpolates a surface across a regular grid from an irregular set of data points using the diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java index 30ceb2e77..375a05867 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java @@ -39,7 +39,7 @@ import org.integratedmodelling.klab.utils.NumberUtils; import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeatureType; public class KrigingResolver extends AbstractContextualizer implements IResolver, IExpression { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java index 6a77f0e8c..84fb4c8db 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java @@ -31,7 +31,7 @@ import org.integratedmodelling.klab.rest.StateSummary; import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.utils.Parameters; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeature; public class MaximaFinderInstantiator extends AbstractContextualizer implements IInstantiator, IExpression { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java index 9c5b9aac1..818646a37 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java @@ -34,7 +34,7 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; public class SentinelResolver extends AbstractContextualizer implements IResolver, IExpression { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java index 8c6a47e6c..144f345be 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java @@ -29,7 +29,7 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; public class ViewshedResolver extends AbstractContextualizer implements IResolver, IExpression { diff --git a/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java b/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java index dfb3debdf..5854cba46 100644 --- a/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java +++ b/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java @@ -46,10 +46,10 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnsupportedFeatureException; import org.integratedmodelling.klab.utils.NumberUtils; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.parameter.GeneralParameterValue; -import org.opengis.parameter.ParameterValueGroup; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValueGroup; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import com.beust.jcommander.internal.Lists; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java index 4298d7282..d3ef6117b 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java @@ -11,10 +11,10 @@ import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.utils.Pair; import org.locationtech.jts.geom.Geometry; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.operation.TransformException; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.operation.TransformException; -public class Envelope implements IEnvelope { +public class Bounds implements IEnvelope { /** * Default minimum resolution in meters when a ROI is created from a user interacting with a @@ -44,7 +44,7 @@ public boolean equals(Object obj) { return false; if (getClass() != obj.getClass()) return false; - Envelope other = (Envelope) obj; + Bounds other = (Bounds) obj; if (envelope == null) { if (other.envelope != null) return false; @@ -58,38 +58,38 @@ public boolean equals(Object obj) { return true; } - public static Envelope create(org.locationtech.jts.geom.Envelope envelope, Projection projection) { - Envelope ret = new Envelope(); + public static Bounds create(org.locationtech.jts.geom.Envelope envelope, Projection projection) { + Bounds ret = new Envelope(); ret.envelope = new ReferencedEnvelope(envelope, projection.getCoordinateReferenceSystem()); ret.projection = projection; return ret; } - public static Envelope create(org.opengis.geometry.Envelope envelope, Projection projection) { - Envelope ret = new Envelope(); + public static Bounds create(org.geotools.api.geometry.Bounds envelope, Projection projection) { + Bounds ret = new Envelope(); ret.envelope = new ReferencedEnvelope(envelope); ret.projection = projection; return ret; } - public static Envelope create(ReferencedEnvelope envelope) { - Envelope ret = new Envelope(); + public static Bounds create(ReferencedEnvelope envelope) { + Bounds ret = new Envelope(); ret.envelope = envelope; ret.projection = Projection.create(envelope.getCoordinateReferenceSystem()); return ret; } - public static Envelope create(double minX, double maxX, double minY, double maxY, IProjection projection) { + public static Bounds create(double minX, double maxX, double minY, double maxY, IProjection projection) { return create(new ReferencedEnvelope(minX, maxX, minY, maxY, ((Projection) projection).getCoordinateReferenceSystem())); } - public Envelope copy() { + public Bounds copy() { return create(new ReferencedEnvelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY(), envelope.getCoordinateReferenceSystem())); } - public static Envelope create(ReferencedEnvelope envelope, boolean swapXY) { - Envelope ret = new Envelope(); + public static Bounds create(ReferencedEnvelope envelope, boolean swapXY) { + Bounds ret = new Envelope(); ret.envelope = swapXY ? new ReferencedEnvelope(envelope.getMinY(), envelope.getMaxY(), envelope.getMinX(), envelope.getMaxX(), envelope.getCoordinateReferenceSystem()) @@ -165,7 +165,7 @@ public Geometry asJTSGeometry() { return Shape.makeCell(this.envelope.getMinX(), this.envelope.getMinY(), this.envelope.getMaxX(), this.envelope.getMaxY()); } - public static Envelope create(double minx, double maxx, double miny, double maxy, Projection crs) { + public static Bounds create(double minx, double maxx, double miny, double maxy, Projection crs) { return create(new ReferencedEnvelope(minx, maxx, miny, maxy, crs.getCoordinateReferenceSystem())); } @@ -180,13 +180,13 @@ public double getHeight() { } @Override - public Envelope transform(IProjection projection, boolean lenient) { + public Bounds transform(IProjection projection, boolean lenient) { if (projection.equals(this.projection)) { return this; } - Envelope ret = new Envelope(); + Bounds ret = new Envelope(); try { ret.envelope = this.envelope.transform(((Projection) projection).crs, lenient); } catch (TransformException | FactoryException e) { @@ -257,7 +257,7 @@ public Pair getResolutionForZoomLevel(int roundTo, double multi public int getScaleRank() { if (this.scaleRank == null) { - Envelope envelope = transform(Projection.getLatLon(), true); + Bounds envelope = transform(Projection.getLatLon(), true); int zoomLevel; double latDiff = envelope.getHeight(); @@ -283,7 +283,7 @@ public ReferencedEnvelope getJTSEnvelope() { } public boolean intersects(IEnvelope envelope) { - return this.envelope.intersects((org.locationtech.jts.geom.Envelope) ((Envelope) envelope).envelope); + return this.envelope.intersects((org.locationtech.jts.geom.Envelope) ((Bounds) envelope).envelope); } @Override @@ -326,7 +326,7 @@ public IEnvelope grow(double factor) { @Override public boolean overlaps(IEnvelope other) { try { - return this.envelope.intersects(((Envelope)other).getJTSEnvelope().toBounds(this.projection.getCRS())); + return this.envelope.intersects(((Bounds)other).getJTSEnvelope().toBounds(this.projection.getCRS())); } catch (TransformException e) { return false; } diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java index fd12f09de..f3d74ed0a 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java @@ -46,7 +46,7 @@ import org.integratedmodelling.klab.utils.MultidimensionalCursor; import org.integratedmodelling.klab.utils.Pair; import org.integratedmodelling.klab.utils.Range; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; /** * diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java index 8a3fa6f07..a52fbff50 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java @@ -3,7 +3,7 @@ import javax.measure.Unit; -import org.geotools.geometry.DirectPosition2D; +import org.geotools.geometry.Position2D; import org.geotools.referencing.CRS; import org.geotools.referencing.CRS.AxisOrder; import org.integratedmodelling.klab.api.observations.scale.space.IProjection; @@ -11,10 +11,10 @@ import org.integratedmodelling.klab.components.geospace.utils.WGS84; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabValidationException; -import org.opengis.geometry.DirectPosition; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.operation.MathTransform; +import org.geotools.api.geometry.Position; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.operation.MathTransform; public class Projection implements IProjection { @@ -235,7 +235,7 @@ public double[] transformCoordinate(double[] coordinate, IProjection other) { if (!this.equals(other)) { try { MathTransform transform = CRS.findMathTransform(((Projection) other).crs, this.crs); - DirectPosition position = transform.transform(new DirectPosition2D(ret[0], ret[1]), null); + Position position = transform.transform(new Position2D(ret[0], ret[1]), null); ret = new double[] { position.getCoordinate()[0], position.getCoordinate()[1] }; } catch (Exception e) { throw new KlabInternalErrorException(e); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java index c4fdca959..a6a2c854d 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java @@ -55,8 +55,8 @@ import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.type.Name; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.type.Name; /** * diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java index 76da1f3d9..d132815f1 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java @@ -41,12 +41,12 @@ import org.geotools.gce.geotiff.GeoTiffFormat; import org.geotools.gce.geotiff.GeoTiffWriteParams; import org.geotools.gce.geotiff.GeoTiffWriter; -import org.geotools.geometry.Envelope2D; +import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.metadata.iso.citation.Citations; import org.geotools.referencing.CRS; -import org.geotools.styling.ColorMapEntry; -import org.geotools.styling.RasterSymbolizer; +import org.geotools.api.style.ColorMapEntry; +import org.geotools.api.style.RasterSymbolizer; import org.geotools.swing.data.JFileDataStoreChooser; import org.geotools.util.factory.Hints; import org.hortonmachine.gears.utils.files.FileUtilities; @@ -74,13 +74,13 @@ import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.utils.Pair; import org.jaitools.tiledimage.DiskMemImage; -import org.opengis.filter.expression.Literal; -import org.opengis.metadata.Identifier; -import org.opengis.metadata.citation.Citation; -import org.opengis.parameter.GeneralParameterValue; -import org.opengis.parameter.ParameterValueGroup; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.filter.expression.Literal; +import org.geotools.api.metadata.Identifier; +import org.geotools.api.metadata.citation.Citation; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValueGroup; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; public enum GeotoolsUtils { @@ -113,7 +113,7 @@ public GridCoverage2D wrapStateInFloatCoverage(IState state, ILocator locator, F double east = grid.getEast(); double north = grid.getNorth(); CoordinateReferenceSystem crs = ((Projection) grid.getProjection()).getCoordinateReferenceSystem(); - Envelope2D writeEnvelope = new Envelope2D(crs, west, south, east - west, north - south); + ReferencedEnvelope writeEnvelope = ReferencedEnvelope.rect(west, south, east - west, north - south, crs); GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); GridCoverage2D coverage = factory.create("stateraster", ri, writeEnvelope); @@ -142,7 +142,7 @@ public GridCoverage2D getTemporaryFloatStorage(IScale scale) { double east = grid.getEast(); double north = grid.getNorth(); CoordinateReferenceSystem crs = ((Projection) grid.getProjection()).getCoordinateReferenceSystem(); - Envelope2D writeEnvelope = new Envelope2D(crs, west, south, east - west, north - south); + ReferencedEnvelope writeEnvelope = ReferencedEnvelope.rect(west, south, east - west, north - south, crs); GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); GridCoverage2D coverage = factory.create("stateraster", ri, writeEnvelope); @@ -173,7 +173,7 @@ public GridCoverage2D getTemporaryIntStorage(IScale scale) { double east = grid.getEast(); double north = grid.getNorth(); CoordinateReferenceSystem crs = ((Projection) grid.getProjection()).getCoordinateReferenceSystem(); - Envelope2D writeEnvelope = new Envelope2D(crs, west, south, east - west, north - south); + ReferencedEnvelope writeEnvelope = ReferencedEnvelope.rect(west, south, east - west, north - south, crs); GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); GridCoverage2D coverage = factory.create("stateraster", ri, writeEnvelope); @@ -754,7 +754,7 @@ public void createBigTiff(File file) { tileHeight = height; } - Envelope2D envelope = new Envelope2D(crs, 0, 0, width, height); + ReferencedEnvelope envelope = ReferencedEnvelope.rect(0, 0, width, height, crs); SampleModel sampleModel = new ComponentSampleModel(DataBuffer.TYPE_FLOAT, tileWidth, tileHeight, 1, tileWidth, new int[]{0}); DiskMemImage img = new DiskMemImage(width, height, sampleModel); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java index df6093bd0..d4cae6013 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java @@ -42,18 +42,18 @@ import org.geotools.map.Layer; import org.geotools.map.MapContent; import org.geotools.map.MapViewport; -import org.geotools.styling.FeatureTypeStyle; -import org.geotools.styling.Fill; -import org.geotools.styling.Graphic; -import org.geotools.styling.LineSymbolizer; -import org.geotools.styling.Mark; -import org.geotools.styling.PointSymbolizer; -import org.geotools.styling.PolygonSymbolizer; -import org.geotools.styling.Rule; +import org.geotools.api.style.FeatureTypeStyle; +import org.geotools.api.style.Fill; +import org.geotools.api.style.Graphic; +import org.geotools.api.style.LineSymbolizer; +import org.geotools.api.style.Mark; +import org.geotools.api.style.PointSymbolizer; +import org.geotools.api.style.PolygonSymbolizer; +import org.geotools.api.style.Rule; import org.geotools.styling.SLD; -import org.geotools.styling.Stroke; -import org.geotools.styling.Style; -import org.geotools.styling.StyleFactory; +import org.geotools.api.style.Stroke; +import org.geotools.api.style.Style; +import org.geotools.api.style.StyleFactory; import org.geotools.swing.JMapFrame; import org.integratedmodelling.klab.Concepts; import org.integratedmodelling.klab.api.data.ILocator; @@ -77,9 +77,9 @@ import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.filter.FilterFactory; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.filter.FilterFactory; /** * Generic spatial visualizer/debugger that is initialized with a spatial extent and you can just diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java index 60608cdda..ac3d3a233 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java @@ -25,12 +25,12 @@ import org.geotools.factory.CommonFactoryFinder; import org.geotools.renderer.lite.RendererUtilities; import org.geotools.renderer.lite.gridcoverage2d.GridCoverageRenderer; -import org.geotools.styling.ColorMap; -import org.geotools.styling.ContrastEnhancement; -import org.geotools.styling.RasterSymbolizer; -import org.geotools.styling.ShadedRelief; +import org.geotools.api.style.ColorMap; +import org.geotools.api.style.ContrastEnhancement; +import org.geotools.api.style.RasterSymbolizer; +import org.geotools.api.style.ShadedRelief; import org.geotools.styling.StyleBuilder; -import org.geotools.styling.StyleFactory; +import org.geotools.api.style.StyleFactory; import org.integratedmodelling.klab.Concepts; import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Observations; @@ -57,7 +57,7 @@ import org.integratedmodelling.klab.utils.Pair; import org.integratedmodelling.klab.utils.Range; import org.integratedmodelling.klab.utils.Triple; -import org.opengis.style.ContrastMethod; +import org.geotools.api.style.ContrastMethod; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java index 2c4b039b1..403fd5011 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java @@ -82,13 +82,13 @@ import org.geotools.coverage.grid.GridCoverageFactory; import org.geotools.coverage.grid.GridEnvelope2D; import org.geotools.feature.SchemaException; -import org.geotools.geometry.Envelope2D; +import org.geotools.geometry.jts.ReferencedEnvelope; //import org.geotools.graph.structure.Graph; //import org.geotools.graph.structure.Node; import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultEngineeringCRS; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; //import org.thema.common.Config; //import org.thema.common.JTS; //import org.thema.common.ProgressBar; @@ -297,7 +297,7 @@ public final class Project { // codes.remove((int)noData); // } // -// Envelope2D gZone = cov.getEnvelope2D(); +// ReferencedEnvelope gZone = cov.getEnvelope2D(); // zone = gZone.getBounds2D(); // CoordinateReferenceSystem crs = cov.getCoordinateReferenceSystem2D(); // if(crs instanceof DefaultEngineeringCRS) { @@ -320,7 +320,7 @@ public final class Project { // } // resolution = grid2space.getMatrixEntries()[0]; // -// Envelope2D extZone = new Envelope2D(crs, +// ReferencedEnvelope extZone = new ReferencedEnvelope(crs, // gZone.x-resolution, gZone.y-resolution, gZone.width+2*resolution, gZone.height+2*resolution); // // TreeMap envMap = new TreeMap<>(); @@ -1565,7 +1565,7 @@ public List getPatches() { // } // // GridCoverage2D gridCov = new GridCoverageFactory().create("rasterpatch", -// newRaster, new Envelope2D(getCRS(), zone)); +// newRaster, new ReferencedEnvelope(getCRS(), zone)); // IOImage.saveTiffCoverage(new File(dir, PATCH_RASTER), gridCov); // // TreeSet newCodes = new TreeSet<>(codes); @@ -1711,7 +1711,7 @@ public List getPatches() { // } // // GridCoverage2D gridCov = new GridCoverageFactory().create("rasterpatch", -// newRaster, new Envelope2D(getCRS(), zone)); +// newRaster, new ReferencedEnvelope(getCRS(), zone)); // IOImage.saveTiffCoverage(new File(dir, PATCH_RASTER), gridCov); // // TreeSet newCodes = new TreeSet<>(codes); @@ -1970,7 +1970,7 @@ public List getPatches() { // } // // GridEnvelope2D grid = cov.getGridGeometry().getGridRange2D(); -// Envelope2D env = cov.getEnvelope2D(); +// ReferencedEnvelope env = cov.getEnvelope2D(); // double res = env.getWidth() / grid.getWidth(); // if(res != resolution) { // throw new IllegalArgumentException(java.util.ResourceBundle.getBundle("org/thema/graphab/Bundle").getString("Resolution_does_not_match.")); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java index ab4c5d7e1..6017ac5ec 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java @@ -41,8 +41,8 @@ import org.integratedmodelling.klab.utils.CamelCase; import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; -import org.opengis.feature.Property; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.Property; +import org.geotools.api.feature.simple.SimpleFeature; public class OpenBuildingsInstantiator extends AbstractContextualizer implements IInstantiator, IExpression { private IDirectObservation contextSubject = null; diff --git a/pom.xml b/pom.xml index 1eccc584d..839f76d2b 100644 --- a/pom.xml +++ b/pom.xml @@ -111,8 +111,8 @@ 17 - 28.0 - 0.10.13 + 34.0 + 0.11.0-SNAPSHOT 1.20.0 1.5.0 17 diff --git a/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java b/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java index 832434e3a..6de5b1e69 100644 --- a/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java +++ b/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; -import org.geotools.data.DataStore; +import org.geotools.api.data.DataStore; import org.geotools.data.wfs.WFSDataStore; import org.geotools.data.wfs.WFSDataStoreFactory; import org.geotools.http.SimpleHttpClient; From 4e728d34977ad3f368a7b5891d7aa83d2faedf22 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Thu, 12 Feb 2026 10:46:42 +0100 Subject: [PATCH 06/74] Remove singleton annotation as component IS a singleton https://stackoverflow.com/questions/5907501/when-annotating-a-class-with-component-does-this-mean-it-is-a-spring-bean-and --- .../java/org/integratedmodelling/klab/node/NodeApplication.java | 2 -- .../klab/semantic/SemanticServerApplication.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/klab.node/src/main/java/org/integratedmodelling/klab/node/NodeApplication.java b/klab.node/src/main/java/org/integratedmodelling/klab/node/NodeApplication.java index 2790da8ba..aa2a4426b 100644 --- a/klab.node/src/main/java/org/integratedmodelling/klab/node/NodeApplication.java +++ b/klab.node/src/main/java/org/integratedmodelling/klab/node/NodeApplication.java @@ -3,7 +3,6 @@ import java.util.Arrays; import javax.annotation.PreDestroy; -import javax.inject.Singleton; import org.apache.catalina.startup.Tomcat; import org.springframework.beans.factory.annotation.Autowired; @@ -23,7 +22,6 @@ import org.springframework.web.client.RestTemplate; @Component -@Singleton @EnableAutoConfiguration @ComponentScan(basePackages = { "org.integratedmodelling.klab.node.security", "org.integratedmodelling.klab.node.resources", "org.integratedmodelling.klab.node.controllers", diff --git a/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java b/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java index 609c42f99..7994029ac 100644 --- a/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java +++ b/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java @@ -3,7 +3,6 @@ import java.util.Arrays; import javax.annotation.PreDestroy; -import javax.inject.Singleton; import org.apache.catalina.startup.Tomcat; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -18,7 +17,6 @@ import org.springframework.web.client.RestTemplate; @Component -@Singleton @EnableAutoConfiguration @ComponentScan(basePackages = { /* * "org.integratedmodelling.klab.node.security", From 577ea8971b79c5d99ba47e8fcad3093709663e88 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Thu, 12 Feb 2026 10:47:25 +0100 Subject: [PATCH 07/74] Revert automatic modification as Envelope is a klab custom class --- .../components/geospace/extents/Envelope.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java index d3ef6117b..4ae6a200a 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java @@ -14,7 +14,7 @@ import org.geotools.api.referencing.FactoryException; import org.geotools.api.referencing.operation.TransformException; -public class Bounds implements IEnvelope { +public class Envelope implements IEnvelope { /** * Default minimum resolution in meters when a ROI is created from a user interacting with a @@ -44,7 +44,7 @@ public boolean equals(Object obj) { return false; if (getClass() != obj.getClass()) return false; - Bounds other = (Bounds) obj; + Envelope other = (Envelope) obj; if (envelope == null) { if (other.envelope != null) return false; @@ -58,38 +58,38 @@ public boolean equals(Object obj) { return true; } - public static Bounds create(org.locationtech.jts.geom.Envelope envelope, Projection projection) { - Bounds ret = new Envelope(); + public static Envelope create(org.locationtech.jts.geom.Envelope envelope, Projection projection) { + Envelope ret = new Envelope(); ret.envelope = new ReferencedEnvelope(envelope, projection.getCoordinateReferenceSystem()); ret.projection = projection; return ret; } - public static Bounds create(org.geotools.api.geometry.Bounds envelope, Projection projection) { - Bounds ret = new Envelope(); + public static Envelope create(org.geotools.api.geometry.Bounds envelope, Projection projection) { + Envelope ret = new Envelope(); ret.envelope = new ReferencedEnvelope(envelope); ret.projection = projection; return ret; } - public static Bounds create(ReferencedEnvelope envelope) { - Bounds ret = new Envelope(); + public static Envelope create(ReferencedEnvelope envelope) { + Envelope ret = new Envelope(); ret.envelope = envelope; ret.projection = Projection.create(envelope.getCoordinateReferenceSystem()); return ret; } - public static Bounds create(double minX, double maxX, double minY, double maxY, IProjection projection) { + public static Envelope create(double minX, double maxX, double minY, double maxY, IProjection projection) { return create(new ReferencedEnvelope(minX, maxX, minY, maxY, ((Projection) projection).getCoordinateReferenceSystem())); } - public Bounds copy() { + public Envelope copy() { return create(new ReferencedEnvelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY(), envelope.getCoordinateReferenceSystem())); } - public static Bounds create(ReferencedEnvelope envelope, boolean swapXY) { - Bounds ret = new Envelope(); + public static Envelope create(ReferencedEnvelope envelope, boolean swapXY) { + Envelope ret = new Envelope(); ret.envelope = swapXY ? new ReferencedEnvelope(envelope.getMinY(), envelope.getMaxY(), envelope.getMinX(), envelope.getMaxX(), envelope.getCoordinateReferenceSystem()) @@ -165,7 +165,7 @@ public Geometry asJTSGeometry() { return Shape.makeCell(this.envelope.getMinX(), this.envelope.getMinY(), this.envelope.getMaxX(), this.envelope.getMaxY()); } - public static Bounds create(double minx, double maxx, double miny, double maxy, Projection crs) { + public static Envelope create(double minx, double maxx, double miny, double maxy, Projection crs) { return create(new ReferencedEnvelope(minx, maxx, miny, maxy, crs.getCoordinateReferenceSystem())); } @@ -180,13 +180,13 @@ public double getHeight() { } @Override - public Bounds transform(IProjection projection, boolean lenient) { + public Envelope transform(IProjection projection, boolean lenient) { if (projection.equals(this.projection)) { return this; } - Bounds ret = new Envelope(); + Envelope ret = new Envelope(); try { ret.envelope = this.envelope.transform(((Projection) projection).crs, lenient); } catch (TransformException | FactoryException e) { @@ -257,7 +257,7 @@ public Pair getResolutionForZoomLevel(int roundTo, double multi public int getScaleRank() { if (this.scaleRank == null) { - Bounds envelope = transform(Projection.getLatLon(), true); + Envelope envelope = transform(Projection.getLatLon(), true); int zoomLevel; double latDiff = envelope.getHeight(); @@ -283,7 +283,7 @@ public ReferencedEnvelope getJTSEnvelope() { } public boolean intersects(IEnvelope envelope) { - return this.envelope.intersects((org.locationtech.jts.geom.Envelope) ((Bounds) envelope).envelope); + return this.envelope.intersects((org.locationtech.jts.geom.Envelope) ((Envelope) envelope).envelope); } @Override @@ -326,7 +326,7 @@ public IEnvelope grow(double factor) { @Override public boolean overlaps(IEnvelope other) { try { - return this.envelope.intersects(((Bounds)other).getJTSEnvelope().toBounds(this.projection.getCRS())); + return this.envelope.intersects(((Envelope)other).getJTSEnvelope().toBounds(this.projection.getCRS())); } catch (TransformException e) { return false; } From af97085deff6a71c0bc42c1799c9888bc33d59ce Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Thu, 12 Feb 2026 10:53:38 +0100 Subject: [PATCH 08/74] More import arrangements --- .../datacube/ChunkedDatacubeRepository.java | 6 ++--- .../klab/ogc/integration/Geoserver.java | 6 ++--- .../klab/raster/files/RasterEncoder.java | 23 +++++++++---------- .../hydrology/MarineFloodResolver.java | 8 +------ .../hydrology/StreamOutletInstantiator.java | 5 ++-- .../cdm/utils/NetCDFUtils.java | 11 ++++----- .../raster/FloatRasterWrapper.java | 2 +- 7 files changed, 25 insertions(+), 36 deletions(-) diff --git a/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java b/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java index 835ede208..7e0a232ea 100644 --- a/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java +++ b/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java @@ -24,10 +24,10 @@ import java.util.logging.Level; import javax.annotation.Nullable; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; import org.apache.commons.io.FileUtils; +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; import org.geotools.coverage.grid.GridCoverage2D; import org.integratedmodelling.adapter.datacube.api.IDatacube; import org.integratedmodelling.klab.Configuration; @@ -40,8 +40,6 @@ import org.integratedmodelling.klab.api.data.IResource.Availability; import org.integratedmodelling.klab.api.data.adapters.IKlabData.Builder; import org.integratedmodelling.klab.api.data.mediation.IUnit; -import org.integratedmodelling.klab.api.observations.IState; -import org.integratedmodelling.klab.api.observations.scale.IExtent; import org.integratedmodelling.klab.api.observations.scale.IScale; import org.integratedmodelling.klab.api.observations.scale.space.IGrid; import org.integratedmodelling.klab.api.observations.scale.space.ISpace; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java index 4fed06170..d6f783b26 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/integration/Geoserver.java @@ -15,9 +15,10 @@ import java.util.function.Function; import javax.annotation.Nullable; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; +import org.geotools.api.coverage.grid.GridCoverage; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.gce.geotiff.GeoTiffReader; import org.integratedmodelling.klab.Configuration; @@ -39,7 +40,6 @@ import org.integratedmodelling.klab.utils.FileUtils; import org.integratedmodelling.klab.utils.MiscUtilities; import org.integratedmodelling.klab.utils.NumberUtils; -import org.geotools.api.coverage.grid.GridCoverage; import kong.unirest.GetRequest; import kong.unirest.HttpRequestWithBody; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java index a8489a7ad..144b9afc1 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/raster/files/RasterEncoder.java @@ -21,14 +21,16 @@ import java.util.Map; import java.util.Set; -import javax.media.jai.Interpolation; -import javax.media.jai.InterpolationBicubic; -import javax.media.jai.InterpolationBicubic2; -import javax.media.jai.InterpolationBilinear; -import javax.media.jai.InterpolationNearest; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; - +import org.eclipse.imagen.Interpolation; +import org.eclipse.imagen.InterpolationBicubic; +import org.eclipse.imagen.InterpolationBicubic2; +import org.eclipse.imagen.InterpolationBilinear; +import org.eclipse.imagen.InterpolationNearest; +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; +import org.eclipse.imagen.media.range.NoDataContainer; +import org.geotools.api.coverage.grid.GridCoverage; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.geotools.coverage.grid.GeneralGridEnvelope; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.GridGeometry2D; @@ -36,6 +38,7 @@ import org.geotools.coverage.grid.io.GridCoverage2DReader; import org.geotools.coverage.grid.io.GridFormatFinder; import org.geotools.coverage.processing.Operations; +import org.geotools.coverage.util.CoverageUtilities; import org.geotools.gce.geotiff.GeoTiffFormat; import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.referencing.CRS; @@ -67,10 +70,6 @@ import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.utils.MiscUtilities; import org.integratedmodelling.klab.utils.NumberUtils; -import org.geotools.api.coverage.grid.GridCoverage; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; -import org.geotools.coverage.util.CoverageUtilities; -import it.geosolutions.jaiext.range.NoDataContainer; import groovy.lang.Binding; import groovy.lang.GroovyShell; diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java index a99c1bb5c..591942016 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java @@ -3,11 +3,9 @@ import static org.hortonmachine.gears.libs.modules.HMConstants.floatNovalue; import java.awt.image.DataBuffer; - -import javax.media.jai.iterator.RandomIter; +import java.util.ArrayList; import org.geotools.coverage.grid.GridCoverage2D; -import org.geotools.coverage.util.CoverageUtilities; import org.hortonmachine.gears.libs.modules.HMRaster; import org.hortonmachine.gears.utils.RegionMap; import org.integratedmodelling.klab.api.data.general.IExpression; @@ -20,10 +18,6 @@ import org.integratedmodelling.klab.exceptions.KlabException; import org.integratedmodelling.klab.utils.Parameters; -import oms3.gen.doubleAccess; - -import java.util.ArrayList; - public class MarineFloodResolver extends AbstractContextualizer implements IResolver, IExpression { private Double decayFact; diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java index 4229b664b..7bfae9f4d 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java @@ -9,9 +9,8 @@ import java.util.Comparator; import java.util.List; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; - +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; import org.hortonmachine.gears.libs.modules.FlowNode; import org.hortonmachine.hmachine.modules.demmanipulation.markoutlets.OmsMarkoutlets; import org.integratedmodelling.geoprocessing.TaskMonitor; diff --git a/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java b/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java index 5854cba46..8b9f36231 100644 --- a/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java +++ b/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java @@ -25,8 +25,11 @@ import java.util.logging.Level; import java.util.logging.Logger; -import javax.media.jai.RasterFactory; - +import org.eclipse.imagen.RasterFactory; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValueGroup; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.GridCoverageFactory; import org.geotools.coverage.grid.io.AbstractGridFormat; @@ -46,10 +49,6 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnsupportedFeatureException; import org.integratedmodelling.klab.utils.NumberUtils; -import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.api.parameter.GeneralParameterValue; -import org.geotools.api.parameter.ParameterValueGroup; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import com.beust.jcommander.internal.Lists; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java index bad0f56e2..3ae4cc525 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java @@ -4,7 +4,7 @@ import java.awt.image.Raster; -import com.sun.media.jai.iterator.WrapperRI; +import org.eclipse.imagen.media.iterator.WrapperRI; /** * RenderedImage wrapper for {@link Raster}s. From da813008add466fa5029ce1a70ac618b0b0b0b47 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 13 Feb 2026 16:11:59 +0100 Subject: [PATCH 09/74] Solved but need to check why the FileUtils class is not the last version --- .../klab/data/storage/ResourceCatalog.java | 6 +++--- .../klab/node/resources/ResourceCatalog.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java b/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java index 6a5348401..ee585ce84 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java @@ -172,7 +172,7 @@ public IResource put(String key, IResource value) { try { File resFile = new File(resourcePath + File.separator + "resource.json"); if (!resFile.exists() || resFile.lastModified() < ref.getResourceTimestamp()) { - FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8.name()); } } catch (IOException e) { throw new KlabIOException(e); @@ -329,7 +329,7 @@ public IResource move(IResource resource, IProject destinationProject) { } } FileUtils.writeStringToFile(new File(newData.getFirst() + File.separator + "resource.json"), - JsonUtils.printAsJson(newData.getSecond()), StandardCharsets.UTF_8); + JsonUtils.printAsJson(newData.getSecond()), StandardCharsets.UTF_8.name()); FileUtils.deleteDirectory(previousDir); resources.remove(resource.getUrn()); resources.put(resource.getUrn(), newData.getSecond()); @@ -371,7 +371,7 @@ public IResource update(IResource resource, String message) { try { File resFile = new File(resourcePath + File.separator + "resource.json"); if (!resFile.exists() || resFile.lastModified() < ref.getResourceTimestamp()) { - FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8.name()); } } catch (IOException e) { throw new KlabIOException(e); diff --git a/klab.node/src/main/java/org/integratedmodelling/klab/node/resources/ResourceCatalog.java b/klab.node/src/main/java/org/integratedmodelling/klab/node/resources/ResourceCatalog.java index 61b1e9073..b9dda098e 100644 --- a/klab.node/src/main/java/org/integratedmodelling/klab/node/resources/ResourceCatalog.java +++ b/klab.node/src/main/java/org/integratedmodelling/klab/node/resources/ResourceCatalog.java @@ -140,7 +140,7 @@ public IResource importResource(ResourceReference reference, EngineAuthorization File metadata = new File(resourcePath + File.separator + "resource.json"); try { FileUtils.writeStringToFile(metadata, JsonUtils.printAsJson(resource.getReference()), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8.name()); // publish, validate and insert ret = publisher.publish(resource, this, Klab.INSTANCE.getRootMonitor()); @@ -303,7 +303,7 @@ public IResource importResource(File resourcePath, EngineAuthorization user) { try { // write out the modified resource, overwriting the original FileUtils.writeStringToFile(metadata, JsonUtils.printAsJson(((Resource) ret).getReference()), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8.name()); } catch (IOException e) { throw new KlabIOException(e); } From cead2326efc48e153b01485dffbf0c70a6a1780f Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 13 Feb 2026 16:51:16 +0100 Subject: [PATCH 10/74] Apply JAI to ImageM update ant file --- .../klab/ogc/RasterAdapter.java | 2 +- .../geospace/events/EventBuilder.java | 2 +- .../processing/NeighborhoodResolver.java | 2 +- .../geospace/utils/GeotoolsUtils.java | 24 +++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/RasterAdapter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/RasterAdapter.java index b44e748c1..f08d81a59 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/RasterAdapter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/RasterAdapter.java @@ -75,7 +75,7 @@ public class RasterAdapter implements IResourceAdapter { public static final String INTERPOLATION_TYPE_FIELD = "interpolation"; /** - * Possible values of interpolation type (JAI classes) + * Possible values of interpolation type (ImageN classes) */ public static final String[] INTERPOLATION_TYPE_VALUES = { "bilinear", "nearest", "bicubic", "bicubic2" }; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java index 1fd8b6d6e..6257a2d90 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java @@ -21,7 +21,7 @@ * find the earliest start among them. Then reconstruct based on that time instead of the time at the call. * * 2. For each layer: - * 2.1 use JAI vectorizer and tag each shape in a layer with a progressive ID and rasterize the ID over a new layer. + * 2.1 use ImageN vectorizer and tag each shape in a layer with a progressive ID and rasterize the ID over a new layer. * 2.2 if there is a previous layer, connect the shapes that overlap. Shapes that don't (or end of observations) causes * event storage. * diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java index 5e9d6884c..abe03db12 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java @@ -167,7 +167,7 @@ public IState resolve(IState target, IContextualizationScope context) throws Kla } /* - * build offset mask for quick addressing TODO can use kernels from JAI tools + * build offset mask for quick addressing TODO can use kernels from ImageN tools */ this.maskSize = hCells * 2 + 1; this.offsetMask = new Pair[maskSize][maskSize]; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java index d132815f1..9f006b412 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java @@ -19,15 +19,15 @@ import java.util.Set; import java.util.function.Function; -import javax.media.jai.ImageLayout; -import javax.media.jai.JAI; -import javax.media.jai.ParameterBlockJAI; -import javax.media.jai.RasterFactory; -import javax.media.jai.RenderedOp; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; -import javax.media.jai.iterator.RectIterFactory; -import javax.media.jai.iterator.WritableRectIter; +import org.eclipse.imagen.ImageLayout; +import org.eclipse.imagen.ImageN; +import org.eclipse.imagen.ParameterBlockImageN; +import org.eclipse.imagen.RasterFactory; +import org.eclipse.imagen.RenderedOp; +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; +import org.eclipse.imagen.iterator.RectIterFactory; +import org.eclipse.imagen.iterator.WritableRectIter; import org.eclipse.lsp4j.AbstractTextDocumentRegistrationAndWorkDoneProgressOptions; import org.geotools.coverage.Category; @@ -1020,7 +1020,7 @@ public static void main(String[] args) throws Exception { return; } - ParameterBlockJAI pb = new ParameterBlockJAI("Constant"); + ParameterBlockImageN pb = new ParameterBlockImageN("Constant"); pb.setParameter("width", (float) IMAGE_WIDTH); pb.setParameter("height", (float) IMAGE_HEIGHT); pb.setParameter("bandValues", new Double[]{0.0d}); @@ -1031,9 +1031,9 @@ public static void main(String[] args) throws Exception { layout.setTileWidth(tileWidth); layout.setTileHeight(tileWidth); - RenderingHints hints = new RenderingHints(JAI.KEY_IMAGE_LAYOUT, layout); + RenderingHints hints = new RenderingHints(ImageN.KEY_IMAGE_LAYOUT, layout); - RenderedOp image = JAI.create("Constant", pb, hints); + RenderedOp image = ImageN.create("Constant", pb, hints); GeoTiffWriter writer = new GeoTiffWriter(file, null); GridCoverageFactory factory = new GridCoverageFactory(); From 70d730ec258392acb62f6bb0b801eb8a702ac30d Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 13 Feb 2026 18:41:44 +0100 Subject: [PATCH 11/74] Add jaitools disappeared classes --- .../java/org/jaitools/CollectionFactory.java | 195 +++ .../org/jaitools/DaemonThreadFactory.java | 113 ++ .../jaitools/tilecache/BasicCacheVisitor.java | 205 +++ .../tilecache/DiskCacheFailedException.java | 52 + .../jaitools/tilecache/DiskCachedTile.java | 700 +++++++++ .../tilecache/DiskMemCacheException.java | 53 + .../jaitools/tilecache/DiskMemTileCache.java | 1277 +++++++++++++++++ .../tilecache/DiskMemTileCacheVisitor.java | 49 + .../tilecache/TileAccessTimeComparator.java | 52 + .../tilecache/TileNotResidentException.java | 45 + .../org/jaitools/tiledimage/DiskMemImage.java | 818 +++++++++++ .../tiledimage/DiskMemImageGraphics.java | 1027 +++++++++++++ .../PixelOutsideImageException.java | 49 + 13 files changed, 4635 insertions(+) create mode 100644 klab.engine/src/main/java/org/jaitools/CollectionFactory.java create mode 100644 klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java create mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java create mode 100644 klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java create mode 100644 klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java create mode 100644 klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java diff --git a/klab.engine/src/main/java/org/jaitools/CollectionFactory.java b/klab.engine/src/main/java/org/jaitools/CollectionFactory.java new file mode 100644 index 000000000..4ef9eaeb8 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/CollectionFactory.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.Stack; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Convenience methods to create generic collections + * following the DRY (Don't Repeat Yourself) principle. + * Examples of use: + * 

+ * // simple
+ * List<Integer> list1 = CollectionFactory.list();
+ * 
+ * // nested
+ * List<List<String>> list2 = CollectionFactory.list();
+ * 
+ * // a multi-map
+ * Map<String, List<Integer>> multiMap = CollectionFactory.map();
+ *
+ * + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class CollectionFactory { + /** + * Returns a new {@code Map} object. The returned + * object is a {@link HashMap}. + *

+ * Example of use: + * 


+     * Map<Foo, Bar> myMap = CollectionFactory.map();
+     *
+ * + * @param key type + * @param value type + * @return a new Map<K,V> instance + */ + public static Map map() { + return new HashMap(); + } + + /** + * Returns a new {@code Map} object that maintains insertion + * order. The returned object is a {@link LinkedHashMap}. + *

+ * Example of use: + * 


+     * Map<Foo, Bar> myMap = CollectionFactory.orderedMap();
+     *
+ * + * @param key type + * @param value type + * @return a new Map<K,V> instance + */ + public static Map orderedMap() { + return new LinkedHashMap(); + } + + /** + * Returns a new {@code SortedMap} object. Key type {@code K} must + * implement {@linkplain Comparable}. The returned object is a + * {@link TreeMap}. + *

+ * Example of use: + * 


+     * Map<Foo, Bar> myMap = CollectionFactory.sortedMap();
+     *
+ * + * @param key type + * @param value type + * @return a new SortedMap<K,V> instance + */ + public static SortedMap sortedMap() { + return new TreeMap(); + } + + /** + * Returns a new {@code List} object. The returned object is + * an {@link ArrayList}. + *

+ * Example of use: + * 


+     * List<Foo> myList = CollectionFactory.list();
+     *
+ * + * @param element type + * @return a new List<T> instance + */ + public static List list() { + return new ArrayList(); + } + + /** + * Returns a new {@code Stack} object. + *

+ * Example of use: + * 


+     * Stack<Foo> myStack = CollectionFactory.stack();
+     *
+ * + * @param element type + * @return a new Stack<T> instance + */ + public static Stack stack() { + return new Stack(); + } + + /** + * Returns a new {@code Set} instance. The returned object is + * a {@link HashSet}. + *

+ * Example of use: + * 


+     * Set<Foo> mySet = CollectionFactory.set();
+     *
+ * + * @param element type + * @return a new HashSet<T> instance + */ + public static Set set() { + return new HashSet(); + } + + /** + * Returns a new {@code Set} instance that maintains + * the insertion order of its elements. + *

+ * Example of use: + * 


+     * Set<Foo> mySet = CollectionFactory.set();
+     *
+ * + * @param element type + * @return a new LinkedHashSet<T> instance + */ + public static Set orderedSet() { + return new LinkedHashSet(); + } + + /** + * Returns a new {@code SortedSet} instance. The type {@code T} + * must implement {@linkplain Comparable}. The returned object + * is a {@link TreeSet}. + *

+ * Example of use: + * 


+     * Set<MyObject> foo = CollectionFactory.sortedSet();
+     *
+ * + * @param element type + * @return a new TreeSet<T> instance + */ + public static SortedSet sortedSet() { + return new TreeSet(); + } + + +} diff --git a/klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java b/klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java new file mode 100644 index 000000000..1b020aa26 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2011, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Simple ThreadFactory implementation to supply daemon threads with + * specified priority. Used by JAITools classes that run polling services + * on background threads to avoid blocking application exit. + * + * @author Michael Bedward + * @since 1.1 + * @version $Id$ + */ +public class DaemonThreadFactory implements ThreadFactory { + + private static final String DEFAULT_ROOT_NAME = "daemon-"; + + private final ReentrantLock lock = new ReentrantLock(); + private static final AtomicInteger threadCount = new AtomicInteger(0); + private final int priority; + private final String rootName; + + + /** + * Creates a new factory which will supply daemon threads having + * normal priority. + */ + public DaemonThreadFactory() { + this(Thread.NORM_PRIORITY, DEFAULT_ROOT_NAME); + } + + /** + * Creates a new factory which will supply daemon threads to run + * at the specified priority. + * + * @param priority thread priority + */ + public DaemonThreadFactory(int priority) { + this(priority, DEFAULT_ROOT_NAME); + } + + + /** + * Creates a new factory which will supply daemon threads to run + * at the specified priority. Threads will be named {@code rootName-n} + * where {@code n} is the count of threads produced by all instances + * of this class. + * + * @param priority thread priority + * @param rootName root name to label threads + */ + public DaemonThreadFactory(int priority, String rootName) { + String s = rootName == null ? "" : rootName.trim(); + if (s.length() == 0) { + this.rootName = DEFAULT_ROOT_NAME; + } else if (s.endsWith("-")) { + this.rootName = s; + } else { + this.rootName = s + "-"; + } + + this.priority = Math.min(Thread.MAX_PRIORITY, Math.max(Thread.MIN_PRIORITY, priority)); + } + + /** + * Creates a new daemon thread with name and priority assigned + * as per the values supplied when creating this thread factory. + * + * @param r target for the new thread + * + * @return new thread + */ + public Thread newThread(Runnable r) { + lock.lock(); + try { + Thread t = new Thread(r); + t.setDaemon(true); + t.setName("daemon-" + threadCount.getAndIncrement()); + t.setPriority(priority); + return t; + } finally { + lock.unlock(); + } + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java b/klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java new file mode 100644 index 000000000..c34838840 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.awt.image.RenderedImage; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * A basic visitor class for DiskMemTileCache that can be used + * to examine cache contents. + *

+ * Example of use: + * 


+ * // Query the cache for tiles that belong to a given image
+ * // and that are currently resident in memory
+ * DiskMemTileCache cache = ...
+ * RenderedImage owner = ...
+ *
+ * Map filters = CollectionFactory.newMap();
+ * filters.put(BasicCacheVisitor.Key.OWNER, owner);
+ * filters.put(BasicCachevisitor.Key.RESIDENT, Boolean.TRUE);
+ *
+ * BasicCacheVisitor visitor = new BasicCacheVisitor();
+ * visitor.setFilters(filters);
+ * cache.accept(visitor);
+ *
+ * for (DiskCachedTile tile : visitor.getTiles()) {
+ *     System.out.println(String.format("tile %d,%d", tile.getTileX(), tile.getTileY()));
+ * }
+ *
+ * + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class BasicCacheVisitor implements DiskMemTileCacheVisitor { + + /** + * Defines keys to filter the tles visited + */ + public enum Key { + /** + * Filter on the image that owns the tile. The value + * provided for this key must be a RenderedImage. + */ + OWNER(RenderedImage.class), + + /** + * Filter on whether the tile is currently resident + * in memory. The value provided for this key must + * be of type Boolean. + */ + RESIDENT(Boolean.class); + + + private Class clazz; + + private Key(Class clazz) { + this.clazz = clazz; + } + + /** + * Gets the value class. + * + * @return the class + */ + public Class getValueClass() { + return clazz; + } + } + + private List tiles = new ArrayList(); + private Map filters = new HashMap(); + + + /** + * Set filter conditions. Any existing conditions are cleared. + * + * @param params a Map where each key is one of the constants + * defined by BasicCacheVisitor.Key and each value is the + * corresponding value to match tiles against + */ + public void setFilters(Map params) { + filters.clear(); + for (Entry e : params.entrySet()) { + addFilter(e.getKey(), e.getValue()); + } + } + + /** + * Set a single filter condition. Any existing conditions + * are cleared + * + * @param key one of the constants defined by BasicCacheVisitor.Key + * @param value corresponding value to match tiles against + */ + public void setFilter(Key key, Object value) { + filters.clear(); + addFilter(key, value); + } + + /** + * Clear all existing filter conditions + */ + public void clearFilters() { + filters.clear(); + } + + /** + * Returns an unmodifiable collection of tiles retrieved from + * the cache. + * + * @return tiles retrieved from the cache + */ + public Collection getTiles() { + return Collections.unmodifiableCollection(tiles); + } + + + /** + * This method is called by the cache for each tile in turn. Tiles that + * pass this visitor's filter conditions, if any, will be added to the + * visitor's tile collection. + * + * @param tile the cached tile being visited + * @param isResident set by the cache to indicate whether the tile is + * currently resident in memory + */ + public void visit(DiskCachedTile tile, boolean isResident) { + if (filters.isEmpty()) { + tiles.add(tile); + + } else { + boolean pass = true; + if (filters.containsKey(Key.OWNER)) { + if ( !(tile.getOwner().equals(filters.get(Key.OWNER))) ) { + pass = false; + } + } + + if (pass && filters.containsKey(Key.RESIDENT)) { + if ( ((Boolean)filters.get(Key.RESIDENT)).booleanValue() != isResident) { + pass = false; + } + } + + if (pass) { + tiles.add(tile); + } + } + } + + /** + * Clear collected tile data. Useful if an instance of this + * class makes multiple visits to a cache. + */ + public void clear() { + tiles.clear(); + } + + /** + * Helper method for setFilters and setFilter + * @param key + * @param value + */ + private void addFilter(Key key, Object value) { + if (key.getValueClass().isAssignableFrom(value.getClass())) { + filters.put(key, value); + } else { + throw new IllegalArgumentException( + String.format("Object of type %s cannot be used as value for %s", + value.getClass().getName(), key.toString())); + } + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java new file mode 100644 index 000000000..8ed3ec1df --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.awt.image.RenderedImage; + +/** + * Exception thrown when by (@code DiskMemTileCache} when a an attempt + * to cache a tile to disk has failed. + * + * @see DiskMemTileCache + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class DiskCacheFailedException extends DiskMemCacheException { + + /** + * Creates a new instance. + * + * @param owner the owning image + * @param tileX tile X ordinate + * @param tileY tile Y ordinate + */ + public DiskCacheFailedException(RenderedImage owner, int tileX, int tileY) { + super(owner, tileX, tileY, "could not be written to disk"); + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java new file mode 100644 index 000000000..201c3876f --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java @@ -0,0 +1,700 @@ +/* + * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.awt.Point; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferByte; +import java.awt.image.DataBufferDouble; +import java.awt.image.DataBufferFloat; +import java.awt.image.DataBufferInt; +import java.awt.image.DataBufferShort; +import java.awt.image.DataBufferUShort; +import java.awt.image.Raster; +import java.awt.image.RenderedImage; +import java.awt.image.WritableRaster; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.imageio.ImageIO; +import javax.imageio.stream.ImageInputStream; +import javax.imageio.stream.ImageOutputStream; +import org.eclipse.imagen.CachedTile; + +/** + * A managed tile class for {@code DiskMemTileCache}. Represents an image tile + * that can be cached on disk and/or in memory. + * + * @see DiskMemTileCache + * + * @author Michael Bedward + * @author Simone Giannecchini, GeoSolutions SAS + * @since 1.0 + * @version $Id$ + */ +public final class DiskCachedTile implements CachedTile { + + private static final Logger LOGGER = Logger.getLogger("org.jaitools.tilecache"); + + /** + * Constants identifying tile actions. + */ + public enum TileAction { + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile has been added to the cache + */ + ACTION_ADDED(0, "added to cache"), + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile has been added to the cache and immediately loaded + * into memory + */ + ACTION_ADDED_RESIDENT(1, "added to cache and placed into memory"), + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile becomes resident in memory + */ + ACTION_RESIDENT(2, "placed into memory"), + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile is removed from memory + */ + ACTION_NON_RESIDENT(3, "removed from memory"), + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile is removed from the cache entirely + */ + ACTION_REMOVED(4, "removed from the cache"), + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile is accessed via the cache + */ + ACTION_ACCESSED(5, "accessed"), + + /** + * Value that will be returned by {@linkplain #getAction()} when + * the tile's raster has been garbage collected + */ + ACTION_GARBAGE_COLLECTED(6, "garbage collected"); + + /** + * Map for the reverse lookup facility + */ + private static final Map lookup; + static { + lookup = new HashMap(); + for (TileAction t : EnumSet.allOf(TileAction.class)) { + lookup.put(t.getAction(), t); + } + } + /** An int associated to this action.*/ + private final int action; + /** Description of the action. */ + private final String desc; + + /** + * Private constructor to have maximum control over the values we use for this action. + * @param action an int associated to this action for interoperability with {@link CachedTile} interface. + */ + private TileAction(final int action, final String desc) { + this.action = action; + this.desc = desc; + } + + /** + * Retrieves an int associated to this action for interoperability with {@link CachedTile} interface. + * @return an int associated to this action for interoperability with {@link CachedTile} interface. + */ + public int getAction() { + return action; + } + + /** + * Retrieves a description of this action + * @return a description of this action + */ + public String getDescription() { + return desc; + } + + /** + * Reverse lookup. + * @param actionValue integer value of the action + * @return the corresponding TileAction + */ + public static TileAction get(int actionValue) { + return lookup.get(actionValue); + } + + /** + * The default action. + * @return the default action. + */ + public static TileAction getDefault() { + return ACTION_ACCESSED; + } + } + + /** + * The prefix used for temporary cache files + * data + */ + public static final String FILE_PREFIX = "tile"; + + /** + * The suffix used for temporary cache files + */ + public static final String FILE_SUFFIX = ".tmp"; + + /** + * The folder used to store tiles that are being newly cached to disk. + * If {@code null} (the default) the system's default folder is used. + * It is safe (though not necessarily sensible) to change this property + * after some tiles have already been cached to disk. + */ + private static File cacheFolder = null; + private static final Object folderLock = new Object(); + + private final Object id; + private final WeakReference ownerRef; + private final int tileX; + private final int tileY; + private final Object tileCacheMetric; + private long timeStamp; + private final int numBanks; + private final int dataLen; + private final long memorySize; + private File file; + private final Point location; + private final boolean isWritable; + + private TileAction action =TileAction.getDefault(); + + /** + * Get the current cache folder. This is the folder in which + * newly created temporary files holding tile data will be created. + * + * @return a new {@code File} object for the current cache folder + * or {@code null} if the default system folder is being + * used (ie. the folder corresponding to the {@code java.io.file} + * System property + */ + public static File getCacheFolder() { + File file = null; + if (cacheFolder != null) { + file = new File(cacheFolder.toURI()); + } + + return file; + } + + /** + * Set the current cache folder. This is the folder in which + * newly created temporary files holding tile data will be created. + *

+ * It is safe, though not necessarily sensible, to change the + * cache folder while one or more caches are running. + * + * @param folder the new cache folder or {@code null} to use + * the System's default temporary file folder + */ + public static void setCacheFolder(File folder) { + synchronized(folderLock) { + if (folder == null) { + cacheFolder = null; + } else { + cacheFolder = new File(folder.toURI()); + } + } + } + + /** + * Package-private constructor + * + * @param id the tile's unique identifier, allocated by the cache + * @param owner the image to which this tile belongs + * @param tileX the column index for this tile in the image's tile grid + * @param tileY the row index for this tile in the image's tile grid + * @param raster the image data for this tile + * @param writeToFile if true, the tile's data will be cached to disk + * immediately; if false, disk caching is deferred + * @param tileCacheMetric optional tile cache metric for use in scheduling + * (may be {@code null}) + * + * @throws IOException if an attempt to write the tile to disk fails + */ + DiskCachedTile(Object id, + RenderedImage owner, + int tileX, + int tileY, + Raster raster, + boolean writeToFile, + Object tileCacheMetric) throws IOException { + + if (owner == null || raster == null) { + throw new IllegalArgumentException( + "All of owner, tile and file args must be non-null"); + } + + this.id = id; + this.ownerRef = new WeakReference(owner); + this.tileX = tileX; + this.tileY = tileY; + this.tileCacheMetric = tileCacheMetric; + this.location = raster.getBounds().getLocation(); + this.isWritable = (raster instanceof WritableRaster); + + DataBuffer db = raster.getDataBuffer(); + numBanks = db.getNumBanks(); + dataLen = db.getSize(); + memorySize = DataBuffer.getDataTypeSize(db.getDataType()) / 8L * dataLen * numBanks; + + if (writeToFile ) { + writeData(raster); + } + + setTileTimeStamp(System.currentTimeMillis()); + } + + /** + * Returns a string representation of this cached tile + */ + @Override + public String toString() { + RenderedImage o = getOwner(); + String ostring = o == null ? "null" : o.toString(); + + Raster t = getTile(); + String tstring = t == null ? "null" : t.toString(); + + return getClass().getName() + "@" + Integer.toHexString(hashCode()) + + ": owner = " + ostring + + " tileX = " + Integer.toString(tileX) + + " tileY = " + Integer.toString(tileY) + + " tile = " + tstring + + " id = " + ((id instanceof Long)? Long.toHexString(((Long)id).longValue()) : id.toString()) + + " memorySize = " + Long.toString(memorySize) + + " timeStamp = " + Long.toString(timeStamp) + + " file = " + file.getPath(); + } + + /** + * Do not use this method. + *

+ * It is implemented to satisfy the CAchedTile interface but calling it will + * provoke an UnsupportedOperationException. + *

+ * To get a tile, use {@linkplain DiskBasedTileCache#getTile(java.awt.image.RenderedImage, int, int) } + */ + public Raster getTile() { + throw new UnsupportedOperationException("Can't get a tile directly from a DiskCachedTile object"); + } + + /** + * Gets the image that owns this tile. + * + * @return the owning image + */ + public RenderedImage getOwner() { + return ownerRef.get(); + } + + /** + * Gets the last time of access for this tile. + * + * @return the time stamp + */ + public long getTileTimeStamp() { + return timeStamp; + } + + /** + * Gets the tile cache metric for this tile. + * + * @return the metric + */ + public Object getTileCacheMetric() { + return tileCacheMetric; + } + + /** + * Gets the tile size in bytes. This is actually the + * size of the raster associated with this cached tile instance. + * + * @return tile size in bytes + */ + public long getTileSize() { + return memorySize; + } + + /** + * Gets the most recent action for this tile. More information can be + * retrieved by getting the corresponding {@linkplain DiskCachedTile.TileAction} + * constant as shown here: + * 


+     * int code = tile.getAction();
+     * TileAction action = TileAction.get(code);
+     * System.out.println("tile action: " + action.getDescription());
+     *
+ * + * @return the most recent action + */ + public int getAction() { + return action.ordinal(); + } + + /** + * Queries if this tile has been cached to disk. This method is + * a short-cut for: {@code getFile() != null}. + * + * @return {@code true} if the tile is cached on disk; {@code false} otherwise + */ + public boolean cachedToDisk() { + return file != null; + } + + /** + * Gets this tile's disk cache file. Returns {@code null} if the tile has not + * been cached to disk. + * + * @return the disk cache file for this tile or {@code null} + */ + public File getFile() { + return file; + } + + /** + * Deletes this tile's disk cache file. If the file could not be deleted + * a warning is logged. + */ + public void deleteDiskCopy() { + if (file != null) { + if (!file.delete()) { + LOGGER.log(Level.WARNING, + "Unable to delete cached image tile file: {0}", file.getPath()); + } + } + } + + /** + * Gets the unique ID for this tile. The ID is a combination of + * either the JAI-generated unique ID of the owning image or the image's + * hash key if a unique ID was not available plus the tile index. + * The returned object will be either Long or BigInteger. + * + * @return tile ID + */ + public Object getTileId() { + return id; + } + + /** + * Gets this tile's location in its parent image in pixel coordinates + * + * @return tile origin expressed in parent image coordinates + */ + public Point getLocation() { + return new Point(location); + } + + /** + * Gets the X ordinate of this tile in the parent image tile grid. + * + * @return tile grid X ordinate + */ + public int getTileX() { + return tileX; + } + + /** + * Gets the Y ordinate of this tile in the parent image tile grid. + * + * @return tile grid Y ordinate + */ + public int getTileY() { + return tileY; + } + + /** + * Queries if this tile is writable. + * + * @return {@code true} if writable; {@code false} otherwise + */ + public boolean isWritable() { + return isWritable; + } + + /** + * Package-private method called by the controlling {@linkplain DiskBasedTileCache} + * object when the tile is added to, or removed from, the cache. + */ + void setAction( TileAction action ) { + this.action = action; + } + + /** + * Package-private method called by the controlling {@linkplain DiskBasedTileCache} + * object when the tile is accessed + */ + void setTileTimeStamp(long time) { + this.timeStamp = time; + } + + /** + * Package-private method that reads data for the raster associated with this tile + * from disk + * + * @return a new instance of Raster or WritableRaster + */ + Raster readData() { + ImageInputStream strm = null; + DataBuffer dataBuf = null; + RenderedImage img = ownerRef.get(); + Raster raster = null; + + if (file != null && img != null) { + try { + strm = ImageIO.createImageInputStream(file); + + switch (img.getSampleModel().getDataType()) { + case DataBuffer.TYPE_BYTE: { + byte[][] bankData = new byte[numBanks][dataLen]; + for (int i = 0; i < numBanks; i++) { + int numRead = strm.read(bankData[i], 0, dataLen); + if (numRead < numBanks) { + throw new RuntimeException( + "Cached tile file appears to be truncated"); + } + } + dataBuf = new DataBufferByte(bankData, dataLen); + } + break; + + case DataBuffer.TYPE_DOUBLE: { + double[][] bankData = new double[numBanks][dataLen]; + for (int i = 0; i < numBanks; i++) { + strm.readFully(bankData[i], 0, dataLen); + } + dataBuf = new DataBufferDouble(bankData, dataLen); + } + break; + + case DataBuffer.TYPE_FLOAT: { + float[][] bankData = new float[numBanks][dataLen]; + for (int i = 0; i < numBanks; i++) { + strm.readFully(bankData[i], 0, dataLen); + } + dataBuf = new DataBufferFloat(bankData, dataLen); + } + break; + + case DataBuffer.TYPE_INT: { + int[][] bankData = new int[numBanks][dataLen]; + for (int i = 0; i < numBanks; i++) { + strm.readFully(bankData[i], 0, dataLen); + } + dataBuf = new DataBufferInt(bankData, dataLen); + } + break; + + case DataBuffer.TYPE_SHORT: { + short[][] bankData = new short[numBanks][dataLen]; + for (int i = 0; i < numBanks; i++) { + strm.readFully(bankData[i], 0, dataLen); + } + dataBuf = new DataBufferShort(bankData, dataLen); + } + break; + + case DataBuffer.TYPE_USHORT: { + short[][] bankData = new short[numBanks][dataLen]; + for (int i = 0; i < numBanks; i++) { + strm.readFully(bankData[i], 0, dataLen); + } + dataBuf = new DataBufferUShort(bankData, dataLen); + } + break; + + default: + throw new UnsupportedOperationException("Unsupported image data type"); + } + + } catch (FileNotFoundException ex) { + LOGGER.log(Level.SEVERE, "Failed to read image tile data", ex); + return null; + + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Failed to read image tile data", ex); + return null; + } + finally{ + if(strm!=null) + try{ + strm.close(); + }catch (Throwable e) { + // chew me + } + } + } + + if (dataBuf != null) { + if (isWritable) { + raster = Raster.createWritableRaster(img.getSampleModel(), dataBuf, location); + } else { + raster = Raster.createRaster(img.getSampleModel(), dataBuf, location); + } + } + + return raster; + } + + + /** + * Write data for the raster associated with this tile to + * disk. This may be called by DiskMemTileCache + * as well as be the tile itself. + */ + void writeData(Raster raster) throws IOException { + ImageOutputStream strm = null; + DataBuffer dataBuf = raster.getDataBuffer(); + + if (file == null) { + // first time this tile has been written to disk + file = createFile(); + } + + try { + strm = ImageIO.createImageOutputStream(file); + + switch (dataBuf.getDataType()) { + case DataBuffer.TYPE_BYTE: + { + byte[] bankData; + for (int i = 0; i < numBanks; i++) { + bankData = ((DataBufferByte) dataBuf).getData(i); + strm.write(bankData); + } + } + break; + + case DataBuffer.TYPE_DOUBLE: + { + double[] bankData; + for (int i = 0; i < numBanks; i++) { + bankData = ((DataBufferDouble) dataBuf).getData(i); + strm.writeDoubles(bankData, 0, dataLen); + } + } + break; + + case DataBuffer.TYPE_FLOAT: + { + float[] bankData; + for (int i = 0; i < numBanks; i++) { + bankData = ((DataBufferFloat) dataBuf).getData(i); + strm.writeFloats(bankData, 0, dataLen); + } + } + break; + + case DataBuffer.TYPE_INT: + { + int[] bankData; + for (int i = 0; i < numBanks; i++) { + bankData = ((DataBufferInt) dataBuf).getData(i); + strm.writeInts(bankData, 0, dataLen); + } + } + break; + + case DataBuffer.TYPE_SHORT: + { + short[] bankData; + for (int i = 0; i < numBanks; i++) { + bankData = ((DataBufferShort) dataBuf).getData(i); + strm.writeShorts(bankData, 0, dataLen); + } + } + break; + + case DataBuffer.TYPE_USHORT: + { + short[] bankData; + for (int i = 0; i < numBanks; i++) { + bankData = ((DataBufferUShort) dataBuf).getData(i); + strm.writeShorts(bankData, 0, dataLen); + } + } + break; + + default: + throw new UnsupportedOperationException("Unsupported image data type"); + } + + + } catch (FileNotFoundException ex) { + LOGGER.log(Level.SEVERE, "Failed to write image tile data", ex); + + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Failed to write image tile data", ex); + } + finally{ + if(strm!=null) + try{ + strm.close(); + }catch (Throwable e) { + // chew me + } + } + } + + /** + * Create a file to cache the given tile on disk. Presently + * this method does nothing more than delegate to File.createTempFile + * + * @throws java.io.IOException + */ + private File createFile() throws IOException { + synchronized(folderLock) { + return File.createTempFile(FILE_PREFIX, FILE_SUFFIX, cacheFolder); + } + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java new file mode 100644 index 000000000..a0867f145 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.awt.image.RenderedImage; + +/** + * Base exception class for {@code DiskMemTileCache} errors. + * + * @see DiskMemTileCache + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class DiskMemCacheException extends Exception { + + /** + * Creates a new instance. + * + * @param owner the owning image + * @param tileX tile X ordinate + * @param tileY tile Y ordinate + * @param msg error message + */ + public DiskMemCacheException(RenderedImage owner, int tileX, int tileY, String msg) { + super(String.format("tile at %d,%d of image %s %s", + tileX, tileY, owner.toString(), msg)); + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java new file mode 100644 index 000000000..898fd8488 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java @@ -0,0 +1,1277 @@ +/* + * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.awt.Point; +import java.awt.image.Raster; +import java.awt.image.RenderedImage; +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Observable; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.eclipse.imagen.CachedTile; +import org.eclipse.imagen.PlanarImage; +import org.eclipse.imagen.TileCache; + +import org.jaitools.CollectionFactory; +import org.jaitools.DaemonThreadFactory; + + +/** + * This class implements ImageN {@linkplain org.eclipse.imagen.TileCache}. It can store + * cached tiles on disk to allow applications to work with very large volumes of + * tiled image data without being limited by available memory. A subset of tiles + * (by default, the most recently accessed) are cached in memory to reduce access + * time. + *

+ * + * The default behaviour is to cache newly added tiles into memory. If the cache + * needs to free memory to accommodate a tile, it does so by removing lowest priority + * tiles from memory and caching them to disk. Optionally, the user can specify + * that newly added tiles are cached to disk immediately. + *

+ * + * Unlike the standard ImageN {@code TileCache} implementation, resident tiles are cached + * using strong references. This is to support the use of this class with + * {@linkplain org.jaitools.tiledimage.DiskMemImage} as well as operations that need to + * cache tiles that are expensive to create (e.g. output of a time-consuming analysis). + * A disadvantage of this design is that when the cache is being used for easily + * generated tiles it can end up unnecessarily holding memory that is more urgently + * required by other parts of an application. To avoid this happening, the cache can + * be set to auto-flush resident tiles at regular intervals. + *

+ * + *

Implementation note

+ * Tile polling and auto-flushing of memory resident tiles (if enabled) both run + * on low-priority background threads. These are marked as daemon threads to + * avoid these services blocking application shutdown. + * + * @author Michael Bedward + * @author Simone Giannecchini, GeoSolutions SAS + * + * @since 1.0 + * @version $Id$ + * + * @see DiskCachedTile + * @see TileAccessTimeComparator + */ +public class DiskMemTileCache extends Observable implements TileCache { + + private static final Logger LOGGER = Logger.getLogger("org.jaitools.tilecache"); + + /** + * The default memory capacity in bytes (64 * 2^20 = 64Mb) + * @see #setMemoryCapacity(long) + */ + public static final long DEFAULT_MEMORY_CAPACITY = 64L * 1024L * 1024L; + + /** + * The default memory threshold value (0.75) + * + * @see #setMemoryThreshold(float) + */ + public static final float DEFAULT_MEMORY_THRESHOLD = 0.75F; + + /** + * The default minimum period (2.5 seconds) of cache inactivity that + * must elapse before memory-resident tiles are automatically flushed. + * + * @see #setAutoFlushMemoryInterval(long) + */ + public static final long DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL = 2500; + + /** + * The default interval (2 seconds) for polling each tile to check if + * its owning image has been garbage collected. + * + * @see #setTilePollingInterval(long) + */ + public static final long DEFAULT_TILE_POLLING_INTERVAL = 2000L; + + + // @todo use ImageN ParameterList or some other ready-made class for this ? + private static class ParamDesc { + String key; + Class clazz; + Object defaultValue; + + ParamDesc(String key, Class clazz, Object defaultValue) { + this.key = key; this.clazz = clazz; this.defaultValue = defaultValue; + } + + boolean typeOK (Object value) { + if (Number.class.isAssignableFrom(clazz)) { + return Number.class.isAssignableFrom(value.getClass()); + + } else { + return clazz.isAssignableFrom(value.getClass()); + } + } + } + + /** + * Key for the parameter controlling initial memory capacity of the + * tile cache. This determines the maximum number of tiles that can + * be resident concurrently. The value must be numeric and will be + * treated as Long. + * @see #setMemoryCapacity(long) + * @see #DEFAULT_MEMORY_CAPACITY + */ + public static final String KEY_INITIAL_MEMORY_CAPACITY = "memcapacity"; + + /** + * Key for the parameter controlling whether newly added tiles + * are immediately cached to disk as well as in memory. The value + * must be one Boolean. If the value is {@code Boolean.FALSE} (the default), + * disk caching of tiles is deferred until required (ie. when + * memory needs to be freed for other tiles). + */ + public static final String KEY_ALWAYS_DISK_CACHE = "diskcache"; + + /** + * Key for the parameter controlling whether the cache will auto-flush + * memory-resident tiles. The value must be Boolean. If the value is + * {@code Boolean.TRUE}, auto-flushing of resident tiles will be enabled + * when the cache is created. The default is {@code Boolean.FALSE}. + * @see #setAutoFlushMemoryEnabled(boolean) + */ + public static final String KEY_AUTO_FLUSH_MEMORY_ENABLED = "enableautoflush"; + + /** + * Key for the cache auto-flush interval parameter. The value must be numeric + * and represents the interval, in milliseconds, between auto-flushes of + * resident tiles. Values less than or equal to zero are ignored. + * @see #setAutoFlushMemoryInterval(long) + * @see #DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL + */ + public static final String KEY_AUTO_FLUSH_MEMORY_INTERVAL = "autoflushinterval"; + + private static final Map paramDescriptors; + static { + ParamDesc desc; + paramDescriptors = new HashMap(); + + desc = new ParamDesc(KEY_INITIAL_MEMORY_CAPACITY, Number.class, DEFAULT_MEMORY_CAPACITY); + paramDescriptors.put( desc.key, desc ); + + desc = new ParamDesc(KEY_ALWAYS_DISK_CACHE, Boolean.class, Boolean.FALSE); + paramDescriptors.put( desc.key, desc ); + + desc = new ParamDesc(KEY_AUTO_FLUSH_MEMORY_ENABLED, Boolean.class, Boolean.FALSE); + paramDescriptors.put( desc.key, desc ); + + desc = new ParamDesc(KEY_AUTO_FLUSH_MEMORY_INTERVAL, Number.class, DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL); + paramDescriptors.put( desc.key, desc ); + } + + // maximum memory available for resident tiles + private long memCapacity; + + // current memory used for resident tiles + private long curMemory; + + /* + * A value between 0.0 and 1.0 that may be used for memory control + * if the param KEY_USE_MEMORY_THRESHOLD is TRUE. + */ + private float memThreshold; + + private boolean writeNewTilesToDisk; + + /** + * Map of all cached tiles. + */ + protected Map tiles; + + /** + * Memory-resident tiles. + */ + protected Map residentTiles; + + /** + * A tile comparator used to determine the priority of tiles for + * storage in memory. + */ + private Comparator comparator; + + /* List of tile references that is sorted into tile priority order when + * required for memory swapping. + *

+ * Implementation note: we use this in preference to a SortedSet or similar + * because of the complications of using the remove(obj) method with a sorted + * collection, where the comparator is used rather than the equals method. + */ + + /** + * Tiles sorted according to the current tile priority comparator. + */ + protected List sortedResidentTiles; + + // whether to send cache diagnostics to observers + private boolean diagnosticsEnabled; + + // Lock for tile access + private final ReentrantLock tileLock = new ReentrantLock(); + + // Variables used for auto-flushing of resident tiles + private ScheduledExecutorService flushService; + private ScheduledFuture flushFuture; + private long autoFlushInterval = DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL; + private AtomicBoolean okToFlush = new AtomicBoolean(false); + + // Variables used for polling each tile to check if its owning image has been + // garbage collected + private final ScheduledExecutorService tilePollingService; + private ScheduledFuture tilePollingFuture; + private long tilePollingInterval = DEFAULT_TILE_POLLING_INTERVAL; + + + /** + * Creates a new cache with all parameters set to their default values. + */ + public DiskMemTileCache() { + this(null); + } + + /** + * Creates a new cache. + * + * @param params an optional map of parameters (may be empty or {@code null}) + */ + public DiskMemTileCache(Map params) { + if (params == null) { + params = Collections.emptyMap(); + } + + diagnosticsEnabled = false; + tiles = new HashMap(); + residentTiles = CollectionFactory.map(); + curMemory = 0L; + memThreshold = DEFAULT_MEMORY_THRESHOLD; + + Object o; + ParamDesc desc; + + desc= paramDescriptors.get(KEY_INITIAL_MEMORY_CAPACITY); + memCapacity = (Long)desc.defaultValue; + o = params.get(desc.key); + if (o != null) { + if (desc.typeOK(o)) { + memCapacity = ((Number)o).longValue(); + } + } + + desc = paramDescriptors.get(KEY_ALWAYS_DISK_CACHE); + writeNewTilesToDisk = (Boolean)desc.defaultValue; + o = params.get(desc.key); + if (o != null) { + if (desc.typeOK(o)) { + writeNewTilesToDisk = (Boolean)o; + } + } + + desc = paramDescriptors.get(KEY_AUTO_FLUSH_MEMORY_INTERVAL); + autoFlushInterval = ((Number)desc.defaultValue).longValue(); + o = params.get(desc.key); + if (o != null) { + if (desc.typeOK(o)) { + long lval = ((Number)o).longValue(); + if (lval > 0) { + autoFlushInterval = lval; + } + } + } + + desc = paramDescriptors.get(KEY_AUTO_FLUSH_MEMORY_ENABLED); + o = params.get(desc.key); + if (o != null) { + if (desc.typeOK(o)) { + setAutoFlushMemoryEnabled((Boolean)o); + } + } + + comparator = new TileAccessTimeComparator(); + sortedResidentTiles = new ArrayList(); + + tilePollingService = Executors.newSingleThreadScheduledExecutor( + new DaemonThreadFactory(Thread.MIN_PRIORITY, "cache-polling")); + + startTilePolling(); + } + + /** + * Adds a tile to the cache if not already present. + * + * @param owner the image that this tile belongs to + * @param tileX the tile column + * @param tileY the tile row + * @param data the tile data + */ + public void add(RenderedImage owner, int tileX, int tileY, Raster data) { + add(owner, tileX, tileY, data, null); + } + + /** + * Adds a tile to the cache if not already present. + * + * @param owner the image that this tile belongs to + * @param tileX the tile column + * @param tileY the tile row + * @param data the tile data + * @param tileCacheMetric optional tile cache metric (may be {@code null} + */ + public void add(RenderedImage owner, + int tileX, + int tileY, + Raster data, + Object tileCacheMetric) { + + tileLock.lock(); + + try { + okToFlush.set(false); + Object key = getTileId(owner, tileX, tileY); + if (tiles.containsKey(key)) { + // tile is already cached + return; + } + + DiskCachedTile tile = new DiskCachedTile( + key, owner, tileX, tileY, data, writeNewTilesToDisk, tileCacheMetric); + tiles.put(key, tile); + + if ( makeResident(tile, data) ) { + tile.setAction(DiskCachedTile.TileAction.ACTION_ADDED_RESIDENT); + } else { + tile.setAction(DiskCachedTile.TileAction.ACTION_ADDED); + } + + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Unable to cache this tile on disk", ex); + + } finally { + tileLock.unlock(); + } + } + + /** + * Removes a tile from the cache. + * + * @param owner the image that this tile belongs to + * @param tileX the tile column + * @param tileY the tile row + */ + public void remove(RenderedImage owner, int tileX, int tileY) { + tileLock.lock(); + + try { + okToFlush.set(false); + Object key = getTileId(owner, tileX, tileY); + DiskCachedTile tile = tiles.get(key); + + if (tile == null) { + return; + } + + if (residentTiles.containsKey(key)) { + try { + removeResidentTile(key, false); + + } catch (DiskCacheFailedException ex) { + /* + * It would be nicer to just throw this exception + * upwards but we can't in the overidden method + */ + LOGGER.log(Level.SEVERE, null, ex); + } + } + + tile.deleteDiskCopy(); + + tile.setAction(DiskCachedTile.TileAction.ACTION_REMOVED); + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + + tiles.remove(key); + + } finally { + tileLock.unlock(); + } + } + + /** + * Gets the specified tile from the cache if present. If the tile is + * cached but not resident in memory it will be read from the cache's + * disk storage and made resident. + * + * @param owner the image that the tile belongs to + * @param tileX the tile column + * @param tileY the tile row + * @return the requested tile or {@code null} if the tile was not cached + */ + public Raster getTile(RenderedImage owner, int tileX, int tileY) { + tileLock.lock(); + + try { + okToFlush.set(false); + Raster r = null; + Object key = getTileId(owner, tileX, tileY); + + DiskCachedTile tile = tiles.get(key); + if (tile != null) { + + // is the tile resident ? + r = residentTiles.get(key); + if (r == null) { + /* + * The tile is not resident. Attempt + * to read it from the disk. + */ + r = tile.readData(); + if (r == null) { + /* The tile was not cached on disk. It may have + * been resident only, and then flushed. + */ + return null; + } + + if (makeResident(tile, r)) { + tile.setAction(DiskCachedTile.TileAction.ACTION_RESIDENT); + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + } + } + + tile.setAction(DiskCachedTile.TileAction.ACTION_ACCESSED); + tile.setTileTimeStamp(System.currentTimeMillis()); + + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + } + + return r; + } finally { + tileLock.unlock(); + } + } + + /** + * Gets all cached tiles associated with the given image. + * The tiles will be loaded into memory as space allows. + * + * @param owner the image for which tiles are requested + * @return an array of tile Rasters + */ + public Raster[] getTiles(RenderedImage owner) { + tileLock.lock(); + + try { + okToFlush.set(false); + int minX = owner.getMinTileX(); + int minY = owner.getMinTileY(); + int numX = owner.getNumXTiles(); + int numY = owner.getNumYTiles(); + + List keys = new ArrayList(); + for (int y = minY, ny = 0; ny < numY; y++, ny++) { + for (int x = minX, nx = 0; nx < numX; x++, nx++) { + Object key = getTileId(owner, x, y); + if (tiles.containsKey(key)) { + keys.add(key); + } + } + } + + Raster[] rasters = new Raster[keys.size()]; + int k = 0; + for (Object key : keys) { + DiskCachedTile tile = tiles.get(key); + Raster r = residentTiles.get(tile.getTileId()); + if (r == null) { + r = tile.readData(); + makeResident(tile, r); + } + + rasters[k++] = r; + + tile.setTileTimeStamp(System.currentTimeMillis()); + tile.setAction(DiskCachedTile.TileAction.ACTION_ACCESSED); + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + } + + return rasters; + + } finally { + tileLock.unlock(); + } + } + + /** + * Removes all tiles that belong to the given image from the cache. + * + * @param owner the image owning the tiles to be removed + */ + public void removeTiles(RenderedImage owner) { + tileLock.lock(); + try { + for (int y = owner.getMinTileY(), ny = 0; ny < owner.getNumYTiles(); y++, ny++) { + for (int x = owner.getMinTileX(), nx = 0; nx < owner.getNumXTiles(); x++, nx++) { + remove(owner, x, y); + } + } + } finally { + tileLock.unlock(); + } + } + + /** + * Sets the interval between polling each tile to check if its owning image + * has been garbage collected. Any such tiles are removed from the + * cache. + * + * @param interval interval in milliseconds + * (values less than or equal to zero are ignored) + */ + public void setTilePollingInterval(long interval) { + if (interval > 0 && interval != tilePollingInterval) { + stopTilePolling(); + tilePollingInterval = interval; + startTilePolling(); + } + } + + /** + * Sets the interval between polling each tile to check if its owning image + * has been garbage collected. Any such tiles are removed from the + * cache. * + * @return interval in milliseconds + */ + public long getTilePollingInterval() { + return tilePollingInterval; + } + + /** + * Starts the tile polling task which calls {@link #removeNullTiles()} + * at a fixed interval. + */ + private void startTilePolling() { + if (!isPollingTiles()) { + tilePollingFuture = tilePollingService.scheduleAtFixedRate( + new Runnable() { + public void run() { + removeNullTiles(); + } + }, + tilePollingInterval, + tilePollingInterval, + TimeUnit.MILLISECONDS); + } + } + + /** + * Stops the tile polling task. + */ + private void stopTilePolling() { + if (isPollingTiles()) { + tilePollingFuture.cancel(true); + tilePollingFuture = null; + } + } + + private boolean isPollingTiles() { + return tilePollingFuture != null && !tilePollingFuture.isDone(); + } + + /** + * Checks if any tiles have a {@code null} owner (e.g. owning image has been + * garbage collected) and, if so, removes them from the cache. + */ + private void removeNullTiles() { + if (!tileLock.tryLock()) { // jumps the queue of waiting threads + return; + } + + try { + Set nullTileKeys = CollectionFactory.set(); + for (Object key : tiles.keySet()) { + DiskCachedTile tile = tiles.get(key); + if (tile.getOwner() == null) { + nullTileKeys.add(key); + } + } + + for (Object key : nullTileKeys) { + DiskCachedTile tile = tiles.get(key); + tile.deleteDiskCopy(); + if (residentTiles.containsKey(key)) { + residentTiles.remove(key); + sortedResidentTiles.remove(tile); + curMemory -= tile.getTileSize(); + } + tiles.remove(key); + } + + } finally { + tileLock.unlock(); + } + } + + /** + * Adds all tiles for the given image to the cache. + * + * @param owner the image that the tiles belong to + * @param tileIndices an array of Points specifying the column-row coordinates + * of each tile + * @param tiles tile data in the form of Raster objects + * @param tileCacheMetric optional metric (may be {@code null}) + */ + public void addTiles(RenderedImage owner, + Point[] tileIndices, + Raster[] tiles, + Object tileCacheMetric) { + + if (tileIndices.length != tiles.length) { + throw new IllegalArgumentException( + "tileIndices and tiles args must be the same length"); + } + + tileLock.lock(); + try { + for (int i = 0; i < tiles.length; i++) { + add(owner, tileIndices[i].x, tileIndices[i].y, tiles[i], tileCacheMetric); + } + + } finally { + tileLock.unlock(); + } + } + + /** + * Gets the specified tiles for the given image. + * + * @param owner the image that the tiles belong to + * @param tileIndices an array of Points specifying the column-row coordinates + * of each tile + * @return data for the requested tiles as Raster objects + */ + public Raster[] getTiles(RenderedImage owner, Point[] tileIndices) { + tileLock.lock(); + try { + Raster[] r = null; + + if (tileIndices.length > 0) { + r = new Raster[tileIndices.length]; + for (int i = 0; i < tileIndices.length; i++) { + r[i] = getTile(owner, tileIndices[i].x, tileIndices[i].y); + } + } + + return r; + + } finally { + tileLock.unlock(); + } + } + + /** + * Removes ALL tiles from the cache: all resident tiles will be + * removed from memory and all files for disk-cached tiles will + * be discarded. + *

+ * The update action of each tile will be set to {@linkplain DiskCachedTile#ACTION_REMOVED}. + */ + public void flush() { + tileLock.lock(); + + try { + flushMemory(); + + for (DiskCachedTile tile : tiles.values()) { + tile.deleteDiskCopy(); + tile.setAction(DiskCachedTile.TileAction.ACTION_REMOVED); + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + } + tiles.clear(); + + } finally { + tileLock.unlock(); + } + } + + /** + * Removes all resident tiles from memory. No rewriting of tile data + * to disk is done. + */ + public void flushMemory() { + tileLock.lock(); + try { + residentTiles.clear(); + sortedResidentTiles.clear(); + curMemory = 0; + + } finally { + tileLock.unlock(); + } + } + + /** + * Frees memory for resident tiles so that the fraction of memory occupied is + * no more than the current value of the mamory threshold. + * + * @see DiskMemTileCache#setMemoryThreshold(float) + */ + public void memoryControl() { + tileLock.lock(); + try { + long maxUsed = (long) (memThreshold * memCapacity); + long toFree = curMemory - maxUsed; + if (toFree > 0) { + defaultMemoryControl(toFree); + } + } finally { + tileLock.unlock(); + } + } + + /** + * Makes the requested amount of memory cache available, removing + * resident tiles as necessary. + * + * @param memRequired memory requested (bytes) + */ + private void defaultMemoryControl( long memRequired ) { + if (memRequired > memCapacity) { + // @todo something better than this... + throw new RuntimeException("space required is greater than cache memory capacity"); + } + + /* + * Remove one or more lowest priority tiles to free + * space + */ + Collections.sort(sortedResidentTiles, comparator); + while (memCapacity - curMemory < memRequired && !sortedResidentTiles.isEmpty()) { + Object key = sortedResidentTiles.get(sortedResidentTiles.size()-1).getTileId(); + + try { + removeResidentTile(key, true); + } catch (DiskCacheFailedException ex) { + /* + * It would be nicer to just throw this exception + * upwards be we can't in the overidden method + */ + LOGGER.log(Level.SEVERE, null, ex); + } + } + } + + + /** + * Does nothing. + * + * @deprecated Please do not use this method + */ + @Deprecated + public void setTileCapacity(int arg0) { + } + + /** + * Always returns 0. + * + * @deprecated Please do not use this method + */ + public int getTileCapacity() { + return 0; + } + + /** + * Resets the memory capacity of the cache. Setting capacity to 0 will + * flush all resident tiles from memory. Setting a capacity less than the + * current capacity could cause some memory-resident tiles being + * removed from memory. + * + * @param newCapacity requested memory capacity for resident tiles + */ + public void setMemoryCapacity(long newCapacity) { + tileLock.lock(); + + try { + okToFlush.set(false); + if (newCapacity < 0) { + throw new IllegalArgumentException("memory capacity must be >= 0"); + } + + long oldCapacity = memCapacity; + memCapacity = newCapacity; + + if (newCapacity == 0) { + flushMemory(); + + } else if (newCapacity < oldCapacity && curMemory > newCapacity) { + /* + * Note: we free memory here directly rather than using + * memoryControl or defaultMemoryControl methods because + * they will fail when memCapacity has been reduced + */ + Collections.sort(sortedResidentTiles, comparator); + while (curMemory > newCapacity) { + Object key = sortedResidentTiles.get(sortedResidentTiles.size() - 1).getTileId(); + try { + removeResidentTile(key, true); + } catch (DiskCacheFailedException ex) { + /* + * It would be nicer to just throw this exception + * upwards be we can't in the overidden method + */ + LOGGER.log(Level.SEVERE, null, ex); + } + } + } + } finally { + tileLock.unlock(); + } + } + + /** + * Gets the amount of memory, in bytes, allocated for storage of + * resident tiles. + * + * @return resident tile memory capacity in bytes + */ + public long getMemoryCapacity() { + return memCapacity; + } + + /** + * Gets the amount of memory currently being used for storage of + * memory-resident tiles. + * + * @return current memory use in bytes + */ + public long getCurrentMemory() { + return curMemory; + } + + /** + * Sets the memoryThreshold value to a floating point number that ranges from + * 0.0 to 1.0. When the cache memory is full, the memory usage will be reduced + * to this fraction of the total cache memory capacity. For example, a value + * of .75 will cause 25% of the memory to be cleared, while retaining 75%. + * + * @param newThreshold the new memory threshold between 0 and 1 + * + * @throws IllegalArgumentException if the memoryThreshold is less than 0.0 or greater than 1.0 + */ + public void setMemoryThreshold(float newThreshold) { + if (newThreshold < 0.0F) { + memThreshold = 0.0F; + } else if (newThreshold > 1.0F) { + memThreshold = 1.0F; + } else { + memThreshold = newThreshold; + } + + memoryControl(); + } + + /** + * Returns the memory threshold, which is the fractional amount of cache memory + * to retain during tile removal. This only applies if memory thresholding has + * been enabled by passing the parameter {@linkplain #KEY_USE_MEMORY_THRESHOLD} to + * the constructor with a value of {@code Boolean.TRUE}. + * + * @return the retained fraction of memory + */ + public float getMemoryThreshold() { + return memThreshold; + } + + /** + * Sets the comparator to use to assign memory-residence priority to + * tiles. If {@code comp} is {@code null} the default comparator + * ({@link TileAccessTimeComparator}) will be used. + * + * @param comp the comparator or {@code null} for the default + */ + public void setTileComparator(Comparator comp) { + tileLock.lock(); + try { + if (comp == null) { + // switch to default comparator based on tile access time + comparator = new TileAccessTimeComparator(); + } else { + comparator = comp; + } + + sortedResidentTiles = new ArrayList(); + for (Object key : residentTiles.keySet()) { + sortedResidentTiles.addAll(tiles.values()); + } + Collections.sort(sortedResidentTiles, comparator); + + } finally { + tileLock.unlock(); + } + } + + /** + * Gets the comparator currently used to assign memory-residence + * priority to tiles. + * + * @return the current comparator + */ + public Comparator getTileComparator() { + return comparator; + } + + /** + * Gets the total number of tiles currently in the cache. + * + * @return number of cached tiles + */ + public int getNumTiles() { + return tiles.size(); + } + + /** + * Gets the number of tiles currently residing in the + * cache's memory storage. + * + * @return number of memory-resident tiles + */ + public int getNumResidentTiles() { + return residentTiles.size(); + } + + /** + * Checks whether a given tile is in this cache. + * + * @param owner the owning image + * @param tileX tile column + * @param tileY tile row + * @return {@code true} if the cache contains the tile; {@code false} otherwise + */ + public boolean containsTile(RenderedImage owner, int tileX, int tileY) { + Object key = getTileId(owner, tileX, tileY); + return tiles.containsKey(key); + } + + /** + * Checks whether a given tile is in this cache's memory storage. + * + * @param owner the owning image + * @param tileX tile column + * @param tileY tile row + * @return {@code true} if the tile is in cache memory; {@code false} otherwise + */ + public boolean containsResidentTile(RenderedImage owner, int tileX, int tileY) { + Object key = getTileId(owner, tileX, tileY); + return residentTiles.containsKey(key); + } + + /** + * Informs the cache that a tile's data have changed. The tile should + * be resident in memory as the result of a previous {@code getTile} + * request. If this is the case and the tile was previously written to + * disk, then the cache's disk copy of the tile will be refreshed. + *

+ * If the tile is not resident in memory, for instance + * because of memory swapping for other tile accesses, the disk copy + * will not be refreshed and a {@code TileNotResidentException} is + * thrown. + * + * @param owner the owning image + * @param tileX tile column + * @param tileY tile row + * @throws TileNotResidentException if the tile is not resident + * @throws DiskCacheFailedException if the tile is cached to disk but its data could + * not be updated + */ + public void setTileChanged(RenderedImage owner, int tileX, int tileY) + throws TileNotResidentException, DiskCacheFailedException { + + tileLock.lock(); + try { + okToFlush.set(false); + Object tileId = getTileId(owner, tileX, tileY); + Raster r = residentTiles.get(tileId); + if (r == null) { + throw new TileNotResidentException(owner, tileX, tileY); + } + + DiskCachedTile tile = tiles.get(tileId); + if (tile.cachedToDisk()) { + try { + tile.writeData(r); + } catch (IOException ioEx) { + throw new DiskCacheFailedException(owner, tileX, tileY); + } + } + + } finally { + tileLock.unlock(); + } + } + + /** + * Enables or disables auto-flushing of memory resident with the + * currently set minimum interval. + * + * @param enable {@code true} to enable auto-flushing; {@code false} to disable + * @see #setAutoFlushMemoryInterval(long) + */ + public final void setAutoFlushMemoryEnabled(boolean enable) { + if (enable) { + if (!isAutoFlushMemoryEnabled()) { + if (flushService == null) { + flushService = Executors.newSingleThreadScheduledExecutor( + new DaemonThreadFactory(Thread.MIN_PRIORITY, "cache-flush")); + } + + flushFuture = flushService.scheduleWithFixedDelay( + new Runnable() { + public void run() { + if (okToFlush.getAndSet(true)) { + flushMemory(); + } + } + }, + autoFlushInterval, + autoFlushInterval, + TimeUnit.MILLISECONDS); + } + + } else if (isAutoFlushMemoryEnabled()) { + flushFuture.cancel(true); + } + } + + /** + * Checks whether auto-flushing of memory-resident tiles is currently enabled. + * + * @return {@code true} if the cache is auto-flushing; {@code false} otherwise + */ + public boolean isAutoFlushMemoryEnabled() { + return (flushFuture != null && !flushFuture.isDone()); + } + + /** + * Sets the minimum period of cache inactivity, in milliseconds, that must + * elapse before automatically flushing memory-resident tiles. + * + * @param interval interval in milliseconds + * (values less than or equal to zero are ignored) + */ + public void setAutoFlushMemoryInterval(long interval) { + if (interval > 0 && interval != autoFlushInterval) { + if (isAutoFlushMemoryEnabled()) { + setAutoFlushMemoryEnabled(false); + } + autoFlushInterval = interval; + setAutoFlushMemoryEnabled(true); + } + } + + /** + * Gets the current auto-flush interval. This is the minimum period of + * cache inactivity, in milliseconds, that must elapse before + * automatically flushing tiles. + * + * @return interval in milliseconds + */ + public long getAutoFlushMemoryInterval() { + return autoFlushInterval; + } + + /** + * Enables or disables the publishing of cache messages to Observers. + * + * @param state {@code true} to publish diagnostic messages; {@code false} to suppress them + */ + public void setDiagnostics(boolean state) { + diagnosticsEnabled = state; + } + + /** + * Accepts a {@code DiskMemCacheVisitor} object and calls its + * {@code visit} method for each tile in the cache. + * + * @param visitor the visitor + */ + public void accept(DiskMemTileCacheVisitor visitor) { + tileLock.lock(); + try { + okToFlush.set(false); + for (Object key : tiles.keySet()) { + visitor.visit(tiles.get(key), residentTiles.containsKey(key)); + } + } finally { + tileLock.unlock(); + } + } + + /** + * Adds a raster to those resident in memory. + */ + private boolean makeResident(DiskCachedTile tile, Raster data) { + if (tile.getTileSize() > memCapacity) { + return false; + } + + if (tile.getTileSize() > memCapacity - curMemory) { + memoryControl(); + + /* + * It is possible that the threshold rule fails to + * free enough memory for the tile + */ + if (tile.getTileSize() > memCapacity - curMemory) { + defaultMemoryControl(tile.getTileSize()); + } + } + + residentTiles.put(tile.getTileId(), data); + curMemory += tile.getTileSize(); + + /* + * We don't bother about sort order here. Instead, the list + * will be sorted by tile priority when resident tiles are + * being removed + */ + sortedResidentTiles.add(tile); + + return true; + } + + + /** + * Removes a tile from the cache's memory storage. This may be to free + * space for other tiles, in which case {@code writeData} will be + * set to {@code true} and, if the tile is writable, a request is made to write + * its data to disk again. If the tile is being removed from the cache + * entirely, this method will be called with {@code writeData} set + * to {@code false}. + * + * @param tileId the tile's unique id + * @param writeData if {@code true}, and the tile is writable, its data will be + * written to disk again; otherwise no writing is done. + */ + private void removeResidentTile(Object tileId, boolean writeData) throws DiskCacheFailedException { + DiskCachedTile tile = tiles.get(tileId); + Raster raster = residentTiles.remove(tileId); + sortedResidentTiles.remove(tile); + curMemory -= tile.getTileSize(); + + /** + * If the tile is writable, ie. its data are represented + * by a WritableRaster, we cache it to disk + */ + if (writeData && tile.isWritable()) { + try { + tile.writeData(raster); + } catch (IOException ioEx) { + throw new DiskCacheFailedException(tile.getOwner(), tile.getTileX(), tile.getTileY()); + } + } + + tile.setAction(DiskCachedTile.TileAction.ACTION_NON_RESIDENT); + if (diagnosticsEnabled) { + setChanged(); + notifyObservers(tile); + } + } + + + /** + * Generates a unique ID for this tile. This uses the same technique as the + * Sun memory cache implementation: putting the id of the owning image + * into the upper bytes of a long or BigInteger value and the tile index into + * the lower bytes. + * @param owner the owning image + * @param tileX tile column + * @param tileY tile row + * @return the ID as an Object which will be an instance of either Long or BigInteger + */ + private Object getTileId(RenderedImage owner, + int tileX, + int tileY) { + + long tileId = tileY * (long)owner.getNumXTiles() + tileX; + + BigInteger imageId = null; + + if (owner instanceof PlanarImage) { + imageId = (BigInteger)((PlanarImage)owner).getImageID(); + } + + if (imageId != null) { + byte[] buf = imageId.toByteArray(); + int length = buf.length; + byte[] buf1 = new byte[buf.length + 8]; + System.arraycopy(buf, 0, buf1, 0, length); + for (int i = 7, j = 0; i >= 0; i--, j += 8) + buf1[length++] = (byte)(tileId >> j); + return new BigInteger(buf1); + + } else { + tileId &= 0x00000000ffffffffL; + return Long.valueOf(((long)owner.hashCode() << 32) | tileId); + } + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java new file mode 100644 index 000000000..8f96f6791 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +/** + * A visitor to collect information about tiles in a {@linkplain DiskMemTileCache}. + * This can be used to examine cache performance in more detail than with + * the cache's diagnostic methods and observer events. + * + * @see DiskMemTileCache + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public interface DiskMemTileCacheVisitor { + + /** + * Called by the cache once for each tile + * + * @param tile the tile being visited + * @param isResident set by the cache to indicate whether the tile is + * currently resident in memory + */ + public void visit(DiskCachedTile tile, boolean isResident); + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java b/klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java new file mode 100644 index 000000000..5042bca90 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.io.Serializable; +import java.util.Comparator; +import org.eclipse.imagen.CachedTile; + +/** + * This is the default comparator for CachedTile objects used by + * {@linkplain DiskMemTileCache} to determine priority of tiles for + * storage in memory. It orders tiles from most recent to least + * recent access time. + * + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class TileAccessTimeComparator implements Comparator, Serializable { + + public int compare(CachedTile t1, CachedTile t2) { + long time1 = t1.getTileTimeStamp(); + long time2 = t2.getTileTimeStamp(); + + // t1 > t2 if time1 < time2 + return (time1 > time2 ? -1 : (time1 == time2 ? 0 : 1)); + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java b/klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java new file mode 100644 index 000000000..816dbe8d7 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tilecache; + +import java.awt.image.RenderedImage; + +/** + * Exception thrown when by DiskMemTileCache when a tile was + * not resident in cache memory when required to be. + * + * @see DiskMemTileCache + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class TileNotResidentException extends DiskMemCacheException { + + TileNotResidentException(RenderedImage owner, int tileX, int tileY) { + super(owner, tileX, tileY, " is not resident in memory"); + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java b/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java new file mode 100644 index 000000000..7abbfa13c --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java @@ -0,0 +1,818 @@ +/* + * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tiledimage; + +import java.awt.Graphics2D; +import java.awt.Point; +import java.awt.Rectangle; +import java.awt.image.ColorModel; +import java.awt.image.DataBuffer; +import java.awt.image.Raster; +import java.awt.image.SampleModel; +import java.awt.image.TileObserver; +import java.awt.image.WritableRaster; +import java.awt.image.WritableRenderedImage; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.eclipse.imagen.ImageLayout; +import org.eclipse.imagen.PlanarImage; + +import org.jaitools.tilecache.DiskMemTileCache; +import org.jaitools.CollectionFactory; + + +/** + * A tiled image class similar to JAI's standard {@code TiledImage} that uses a + * {@code DiskMemTileCache} to manage its data. Image tiles are cached in memory + * and, if the volume of image data is greater than available memory, also on disk + * as temporary files. When a tile is accessed it is swapped into memory if not + * already resident. Thus, very large images can be handled, albeit at the cost of + * disk I/O. + *

+ * By default, each {@code DiskMemImage} uses its own tile cache but it is also + * possible for images to share a common tile cache as shown here: + * 


+ *     DiskMemImage image1 = new DiskMemImage(...);
+ *     image1.useCommonTileCache(true);
+ *
+ *     DiskMemImage image2 = new DiskMemImage(...);
+ *     image2.useCommonTileCache(true);
+ *
+ *     DiskMemImage image3 = new DiskMemImage(...);
+ *
+ * In the example above, image1 and image2 will share a common tile cache while + * image3 will use a separate, exclusive tile cache. You can test whether an + * image is using the common cache like this: + * 

+ *     boolean usingCommonCache = image.isUsingCommonTileCache();
+ *
+ * The memory capacity of the common cache can be set like this: + * 

+ *     long memCapacity = 128 * 1024 * 1024; // 128 Mb
+ *     DiskMemImage.getCommonCache().setMemoryCapacity(memCapacity);
+ *
+ * + * @see DiskMemTileCache + * + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class DiskMemImage + extends PlanarImage + implements WritableRenderedImage { + + /** + * The default memory capacity of the common tile cache in bytes + * (256 * 1024 * 1024 = 256 Mb) + */ + public static final long DEFAULT_COMMON_CACHE_SIZE = 256L * 1024 * 1024; + + /** TileCache which will be shared bewtween all Images */ + private static DiskMemTileCache commonCache = null; + + /** + * Get the common tile cache. This method has the side effect of + * creating the common cache if it does not already exist (ie. it + * never returns {@code null}). + * + * @return the common tile cache + * + * @see #isUsingCommonCache() + * @see #setUseCommonCache(boolean) + */ + public static DiskMemTileCache getCommonTileCache() { + if (commonCache == null) { + commonCache = createNewCache(); + } + return commonCache; + } + + /** + * Sets a new common cache. If a common cache has already been + * initialized (either with this method or via {@linkplain #setUseCommonCache}) + * this method does nothing. + * + * @param newCache the cache instance to use as the common tile + * cache for {@code DiskMemImage} objects ({@code null} + * arguments are ignored) + */ + public static void setCommonTileCache(DiskMemTileCache newCache) { + if (newCache != null && commonCache == null) { + commonCache = newCache; + } + } + + /** + * Creates a new tile cache. + * + * @return a new instance of {@code DiskMemTileCache} + */ + private static DiskMemTileCache createNewCache() { + Map cacheParams = new HashMap(); + cacheParams.put(DiskMemTileCache.KEY_INITIAL_MEMORY_CAPACITY, Long.valueOf(DEFAULT_COMMON_CACHE_SIZE)); + cacheParams.put(DiskMemTileCache.KEY_ALWAYS_DISK_CACHE, Boolean.FALSE); + DiskMemTileCache cache = new DiskMemTileCache(cacheParams); + cache.setDiagnostics(false); + + return cache; + } + + /** + * The tile cache in use by this image. It may or may not be the + * common tile cache. Ahthough this is a {@code protected} field, + * it is recommended that sub-classes use {@linkplain #getTileCache()} + * for general purposes. + */ + protected DiskMemTileCache tileCache; + + /** + * Defines the valid range of tile x and y coordinates. + */ + protected Rectangle tileGrid; + + /** + * A 2D array with dimensions corresponding to the tile grid width + * and height that records, for each tile, the number of writers + * that have the tile checked out currently + */ + protected int[][] numWriters; + + /** + * The number of tiles that are currently checked out + * for writing + * + * @see #getWritableTileIndices() + */ + protected int numTilesInUse; + + /** + * The amount of memory (in bytes) required to hold + * the data for an image tile + */ + protected long tileMemorySize; + + /** + * The set of tile observers currently registered with + * this image + */ + protected Set tileObservers; + + /** + * Creates a new image with default values for origin (0, 0), + * x and y tile offsets (0) and {@code ColorModel} + * ({@code null}). + * + * @param width image width + * @param height image height + * @param tileSampleModel a {@code SampleModel} specifying the dimensions + * data type etc. for image tiles + */ + public DiskMemImage( + int width, int height, + SampleModel tileSampleModel) { + + this(0, 0, // minX, minY + width, height, + 0, 0, // tileGridXOffset, tileGridYOffset + tileSampleModel, + null // ColorModel + ); + } + + + /** + * Creates a new image with default values for origin (0, 0) and + * tile offsets (0). + * + * @param width image width + * @param height image height + * @param tileSampleModel the {@code SampleModel} + * @param colorModel the {@code ColorModel} + */ + public DiskMemImage( + int width, int height, + SampleModel tileSampleModel, + ColorModel colorModel) { + + this(0, 0, // minX, minY + width, height, + 0, 0, // tileGridXOffset, tileGridYOffset + tileSampleModel, + colorModel + ); + } + + + /** + * Creates a new image with default values for tile offsets (0) + * and {@code ColorModel} ({@code null}) + * + * @param minX x coordinate of the upper-left image pixel + * @param minY y coordinate of the upper-left image pixel + * @param width image width + * @param height image height + * @param tileSampleModel the {@code SampleModel} + */ + public DiskMemImage( + int minX, int minY, + int width, int height, + SampleModel tileSampleModel) { + + this(minX, minY, + width, height, + 0, 0, // tileGridXOffset, tileGridYOffset + tileSampleModel, + null // ColorModel + ); + } + + + /** + * Creates a new image. Sets the {@code ColorModel} to {@code null} + * + * @param minX x coordinate of the upper-left image pixel + * @param minY y coordinate of the upper-left image pixel + * @param width image width + * @param height image height + * data type etc. for image tiles + * @param tileGridXOffset x coordinate of the upper-left pixel of the upper-left tile + * @param tileGridYOffset y coordinate of the upper-left pixel of the upper-left tile + * @param tileSampleModel the {@code SampleModel} + */ + public DiskMemImage( + int minX, int minY, + int width, int height, + int tileGridXOffset, int tileGridYOffset, + SampleModel tileSampleModel) { + + this(minX, minY, + width, height, + tileGridXOffset, tileGridYOffset, + tileSampleModel, + null // ColorModel + ); + } + + /** + * Creates a new image. + * + * @param minX x coordinate of the upper-left image pixel + * @param minY y coordinate of the upper-left image pixel + * @param width image width + * @param height image height + * data type etc. for image tiles + * @param tileGridXOffset x coordinate of the upper-left pixel of the upper-left tile + * @param tileGridYOffset y coordinate of the upper-left pixel of the upper-left tile + * @param tileSampleModel the {@code SampleModel} + * @param colorModel the {@code ColorModel} + */ + public DiskMemImage( + int minX, int minY, + int width, int height, + int tileGridXOffset, int tileGridYOffset, + SampleModel tileSampleModel, ColorModel colorModel) { + + super(new ImageLayout( + minX, minY, + width, height, + tileGridXOffset, tileGridYOffset, + tileSampleModel.getWidth(), tileSampleModel.getHeight(), + tileSampleModel, colorModel), + + null, null); // sources, properties + + tileGrid = new Rectangle( + getMinTileX(), + getMinTileY(), + getMaxTileX() - getMinTileX() + 1, + getMaxTileY() - getMinTileY() + 1); + + numWriters = new int[tileGrid.width][tileGrid.height]; + numTilesInUse = 0; + + DataBuffer db = tileSampleModel.createDataBuffer(); + tileMemorySize = DataBuffer.getDataTypeSize(db.getDataType()) / 8L * + db.getSize() * db.getNumBanks(); + + tileObservers = CollectionFactory.set(); + + // just to remind us that we are deferring creation of + // the tile cache + tileCache = null; + } + + /** + * Gets a tile for reading. Any changes to the tile's data + * will not be preserved by the cache. + * + * @param tileX the tile's column in the tile grid + * @param tileY the tile's row in the tile grid + * @return the tile data for reading + */ + @Override + public Raster getTile(int tileX, int tileY) { + Raster r = null; + if (tileGrid.contains(tileX, tileY)) { + r = getTileCache().getTile(this, tileX, tileY); + if (r == null) { + r = createTile(tileX, tileY); + getTileCache().add(this, tileX, tileY, r); + } + } + + return r; + } + + /** + * {@inheritDoc} + */ + public void addTileObserver(TileObserver to) { + tileObservers.add(to); + } + + /** + * {@inheritDoc} + */ + public void removeTileObserver(TileObserver to) { + tileObservers.remove(to); + } + + /** + * Checks out a tile for writing. The tile will be unavailable to other + * callers through this method until it is released via + * {@linkplain #releaseWritableTile(int, int)}. If this method is + * called for the same time prior to the tile being released + * it returns {@code null} and a warning message is logged. + * + * @param tileX tile X ordinate + * @param tileY tile Y ordinate + * @return the tile data for writing, or {@code null} if the tile + * is already checked-out + */ + public WritableRaster getWritableTile(int tileX, int tileY) { + WritableRaster r = null; + if (tileGrid.contains(tileX, tileY)) { + numWriters[tileX - tileGrid.x][tileY - tileGrid.y]++ ; + if (numWriters[tileX - tileGrid.x][tileY - tileGrid.y] == 1) { + numTilesInUse++ ; + } + + r = (WritableRaster) getTileCache().getTile(this, tileX, tileY); + if (r == null) { + r = createTile(tileX, tileY); + getTileCache().add(this, tileX, tileY, r); + } + + for (TileObserver obs : tileObservers) { + obs.tileUpdate(this, tileX, tileY, true); + } + } + return r; + } + + /** + * Releases a tile that was checked out for writing. The cache's disk copy of + * the tile's data will be refreshed. + *

+ * If the cache no longer has the tile in its memory storage, e.g. + * because of memory swapping for other tile accesses, the cache + * will be unable to refresh the tile's data on disk. In this case + * a warning message is logged. + *

+ * If the tile was not previously checked-out via + * {@linkplain #getWritableTile(int, int)} a warning message is + * logged. + * + * @param tileX tile X ordinate + * @param tileY tile Y ordinate + */ + public void releaseWritableTile(int tileX, int tileY) { + if (tileGrid.contains(tileX, tileY)) { + numWriters[tileX - tileGrid.x][tileY - tileGrid.y]-- ; + if (numWriters[tileX - tileGrid.x][tileY - tileGrid.y] < 0) { + Logger.getLogger( + DiskMemImage.class.getName()).log(Level.SEVERE, + String.format("Tile %d,%d released more times than it has been checked out", + tileX, tileY)); + + } else if (numWriters[tileX - tileGrid.x][tileY - tileGrid.y] == 0) { + numTilesInUse--; + } + + /* + * TODO: Consider skipping this step. It is mostly here as a + * precaution against the cached tile being garbage collected + * if the system runs very low on memory. + */ + try { + getTileCache().setTileChanged(this, tileX, tileY); + + } catch (Exception ex) { + Logger.getLogger(DiskMemImage.class.getName()). + log(Level.SEVERE, null, ex); + } + + for (TileObserver obs : tileObservers) { + obs.tileUpdate(this, tileX, tileY, false); + } + } + } + + /** + * Tests if a tile is currently checked-out for writing (via + * a call to {@linkplain #getWritableTile(int, int)}. + * + * @param tileX tile X ordinate + * @param tileY tile Y ordinate + * @return {@code true} if the tile is currently checked-out for + * writing; {@code false} otherwise. + */ + public boolean isTileWritable(int tileX, int tileY) { + return numWriters[tileX - tileGrid.x][tileY - tileGrid.y] > 0; + } + + /** + * Gets the indices (tile grid col,row) of tiles that are + * currently checked out for writing. + * + * @return tile indices + */ + public Point[] getWritableTileIndices() { + Point[] indices = null; + + if (numTilesInUse > 0) { + indices = new Point[numTilesInUse]; + int k = 0; + for (int y = tileGrid.y, ny = 0; ny < tileGrid.height; y++, ny++) { + for (int x = tileGrid.x, nx = 0; nx < tileGrid.width; x++, nx++) { + if (numWriters[nx][ny] > 0) { + indices[k++] = new Point(x, y); + } + } + } + } + + return indices; + } + + /** + * Tests if any tiles are currently checked out for writing. + * @return {@code true} if any tiles are currently checked out; + * {@code false} otherwise + */ + public boolean hasTileWriters() { + return numTilesInUse > 0; + } + + + /** + * Gets the image value for the given image position as an integer. + * + * @param x X ordinate + * @param y Y ordinate + * @param b band index + * + * @return image value + * @throws PixelOutsideImageException + */ + public int getSample(int x, int y, int b) throws PixelOutsideImageException { + int tileX = XToTileX(x); + int tileY = YToTileY(y); + Raster t = getTile(tileX, tileY); + if (t == null) { + throw new PixelOutsideImageException(x, y, b); + } + return t.getSample(x, y, b); + } + + /** + * Gets the image value for the given image position as a float. + * + * @param x X ordinate + * @param y Y ordinate + * @param b band index + * + * @return image value + * @throws PixelOutsideImageException + */ + public float getSampleFloat(int x, int y, int b) throws PixelOutsideImageException { + int tileX = XToTileX(x); + int tileY = YToTileY(y); + Raster t = getTile(tileX, tileY); + if (t == null) { + throw new PixelOutsideImageException(x, y, b); + } + return t.getSampleFloat(x, y, b); + } + + /** + * Gets the image value for the given image position as a double. + * + * @param x X ordinate + * @param y Y ordinate + * @param b band index + * + * @return image value + * @throws PixelOutsideImageException + */ + public double getSampleDouble(int x, int y, int b) throws PixelOutsideImageException { + int tileX = XToTileX(x); + int tileY = YToTileY(y); + Raster t = getTile(tileX, tileY); + if (t == null) { + throw new PixelOutsideImageException(x, y, b); + } + return t.getSampleDouble(x, y, b); + } + + /** + * Sets the image value for the given image position as an integer. + * + * @param x X ordinate + * @param y Y ordinate + * @param b band index + * @param value the new value + * + * @throws PixelOutsideImageException + */ + public void setSample(int x, int y, int b, int value) throws PixelOutsideImageException { + int tileX = XToTileX(x); + int tileY = YToTileY(y); + try { + WritableRaster t = getWritableTile(tileX, tileY); + if (t == null) { + throw new PixelOutsideImageException(x, y, b); + } + t.setSample(x, y, b, value); + + } finally { + releaseWritableTile(tileX, tileY); + } + } + + /** + * Sets the image value for the given image position as a float. + * + * @param x X ordinate + * @param y Y ordinate + * @param b band index + * @param value the new value + * @throws PixelOutsideImageException + * + */ + public void setSample(int x, int y, int b, float value) throws PixelOutsideImageException { + int tileX = XToTileX(x); + int tileY = YToTileY(y); + try { + WritableRaster t = getWritableTile(tileX, tileY); + if (t == null) { + throw new PixelOutsideImageException(x, y, b); + } + t.setSample(x, y, b, value); + + } finally { + releaseWritableTile(tileX, tileY); + } + } + + /** + * Sets the image value for the given image position as a double. + * + * @param x X ordinate + * @param y Y ordinate + * @param b band index + * @param value the new value + * @throws PixelOutsideImageException + * + */ + public void setSample(int x, int y, int b, double value) throws PixelOutsideImageException { + int tileX = XToTileX(x); + int tileY = YToTileY(y); + try { + WritableRaster t = getWritableTile(tileX, tileY); + if (t == null) { + throw new PixelOutsideImageException(x, y, b); + } + t.setSample(x, y, b, value); + + } finally { + releaseWritableTile(tileX, tileY); + } + } + + /** + * Copies data from the given {@code Raster} object into this + * image. The bounds of {@code data} will be used to + * place the data and only that portion of {@code data} + * within this image's bounds will be copied. + * + * @param data the data to copy + * @throws IllegalArgumentException if {@code data} is {@code null} + */ + public void setData(Raster data) { + if (data == null) { + throw new IllegalArgumentException("The data argument must not be null"); + } + + Rectangle rBounds = data.getBounds(); + Rectangle common = rBounds.intersection(getBounds()); + if (common.isEmpty()) { + return; + } + + int minTileX = XToTileX(common.x); + int maxTileX = XToTileX(common.x + common.width - 1); + int minTileY = YToTileY(common.y); + int maxTileY = YToTileY(common.y + common.height - 1); + + for (int y = minTileY; y <= maxTileY; y++) { + for (int x = minTileX; x <= maxTileX; x++) { + WritableRaster tile = getWritableTile(x, y); + Rectangle tileOverlap = tile.getBounds().intersection(common); + + Raster dataChild = data.createChild( + tileOverlap.x, tileOverlap.y, + tileOverlap.width, tileOverlap.height, + tileOverlap.x, tileOverlap.y, + null); + + WritableRaster tChild = tile.createWritableChild( + tileOverlap.x, tileOverlap.y, + tileOverlap.width, tileOverlap.height, + tileOverlap.x, tileOverlap.y, + null); + + tChild.setRect(dataChild); + } + } + } + + /** + * Creates a Graphics2D object for drawing operations with this image. + * The graphics object will be an instance of {@code DiskMemImageGraphics}. + *

+ * Note that only images of integral data type support graphics operations. + * + * @return a new Graphics2D object + * @throws UnsupportedOperationException if the image is not of integral data type + * @see DiskMemImageGraphics + */ + public Graphics2D createGraphics() { + int dataType = getSampleModel().getDataType(); + if (dataType == DataBuffer.TYPE_BYTE || + dataType == DataBuffer.TYPE_INT || + dataType == DataBuffer.TYPE_SHORT || + dataType == DataBuffer.TYPE_USHORT) + { + return new DiskMemImageGraphics(this); + + } else { + throw new UnsupportedOperationException("Image must have an integral data type"); + } + } + + /** + * Gets the maximum amount of memory, in bytes, that this + * image will use for in-memory storage of its data. Since + * this class uses a {@code DiskMemTileCache} this is not + * the limit of image size. + * + * @return max memory usage + */ + public long getMemoryCapacity() { + return getTileCache().getMemoryCapacity(); + } + + /** + * Gets the amount of memory (bytes) required to store a single + * tile's data. + * + * @return tile memory size in bytes + */ + public long getTileMemorySize() { + return tileMemorySize; + } + + /** + * Sets whether this image will use the common tile cache. Any tiles + * belonging to this image that are already cached will be transferred + * from the image's current tile cache to the common cache (if {@code useCommon} + * is {@code true}) or to a new exclusive tile cache ((if {@code useCommon} + * is {@code false}). + *

+ * By default, the image will use an exclusive cache. + * + * @param useCommon true to set this image to use the common tile cache; false + * for the image to use its own exclusive cache. + * + * @see #getCommonTileCache() + * @see #isUsingCommonCache() + */ + public void setUseCommonCache(boolean useCommon) { + if (useCommon && !isUsingCommonCache()) { + /* + * transfer any existing tiles to the common cache + */ + if (tileCache != null) { + for (int y = getMinTileY(), ny = 0; ny < getNumYTiles(); y++, ny++) { + for (int x = getMinTileX(), nx = 0; nx < getNumXTiles(); x++, nx++) { + Raster tile = tileCache.getTile(this, x, y); + if (tile != null) { + getCommonTileCache().add(this, x, y, (WritableRaster)tile); + tileCache.remove(this, x, y); + } + } + } + } + tileCache = getCommonTileCache(); + + } else if (isUsingCommonCache()) { + /* + * transfer any existing tiles from the common cache to + * this images new exclusive cache + */ + DiskMemTileCache newCache = createNewCache(); + for (int y = getMinTileY(), ny = 0; ny < getNumYTiles(); y++, ny++) { + for (int x = getMinTileX(), nx = 0; nx < getNumXTiles(); x++, nx++) { + Raster tile = getCommonTileCache().getTile(this, x, y); + if (tile != null) { + newCache.add(this, x, y, (WritableRaster)tile); + getCommonTileCache().remove(this, x, y); + } + } + } + tileCache = newCache; + } + } + + /** + * Gets a reference to the {@code DiskMemTileCache} instance + * used by this image. This method is intended for client + * code that wishes to query cache state or receive cache diagnostic + * messages (via the {@code Observer} interface). It is probably not + * a good idea to manipulate the cache state directly. + * + * @return a live reference to the cache being used by this image + * @see #isUsingCommonCache() + * @see DiskMemTileCache + */ + public DiskMemTileCache getTileCache() { + if (tileCache == null) { + tileCache = createNewCache(); + } + + return tileCache; + } + + /** + * Tests if this image is using the common tile cache. + * + * @return {@code true} if using the common tile cache; {@code false} otherwise + * @see #setUseCommonCache(boolean) + */ + public boolean isUsingCommonCache() { + return tileCache != null && tileCache == commonCache; + } + + /** + * Creates a new image tile + * @param tileX tile X ordinate + * @param tileY tile Y ordinate + * @return the new tile + */ + private WritableRaster createTile(int tileX, int tileY) { + assert(getTileCache().getTile(this, tileX, tileY) == null); + + Point location = new Point(tileXToX(tileX), tileYToY(tileY)); + return createWritableRaster(getSampleModel(), location); + } + +} diff --git a/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java b/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java new file mode 100644 index 000000000..c792c6886 --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java @@ -0,0 +1,1027 @@ +/* + * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tiledimage; + +import java.awt.Color; +import java.awt.Composite; +import java.awt.Font; +import java.awt.FontMetrics; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.GraphicsConfiguration; +import java.awt.Image; +import java.awt.Paint; +import java.awt.Point; +import java.awt.Rectangle; +import java.awt.RenderingHints; +import java.awt.RenderingHints.Key; +import java.awt.Shape; +import java.awt.Stroke; +import java.awt.font.FontRenderContext; +import java.awt.font.GlyphVector; +import java.awt.geom.AffineTransform; +import java.awt.geom.Area; +import java.awt.geom.NoninvertibleTransformException; +import java.awt.geom.Point2D; +import java.awt.geom.Rectangle2D; +import java.awt.image.BufferedImage; +import java.awt.image.BufferedImageOp; +import java.awt.image.ColorModel; +import java.awt.image.ImageObserver; +import java.awt.image.Raster; +import java.awt.image.RenderedImage; +import java.awt.image.SampleModel; +import java.awt.image.WritableRaster; +import java.awt.image.renderable.RenderableImage; +import java.lang.reflect.Method; +import java.text.AttributedCharacterIterator; +import java.util.Hashtable; +import java.util.Map; + +import org.eclipse.imagen.PlanarImage; + +/** + * A Graphics class for drawing into a DiskMemImage. + * As with JAI's TiledImageGraphics class, java.awt + * routines do the work and the purpose of this class is to + * serve the image data in a form that those routines can handle. + *

+ * Most of the methods in this class are identical in function to + * those in Graphics2D; these have not been documented here. + * + * @see DiskMemImage + * + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class DiskMemImageGraphics extends Graphics2D { + + private DiskMemImage targetImage; + private ColorModel colorModel; + private Hashtable properties; + private RenderingHints renderingHints; + + /** + * Constants for paint mode: PAINT or XOR. + */ + public static enum PaintMode { + /** Simple paint mode. */ + PAINT, + /** XOR paint mode. */ + XOR; + }; + + /* + * java.awt.Graphics parameters + */ + private Point origin; + private Shape clip; + private Color color; + private Font font; + private PaintMode paintMode; + private Color XORColor; + + /* + * java.awt.Graphics2D parameters + */ + private Color background; + private Composite composite; + private Paint paint; + private Stroke stroke; + private AffineTransform transform; + + /** + * Constants and associated data for graphics operations + */ + public static enum OpType { + + /** Describes the clearRect method. */ + CLEAR_RECT("clearRect", int.class, int.class, int.class, int.class), + + /** Describes the copyArea method. */ + COPY_AREA("copyArea", int.class, int.class, int.class, int.class, int.class, int.class), + + /** Describes the drawArc method. */ + DRAW_ARC("drawArc", int.class, int.class, int.class, int.class, int.class, int.class), + + /** Describes the drawImage method. */ + DRAW_BUFFERED_IMAGE("drawImage", BufferedImage.class, BufferedImageOp.class, int.class, int.class), + + /** Describes the drawGlyphVector method. */ + DRAW_GLYPH_VECTOR("drawGlyphVector", GlyphVector.class, float.class, float.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_DEST_SRC("drawImage", Image.class, int.class, int.class, int.class, int.class, + int.class, int.class, int.class, int.class, ImageObserver.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_DEST_SRC_COL("drawImage", Image.class, int.class, int.class, int.class, int.class, + int.class, int.class, int.class, int.class, Color.class, ImageObserver.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_TRANSFORM("drawImage", Image.class, AffineTransform.class, ImageObserver.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_XY("drawImage", Image.class, int.class, int.class, ImageObserver.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_XY_COL("drawImage", Image.class, int.class, int.class, Color.class, ImageObserver.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_XYWH("drawImage", Image.class, int.class, int.class, int.class, int.class, ImageObserver.class), + + /** Describes the drawImage method. */ + DRAW_IMAGE_XYWH_COL("drawImage", Image.class, int.class, int.class, int.class, int.class, Color.class, ImageObserver.class), + + /** Describes the drawLine method. */ + DRAW_LINE("drawLine", int.class, int.class, int.class, int.class), + + /** Describes the drawOval method. */ + DRAW_OVAL("drawOval", int.class, int.class, int.class, int.class), + + /** Describes the drawPolygon method. */ + DRAW_POLYGON("drawPolygon", int[].class, int[].class, int.class), + + /** Describes the drawPolyline method. */ + DRAW_POLYLINE("drawPolyline", int[].class, int[].class, int.class), + + /** Describes the drawRenderableImage method. */ + DRAW_RENDERABLE_IMAGE("drawRenderableImage", RenderableImage.class, AffineTransform.class), + + /** Describes the drawRenderedImage method. */ + DRAW_RENDERED_IMAGE("drawRenderedImage", RenderedImage.class, AffineTransform.class), + + /** Describes the drawRoundRect method. */ + DRAW_ROUND_RECT("drawRoundRect", int.class, int.class, int.class, int.class, int.class, int.class), + + /** Describes the draw method. */ + DRAW_SHAPE("draw", Shape.class), + + /** Describes the drawString method. */ + DRAW_STRING_XY("drawString", String.class, float.class, float.class), + + /** Describes the drawString method. */ + DRAW_STRING_ITER_XY("drawString", AttributedCharacterIterator.class, float.class, float.class), + + /** Describes the fill method. */ + FILL("fill", Shape.class), + + /** Describes the fillArc method. */ + FILL_ARC("fillArc", int.class, int.class, int.class, int.class, int.class, int.class), + + /** Describes the fillOval method. */ + FILL_OVAL("fillOval", int.class, int.class, int.class, int.class), + + /** Describes the fillPolygon method. */ + FILL_POLYGON("fillPolygon", int[].class, int[].class, int.class), + + /** Describes the fillRect method. */ + FILL_RECT("fillRect", int.class, int.class, int.class, int.class), + + /** Describes the fillRoundRect method. */ + FILL_ROUND_RECT("fillRoundRect", int.class, int.class, int.class, int.class, int.class, int.class); + + + private String methodName; + private Class[] paramTypes; + + private OpType(String methodName, Class ...types) { + this.methodName = methodName; + this.paramTypes = new Class[types.length]; + System.arraycopy(types, 0, this.paramTypes, 0, types.length); + } + + /** + * Gets the method name. + * @return method name + */ + public String getMethodName() { + return methodName; + } + + /** + * Gets the full method name. + * @return full name + */ + public String getFullMethodName() { + StringBuilder sb = new StringBuilder(); + sb.append(methodName); + sb.append("("); + if (paramTypes.length > 0) { + sb.append(paramTypes[0].getSimpleName()); + for (int i = 1; i < paramTypes.length; i++) { + sb.append(", "); + sb.append(paramTypes[i].getSimpleName()); + } + } + sb.append(")"); + return sb.toString(); + } + + /** + * Gets the number of method arguments + * @return number of arguments + */ + public int getNumArgs() { + return paramTypes.length; + } + + /** + * Gets the types of the arguments. + * @return argument types + */ + public Class[] getArgTypes() { + Class[] copy = new Class[ paramTypes.length]; + System.arraycopy(paramTypes, 0, copy, 0, paramTypes.length); + return copy; + } + } + + /** + * Creates an instance for the given target image + * + * @param targetImage the image to be drawn into + */ + DiskMemImageGraphics(DiskMemImage targetImage) { + this.targetImage = targetImage; + this.renderingHints = new RenderingHints(null); + setColorModel(); + setProperties(); + setGraphicsParams(); + } + + @Override + public void draw(Shape s) { + doDraw(OpType.DRAW_SHAPE, correctForStroke(s.getBounds2D()), s); + } + + @Override + public boolean drawImage(Image img, AffineTransform xform, ImageObserver obs) { + Rectangle2D bounds = new Rectangle(0, 0, img.getWidth(obs), img.getHeight(obs)); + Rectangle2D trBounds = xform.createTransformedShape(bounds).getBounds2D(); + + return doDraw(OpType.DRAW_IMAGE_TRANSFORM, trBounds, img, xform, obs); + } + + @Override + public void drawImage(BufferedImage img, BufferedImageOp op, int x, int y) { + Rectangle2D bounds = op.getBounds2D(img); + doDraw(OpType.DRAW_BUFFERED_IMAGE, bounds, img, op, x, y); + } + + @Override + public void drawRenderedImage(RenderedImage img, AffineTransform xform) { + Rectangle2D bounds = new Rectangle(img.getMinX(), img.getMinY(), + img.getWidth(), img.getHeight()); + + Rectangle2D trBounds = xform.createTransformedShape(bounds).getBounds2D(); + + doDraw(OpType.DRAW_RENDERED_IMAGE, trBounds, img, xform); + } + + @Override + public void drawRenderableImage(RenderableImage img, AffineTransform xform) { + Rectangle2D bounds = new Rectangle2D.Float(img.getMinX(), img.getMinY(), + img.getWidth(), img.getHeight()); + + Rectangle2D trBounds = xform.createTransformedShape(bounds).getBounds2D(); + + doDraw(OpType.DRAW_RENDERABLE_IMAGE, trBounds, img, xform); + } + + @Override + public void drawString(String str, int x, int y) { + drawString( str, (float)x, (float)y ); + } + + @Override + public void drawString(String str, float x, float y) { + Rectangle2D bounds = getFontMetrics().getStringBounds(str, this); + bounds.setRect(x, + y - bounds.getHeight() + 1, + bounds.getWidth(), + bounds.getHeight() ); + + doDraw(OpType.DRAW_STRING_XY, bounds, str, x, y); + } + + @Override + public void drawString(AttributedCharacterIterator iter, int x, int y) { + drawString( iter, (float)x, (float)y ); + } + + @Override + public void drawString(AttributedCharacterIterator iter, float x, float y) { + Rectangle2D bounds = getFontMetrics().getStringBounds( + iter, iter.getBeginIndex(), iter.getEndIndex(), this); + + bounds.setRect(x, + y - bounds.getHeight() + 1, + bounds.getWidth(), + bounds.getHeight() ); + + doDraw(OpType.DRAW_STRING_ITER_XY, bounds, iter, x, y); + } + + @Override + public void drawGlyphVector(GlyphVector g, float x, float y) { + Rectangle2D bounds = g.getVisualBounds(); + + // expand the rectangle by a single pixel width to allow for + // rasterizing round-off errors + // (see apidocs for Graphics2D.drawGlyphVector) + bounds.setFrame(bounds.getMinX() - 1, bounds.getMinY() - 1, + bounds.getWidth() + 2, bounds.getHeight() + 2); + + doDraw(OpType.DRAW_GLYPH_VECTOR, bounds, g, x, y); + } + + @Override + public void fill(Shape s) { + doDraw(OpType.FILL, s.getBounds2D(), s); + } + + @Override + public boolean hit(Rectangle rect, Shape s, boolean onStroke) { + Graphics2D gr = getProxy(); + copyGraphicsParams(gr); + boolean rtnVal = gr.hit(rect, s, onStroke); + gr.dispose(); + return rtnVal; + } + + @Override + public GraphicsConfiguration getDeviceConfiguration() { + Graphics2D gr = getProxy(); + copyGraphicsParams(gr); + GraphicsConfiguration gc = gr.getDeviceConfiguration(); + gr.dispose(); + return gc; + } + + @Override + public void setComposite(Composite comp) { + composite = comp; + } + + @Override + public void setPaint(Paint p) { + paint = p; + } + + @Override + public void setStroke(Stroke s) { + stroke = s; + } + + @Override + public void setRenderingHint(Key hintKey, Object hintValue) { + renderingHints.put(hintKey, hintValue); + } + + @Override + public Object getRenderingHint(Key hintKey) { + return renderingHints.get(hintKey); + } + + @Override + public void setRenderingHints(Map hints) { + renderingHints = new RenderingHints(null); + renderingHints.putAll(hints); + } + + @Override + public void addRenderingHints(Map hints) { + renderingHints.putAll(hints); + } + + @Override + public RenderingHints getRenderingHints() { + return renderingHints; + } + + @Override + public void translate(int x, int y) { + origin.setLocation(x, y); + transform.translate( (double)x, (double)y ); + } + + @Override + public void translate(double tx, double ty) { + transform.translate(tx, ty); + } + + @Override + public void rotate(double theta) { + transform.rotate(theta); + } + + @Override + public void rotate(double theta, double x, double y) { + transform.rotate(theta, x, y); + } + + @Override + public void scale(double sx, double sy) { + transform.scale(sx, sy); + } + + @Override + public void shear(double shx, double shy) { + transform.shear(shx, shy); + } + + @Override + public void transform(AffineTransform Tx) { + transform.concatenate(Tx); + } + + @Override + public void setTransform(AffineTransform Tx) { + transform = Tx; + } + + @Override + public AffineTransform getTransform() { + return transform; + } + + @Override + public Paint getPaint() { + return paint; + } + + @Override + public Composite getComposite() { + return composite; + } + + @Override + public void setBackground(Color color) { + background = color; + } + + @Override + public Color getBackground() { + return background; + } + + @Override + public Stroke getStroke() { + return stroke; + } + + @Override + public void clip(Shape s) { + if(clip == null) { + clip = s; + } else { + Area clipArea = (clip instanceof Area ? (Area)clip : new Area(clip)); + clipArea.intersect(s instanceof Area ? (Area)s : new Area(s)); + clip = clipArea; + } + } + + @Override + public FontRenderContext getFontRenderContext() { + Graphics2D gr = getProxy(); + copyGraphicsParams(gr); + FontRenderContext frc = gr.getFontRenderContext(); + gr.dispose(); + return frc; + } + + /** + * Returns a copy of this object with its current settings + * @return a new instance of this class + */ + @Override + public Graphics create() { + DiskMemImageGraphics gr = new DiskMemImageGraphics(targetImage); + copyGraphicsParams(gr); + return gr; + } + + @Override + public Color getColor() { + return color; + } + + @Override + public void setColor(Color c) { + color = c; + } + + @Override + public void setPaintMode() { + paintMode = PaintMode.PAINT; + } + + @Override + public void setXORMode(Color color) { + paintMode = PaintMode.XOR; + XORColor = color; + } + + @Override + public Font getFont() { + return font; + } + + @Override + public void setFont(Font font) { + if (font != null) { + this.font = font; + } + } + + @Override + public FontMetrics getFontMetrics(Font f) { + Graphics2D gr = getProxy(); + copyGraphicsParams(gr); + FontMetrics fm = gr.getFontMetrics(f); + gr.dispose(); + return fm; + } + + @Override + public Rectangle getClipBounds() { + return clip.getBounds(); + } + + @Override + public void clipRect(int x, int y, int width, int height) { + clip(new Rectangle(x, y, width, height)); + } + + @Override + public void setClip(int x, int y, int width, int height) { + setClip(new Rectangle(x, y, width, height)); + } + + @Override + public Shape getClip() { + return clip; + } + + @Override + public void setClip(Shape clip) { + this.clip = clip; + } + + @Override + public void copyArea(int x, int y, int width, int height, int dx, int dy) { + doDraw(OpType.COPY_AREA, + new Rectangle(x + dx, y + dy, width, height), + x, y, width, height, dx, dy); + } + + @Override + public void drawLine(int x1, int y1, int x2, int y2) { + Rectangle2D bounds = new Rectangle(); + bounds.setFrameFromDiagonal(x1, y1, x2, y2); + doDraw(OpType.DRAW_LINE, correctForStroke(bounds), x1, y1, x2, y2); + } + + @Override + public void fillRect(int x, int y, int width, int height) { + doDraw(OpType.FILL_RECT, new Rectangle(x, y, width, height), x, y, width, height); + } + + @Override + public void clearRect(int x, int y, int width, int height) { + doDraw(OpType.CLEAR_RECT, new Rectangle(x, y, width, height), x, y, width, height); + } + + @Override + public void drawRoundRect(int x, int y, int width, int height, int arcWidth, int arcHeight) { + Rectangle2D bounds = new Rectangle( + x - arcWidth, y - arcHeight, + width + 2 * arcWidth, height + 2 * arcHeight); + + doDraw(OpType.DRAW_ROUND_RECT, correctForStroke(bounds), + x, y, width, height, arcWidth, arcHeight); + } + + @Override + public void fillRoundRect(int x, int y, int width, int height, int arcWidth, int arcHeight) { + Rectangle2D bounds = new Rectangle( + x - arcWidth, y - arcHeight, + width + 2 * arcWidth, height + 2 * arcHeight); + + doDraw(OpType.FILL_ROUND_RECT, bounds, x, y, width, height, arcWidth, arcHeight); + } + + @Override + public void drawOval(int x, int y, int width, int height) { + Rectangle2D bounds = new Rectangle(x, y, width, height); + doDraw(OpType.DRAW_OVAL, correctForStroke(bounds), x, y, width, height); + } + + @Override + public void fillOval(int x, int y, int width, int height) { + Rectangle2D bounds = new Rectangle(x, y, width, height); + doDraw(OpType.FILL_OVAL, bounds, x, y, width, height); + } + + @Override + public void drawArc(int x, int y, int width, int height, int startAngle, int arcAngle) { + Rectangle2D bounds = new Rectangle(x, y, width, height); + doDraw(OpType.DRAW_ARC, correctForStroke(bounds), x, y, width, height, startAngle, arcAngle); + } + + @Override + public void fillArc(int x, int y, int width, int height, int startAngle, int arcAngle) { + Rectangle2D bounds = new Rectangle(x, y, width, height); + doDraw(OpType.FILL_ARC, bounds, x, y, width, height, startAngle, arcAngle); + } + + @Override + public void drawPolyline(int[] xPoints, int[] yPoints, int nPoints) { + doDraw(OpType.DRAW_POLYLINE, + correctForStroke(getPolyBounds(xPoints, yPoints, nPoints)), + xPoints, yPoints, nPoints); + } + + @Override + public void drawPolygon(int[] xPoints, int[] yPoints, int nPoints) { + doDraw(OpType.DRAW_POLYGON, + correctForStroke(getPolyBounds(xPoints, yPoints, nPoints)), + xPoints, yPoints, nPoints); + } + + @Override + public void fillPolygon(int[] xPoints, int[] yPoints, int nPoints) { + doDraw(OpType.FILL_POLYGON, + getPolyBounds(xPoints, yPoints, nPoints), + xPoints, yPoints, nPoints); + } + + @Override + public boolean drawImage(Image img, int x, int y, ImageObserver obs) { + Rectangle2D bounds = new Rectangle(x, y, img.getWidth(obs), img.getHeight(obs)); + return doDraw(OpType.DRAW_IMAGE_XY, bounds, img, x, y, obs); + } + + @Override + public boolean drawImage(Image img, int x, int y, int width, int height, ImageObserver obs) { + Rectangle2D bounds = new Rectangle(x, y, width, height); + return doDraw(OpType.DRAW_IMAGE_XYWH, bounds, img, x, y, width, height, obs); + } + + @Override + public boolean drawImage(Image img, int x, int y, Color bgcolor, ImageObserver obs) { + Rectangle2D bounds = new Rectangle(x, y, img.getWidth(obs), img.getHeight(obs)); + return doDraw(OpType.DRAW_IMAGE_XY_COL, bounds, img, x, y, bgcolor, obs); + } + + @Override + public boolean drawImage(Image img, + int x, int y, int width, int height, + Color bgcolor, ImageObserver obs) { + + Rectangle2D bounds = new Rectangle(x, y, width, height); + return doDraw(OpType.DRAW_IMAGE_XYWH_COL, bounds, img, x, y, width, height, bgcolor, obs); + } + + @Override + public boolean drawImage(Image img, + int dx1, int dy1, int dx2, int dy2, + int sx1, int sy1, int sx2, int sy2, + ImageObserver obs) { + + Rectangle2D bounds = new Rectangle(dx1, dy1, dx2-dx1+1, dy2-dy1+1); + return doDraw(OpType.DRAW_IMAGE_DEST_SRC, bounds, img, dx1, dy1, dx2, dy2, sx1, sy1, sx2, sy2, obs); + } + + @Override + public boolean drawImage(Image img, + int dx1, int dy1, int dx2, int dy2, + int sx1, int sy1, int sx2, int sy2, + Color bgcolor, ImageObserver obs) { + + Rectangle2D bounds = new Rectangle(dx1, dy1, dx2-dx1+1, dy2-dy1+1); + return doDraw(OpType.DRAW_IMAGE_DEST_SRC_COL, bounds, img, + dx1, dy1, dx2, dy2, sx1, sy1, sx2, sy2, + bgcolor, obs); + } + + @Override + public void dispose() { + /* + * No need to do anything here + */ + } + + /** + * Performs the graphics operation by partitioning the work across the image's + * tiles and using Graphics2D routines to draw into each tile. + * + * @param opType the type of operation + * @param bounds bounds of the element to be drawn + * @param args a variable length list of arguments for the operation + */ + private boolean doDraw(OpType opType, Rectangle2D bounds, Object ...args) { + Method method = null; + boolean rtnVal = false; + + try { + method = Graphics2D.class.getMethod(opType.getMethodName(), opType.getArgTypes()); + + } catch (NoSuchMethodException nsmEx) { + // programmer error :-( + throw new RuntimeException("No such method: " + opType.getFullMethodName()); + } + + int minTileX = Math.max(targetImage.XToTileX((int)bounds.getMinX()), + targetImage.getMinTileX()); + + int maxTileX = Math.min(targetImage.XToTileX((int)(bounds.getMaxX() + 0.5)), + targetImage.getMaxTileX()); + + int minTileY = Math.max(targetImage.YToTileY((int)bounds.getMinY()), + targetImage.getMinTileY()); + + int maxTileY = Math.min(targetImage.YToTileY((int)(bounds.getMaxY() + 0.5)), + targetImage.getMaxTileY()); + + for (int tileY = minTileY; tileY <= maxTileY; tileY++) { + int minY = targetImage.tileYToY(tileY); + + for (int tileX = minTileX; tileX <= maxTileX; tileX++) { + int minX = targetImage.tileXToX(tileX); + + WritableRaster tile = targetImage.getWritableTile(tileX, tileY); + + // create a live-copy of the tile with the upper-left corner + // translated to 0,0 + WritableRaster copy = tile.createWritableTranslatedChild(0, 0); + + BufferedImage bufImg = new BufferedImage( + colorModel, + copy, + colorModel.isAlphaPremultiplied(), + properties); + + Graphics2D gr = bufImg.createGraphics(); + + // Note: we use the version of copyGraphicsParams taking a + // Point arg used to adjust the clip area before copying it + // into the graphics object + copyGraphicsParams(gr, new Point(minX, minY)); + + try { + Point2D p2d = gr.getTransform().transform(new Point2D.Double(0, 0), null); + Point p = new Point((int)p2d.getX() - minX, (int)p2d.getY() - minY); + p2d = gr.getTransform().inverseTransform(p, null); + gr.translate(p2d.getX(), p2d.getY()); + + } catch(NoninvertibleTransformException nte) { + // TODO replace this with decent error handling + throw new RuntimeException(nte); + } + + try { + Object oRtnVal = method.invoke(gr, args); + if(oRtnVal != null && oRtnVal.getClass() == boolean.class) { + rtnVal = ((Boolean)oRtnVal).booleanValue(); + } + + } catch(Exception ex) { + // TODO replace this with decent error handling + throw new RuntimeException(ex); + } + + gr.dispose(); + targetImage.releaseWritableTile(tileX, tileY); + } + } + + return rtnVal; + } + + /** + * Takes a bounding rectangle calculated by + * one of the drawing methods and expands it, if necessary, to + * account for the current Stroke width. + *

+ * This correction appears to be missing from JAI's TiledImageGraphics + * class. + * + * @param rect input bounds + * @return expanded bounds as a new Rectangle2D object; or the + * input Rectangle2D if no stroke is set + */ + private Rectangle2D correctForStroke(Rectangle2D rect) { + if (stroke != null) { + Shape shp = stroke.createStrokedShape(rect); + return shp.getBounds2D(); + } else { + return rect; + } + } + + /** + * Gets the bounding rectangle of the given vertices. + * + * @param xPoints X ordinates + * @param yPoints Y ordinates + * @param nPoints number of vertices + * @return the bounding rectangle + */ + private Rectangle2D getPolyBounds(int[] xPoints, int[] yPoints, int nPoints) { + Rectangle bounds = new Rectangle(); + + if (nPoints > 0) { + int minX = xPoints[0]; + int maxX = minX; + int minY = yPoints[0]; + int maxY = minY; + + for (int i = 1; i < nPoints; i++) { + if (xPoints[i] < minX) { + minX = xPoints[i]; + } else if (xPoints[i] > maxX) { + maxX = xPoints[i]; + } + + if (yPoints[i] < minY) { + minY = yPoints[i]; + } else if (yPoints[i] > maxY) { + maxY = yPoints[i]; + } + } + + bounds.setBounds(minX, minY, maxX - minX + 1, maxY - minY + 1); + } + + return bounds; + } + + /** + * Attempts to retrieve or create a ColorModel for the target + * image. + * + * @throws UnsupportedOperationException if a compatible + * ColorModel is not found + */ + private void setColorModel() { + assert(targetImage != null); + + colorModel = targetImage.getColorModel(); + + if (colorModel == null) { + SampleModel sm = targetImage.getSampleModel(); + colorModel = PlanarImage.createColorModel(sm); + + if (colorModel == null) { + // try simple default + if (ColorModel.getRGBdefault().isCompatibleSampleModel(sm)) { + colorModel = ColorModel.getRGBdefault(); + + } else { + // admit defeat + throw new UnsupportedOperationException( + "Failed to get or construct a ColorModel for the image"); + } + } + } + } + + /** + * Retrieves any properties set for the target image. + */ + private void setProperties() { + assert(targetImage != null); + + properties = new Hashtable(); + String[] propertyNames = targetImage.getPropertyNames(); + if (propertyNames != null) { + for (String name : propertyNames) { + properties.put(name, targetImage.getProperty(name)); + } + } + + // TODO: set rendering hints + } + + /** + * Creates a Graphics2D instance based on the first tile of the + * target image and uses its state to set the graphics parameters + * of this instance. + */ + private void setGraphicsParams() { + assert(targetImage != null); + assert(colorModel != null); + assert(properties != null); + + Graphics2D gr = getProxy(); + + origin = new Point(0, 0); + clip = targetImage.getBounds(); + color = gr.getColor(); + font = gr.getFont(); + + paintMode = PaintMode.PAINT; + XORColor = null; + + background = gr.getBackground(); + composite = gr.getComposite(); + paint = null; + stroke = gr.getStroke(); + transform = gr.getTransform(); + + gr.dispose(); + } + + /** + * Copies the current graphics parameters into the given Graphics2D + * object. + * + * @param gr a Graphics2D object + */ + private void copyGraphicsParams(Graphics2D gr) { + copyGraphicsParams(gr, null); + } + + /** + * Copies the current graphics parameters into the given Graphics2D + * object. If {@code workingOrigin} is non-null it is used to translate the + * clip area before copying it across. + * + * @param gr a Graphics2D object + */ + private void copyGraphicsParams(Graphics2D gr, Point workingOrigin) { + gr.translate(origin.x, origin.y); + gr.setColor(getColor()); + + if (workingOrigin == null) { + gr.setClip(clip); + } else { + AffineTransform tr = AffineTransform.getTranslateInstance( + -workingOrigin.x, -workingOrigin.y); + Shape trclip = tr.createTransformedShape(clip); + gr.setClip(trclip); + } + + if(paintMode == PaintMode.PAINT) { + gr.setPaintMode(); + } else if (XORColor != null) { + gr.setXORMode(XORColor); + } + + gr.setFont(font); + + // java.awt.Graphics2D state + gr.setBackground(background); + gr.setComposite(composite); + if(paint != null) { + gr.setPaint(paint); + } + if (renderingHints != null) { + gr.setRenderingHints(renderingHints); + } + gr.setStroke(stroke); + gr.setTransform(transform); + } + + /** + * Helper method for other methods that need an instantiated + * Graphics2D object that is 'representative' of the target image. + * + * @return a new Graphics2D instance + */ + private Graphics2D getProxy() { + Raster tile = targetImage.getTile(targetImage.getMinTileX(), targetImage.getMinTileY()); + WritableRaster tiny = tile.createCompatibleWritableRaster(1, 1); + + BufferedImage img = new BufferedImage( + colorModel, tiny, colorModel.isAlphaPremultiplied(), properties); + + return img.createGraphics(); + } +} diff --git a/klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java b/klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java new file mode 100644 index 000000000..5d12d14be --- /dev/null +++ b/klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2009, Michael Bedward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jaitools.tiledimage; + +/** + * Exception thrown when a given pixel / band location is outside an + * image's bounds + * + * @author Michael Bedward + * @since 1.0 + * @version $Id$ + */ +public class PixelOutsideImageException extends IndexOutOfBoundsException { + + /** + * Creates a new instance. + * + * @param x pixel x coordinate + * @param y pixel y coordinate + * @param b band index + */ + PixelOutsideImageException(int x, int y, int b) { + super(String.format("Pixel location x=%d y=%d band=%d is outside image bounds", x, y, b)); + } + +} From 4ab0b1094739ca1db9f0daae0204dabd445d5a5a Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 16 Feb 2026 10:23:30 +0100 Subject: [PATCH 12/74] Remove code with DiskMemImage and the dependencies It was dead code --- .../geospace/utils/GeotoolsUtils.java | 144 +- .../java/org/jaitools/CollectionFactory.java | 195 --- .../org/jaitools/DaemonThreadFactory.java | 113 -- .../jaitools/tilecache/BasicCacheVisitor.java | 205 --- .../tilecache/DiskCacheFailedException.java | 52 - .../jaitools/tilecache/DiskCachedTile.java | 700 --------- .../tilecache/DiskMemCacheException.java | 53 - .../jaitools/tilecache/DiskMemTileCache.java | 1277 ----------------- .../tilecache/DiskMemTileCacheVisitor.java | 49 - .../tilecache/TileAccessTimeComparator.java | 52 - .../tilecache/TileNotResidentException.java | 45 - .../org/jaitools/tiledimage/DiskMemImage.java | 818 ----------- .../tiledimage/DiskMemImageGraphics.java | 1027 ------------- .../PixelOutsideImageException.java | 49 - 14 files changed, 9 insertions(+), 4770 deletions(-) delete mode 100644 klab.engine/src/main/java/org/jaitools/CollectionFactory.java delete mode 100644 klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java delete mode 100644 klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java index 9f006b412..6f2bcb831 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java @@ -26,27 +26,28 @@ import org.eclipse.imagen.RenderedOp; import org.eclipse.imagen.iterator.RandomIter; import org.eclipse.imagen.iterator.RandomIterFactory; -import org.eclipse.imagen.iterator.RectIterFactory; -import org.eclipse.imagen.iterator.WritableRectIter; - -import org.eclipse.lsp4j.AbstractTextDocumentRegistrationAndWorkDoneProgressOptions; +import org.geotools.api.filter.expression.Literal; +import org.geotools.api.metadata.Identifier; +import org.geotools.api.metadata.citation.Citation; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValueGroup; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.style.ColorMapEntry; +import org.geotools.api.style.RasterSymbolizer; import org.geotools.coverage.Category; import org.geotools.coverage.CoverageFactoryFinder; import org.geotools.coverage.GridSampleDimension; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.GridCoverageFactory; -import org.geotools.coverage.grid.io.AbstractGridCoverageWriter; import org.geotools.coverage.grid.io.AbstractGridFormat; import org.geotools.coverage.grid.io.imageio.GeoToolsWriteParams; import org.geotools.gce.geotiff.GeoTiffFormat; import org.geotools.gce.geotiff.GeoTiffWriteParams; import org.geotools.gce.geotiff.GeoTiffWriter; import org.geotools.geometry.jts.ReferencedEnvelope; -import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.metadata.iso.citation.Citations; import org.geotools.referencing.CRS; -import org.geotools.api.style.ColorMapEntry; -import org.geotools.api.style.RasterSymbolizer; import org.geotools.swing.data.JFileDataStoreChooser; import org.geotools.util.factory.Hints; import org.hortonmachine.gears.utils.files.FileUtilities; @@ -60,7 +61,6 @@ import org.integratedmodelling.klab.api.observations.scale.space.ISpace; import org.integratedmodelling.klab.api.runtime.IContextualizationScope; import org.integratedmodelling.klab.api.services.IConfigurationService; -import org.integratedmodelling.klab.common.Geometry; import org.integratedmodelling.klab.components.geospace.extents.Grid; import org.integratedmodelling.klab.components.geospace.extents.Projection; import org.integratedmodelling.klab.components.geospace.extents.Space; @@ -73,14 +73,6 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.utils.Pair; -import org.jaitools.tiledimage.DiskMemImage; -import org.geotools.api.filter.expression.Literal; -import org.geotools.api.metadata.Identifier; -import org.geotools.api.metadata.citation.Citation; -import org.geotools.api.parameter.GeneralParameterValue; -import org.geotools.api.parameter.ParameterValueGroup; -import org.geotools.api.referencing.FactoryException; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; public enum GeotoolsUtils { @@ -680,124 +672,6 @@ public static WritableRaster createWritableRaster(int width, int height, Class= MIN_SIZE_NEED_TILING) { - isTiled = true; - - // This implementation supposes that tileWidth is - // equal to the width - // of the whole image - tileWidth = width; - - // actually (need improvements) tileHeight is given by - // the default tile size divided by the tileWidth - // multiplied by the - // sample size (in byte) - tileHeight = DEFAULT_TILE_SIZE / (tileWidth * sampleSizeByte); - - // if computed tileHeight is zero, it is setted to 1 - // as precaution - if (tileHeight < 1) { - tileHeight = 1; - } - - } else { - // If no Tiling needed, I set the tile sizes equal to the image - // sizes - tileWidth = width; - tileHeight = height; - } - - ReferencedEnvelope envelope = ReferencedEnvelope.rect(0, 0, width, height, crs); - SampleModel sampleModel = new ComponentSampleModel(DataBuffer.TYPE_FLOAT, tileWidth, tileHeight, 1, tileWidth, - new int[]{0}); - DiskMemImage img = new DiskMemImage(width, height, sampleModel); - - WritableRectIter iter = RectIterFactory.createWritable(img, null); - do { - int x = 0; - do { - iter.setSample(x / tileWidth); - } while(!iter.nextPixelDone()); - } while(!iter.nextLineDone()); - - GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); - - GridCoverage2D gc = factory.create("bigtif", img, envelope); - - GeoTiffFormat fmt = new GeoTiffFormat(); - // getting the write parameters - final GeoTiffWriteParams wp = new GeoTiffWriteParams(); - - // setting compression to Deflate - wp.setCompressionMode(GeoTiffWriteParams.MODE_EXPLICIT); - wp.setCompressionType("Deflate"); - wp.setCompressionQuality(0.75F); - - // setting the tile size to 256X256 - wp.setTilingMode(GeoToolsWriteParams.MODE_EXPLICIT); - wp.setTiling(256, 256); - - // setting the write parameters for this geotiff - final ParameterValueGroup[] params = {fmt.getWriteParameters()}; - - params[0].parameter(AbstractGridFormat.GEOTOOLS_WRITE_PARAMS.getName().toString()).setValue(wp); - - AbstractGridCoverageWriter writer = new GeoTiffWriter(file); - writer.write(gc/* .view(ViewType.GEOPHYSICS) */, params); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - /** * Dump one or more states to rasters in the klab configuration folder. * diff --git a/klab.engine/src/main/java/org/jaitools/CollectionFactory.java b/klab.engine/src/main/java/org/jaitools/CollectionFactory.java deleted file mode 100644 index 4ef9eaeb8..000000000 --- a/klab.engine/src/main/java/org/jaitools/CollectionFactory.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.Stack; -import java.util.TreeMap; -import java.util.TreeSet; - -/** - * Convenience methods to create generic collections - * following the DRY (Don't Repeat Yourself) principle. - * Examples of use: - * 


- * // simple
- * List<Integer> list1 = CollectionFactory.list();
- * 
- * // nested
- * List<List<String>> list2 = CollectionFactory.list();
- * 
- * // a multi-map
- * Map<String, List<Integer>> multiMap = CollectionFactory.map();
- *
- * - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class CollectionFactory { - /** - * Returns a new {@code Map} object. The returned - * object is a {@link HashMap}. - *

- * Example of use: - * 


-     * Map<Foo, Bar> myMap = CollectionFactory.map();
-     *
- * - * @param key type - * @param value type - * @return a new Map<K,V> instance - */ - public static Map map() { - return new HashMap(); - } - - /** - * Returns a new {@code Map} object that maintains insertion - * order. The returned object is a {@link LinkedHashMap}. - *

- * Example of use: - * 


-     * Map<Foo, Bar> myMap = CollectionFactory.orderedMap();
-     *
- * - * @param key type - * @param value type - * @return a new Map<K,V> instance - */ - public static Map orderedMap() { - return new LinkedHashMap(); - } - - /** - * Returns a new {@code SortedMap} object. Key type {@code K} must - * implement {@linkplain Comparable}. The returned object is a - * {@link TreeMap}. - *

- * Example of use: - * 


-     * Map<Foo, Bar> myMap = CollectionFactory.sortedMap();
-     *
- * - * @param key type - * @param value type - * @return a new SortedMap<K,V> instance - */ - public static SortedMap sortedMap() { - return new TreeMap(); - } - - /** - * Returns a new {@code List} object. The returned object is - * an {@link ArrayList}. - *

- * Example of use: - * 


-     * List<Foo> myList = CollectionFactory.list();
-     *
- * - * @param element type - * @return a new List<T> instance - */ - public static List list() { - return new ArrayList(); - } - - /** - * Returns a new {@code Stack} object. - *

- * Example of use: - * 


-     * Stack<Foo> myStack = CollectionFactory.stack();
-     *
- * - * @param element type - * @return a new Stack<T> instance - */ - public static Stack stack() { - return new Stack(); - } - - /** - * Returns a new {@code Set} instance. The returned object is - * a {@link HashSet}. - *

- * Example of use: - * 


-     * Set<Foo> mySet = CollectionFactory.set();
-     *
- * - * @param element type - * @return a new HashSet<T> instance - */ - public static Set set() { - return new HashSet(); - } - - /** - * Returns a new {@code Set} instance that maintains - * the insertion order of its elements. - *

- * Example of use: - * 


-     * Set<Foo> mySet = CollectionFactory.set();
-     *
- * - * @param element type - * @return a new LinkedHashSet<T> instance - */ - public static Set orderedSet() { - return new LinkedHashSet(); - } - - /** - * Returns a new {@code SortedSet} instance. The type {@code T} - * must implement {@linkplain Comparable}. The returned object - * is a {@link TreeSet}. - *

- * Example of use: - * 


-     * Set<MyObject> foo = CollectionFactory.sortedSet();
-     *
- * - * @param element type - * @return a new TreeSet<T> instance - */ - public static SortedSet sortedSet() { - return new TreeSet(); - } - - -} diff --git a/klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java b/klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java deleted file mode 100644 index 1b020aa26..000000000 --- a/klab.engine/src/main/java/org/jaitools/DaemonThreadFactory.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2011, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools; - -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Simple ThreadFactory implementation to supply daemon threads with - * specified priority. Used by JAITools classes that run polling services - * on background threads to avoid blocking application exit. - * - * @author Michael Bedward - * @since 1.1 - * @version $Id$ - */ -public class DaemonThreadFactory implements ThreadFactory { - - private static final String DEFAULT_ROOT_NAME = "daemon-"; - - private final ReentrantLock lock = new ReentrantLock(); - private static final AtomicInteger threadCount = new AtomicInteger(0); - private final int priority; - private final String rootName; - - - /** - * Creates a new factory which will supply daemon threads having - * normal priority. - */ - public DaemonThreadFactory() { - this(Thread.NORM_PRIORITY, DEFAULT_ROOT_NAME); - } - - /** - * Creates a new factory which will supply daemon threads to run - * at the specified priority. - * - * @param priority thread priority - */ - public DaemonThreadFactory(int priority) { - this(priority, DEFAULT_ROOT_NAME); - } - - - /** - * Creates a new factory which will supply daemon threads to run - * at the specified priority. Threads will be named {@code rootName-n} - * where {@code n} is the count of threads produced by all instances - * of this class. - * - * @param priority thread priority - * @param rootName root name to label threads - */ - public DaemonThreadFactory(int priority, String rootName) { - String s = rootName == null ? "" : rootName.trim(); - if (s.length() == 0) { - this.rootName = DEFAULT_ROOT_NAME; - } else if (s.endsWith("-")) { - this.rootName = s; - } else { - this.rootName = s + "-"; - } - - this.priority = Math.min(Thread.MAX_PRIORITY, Math.max(Thread.MIN_PRIORITY, priority)); - } - - /** - * Creates a new daemon thread with name and priority assigned - * as per the values supplied when creating this thread factory. - * - * @param r target for the new thread - * - * @return new thread - */ - public Thread newThread(Runnable r) { - lock.lock(); - try { - Thread t = new Thread(r); - t.setDaemon(true); - t.setName("daemon-" + threadCount.getAndIncrement()); - t.setPriority(priority); - return t; - } finally { - lock.unlock(); - } - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java b/klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java deleted file mode 100644 index c34838840..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/BasicCacheVisitor.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.awt.image.RenderedImage; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -/** - * A basic visitor class for DiskMemTileCache that can be used - * to examine cache contents. - *

- * Example of use: - * 


- * // Query the cache for tiles that belong to a given image
- * // and that are currently resident in memory
- * DiskMemTileCache cache = ...
- * RenderedImage owner = ...
- *
- * Map filters = CollectionFactory.newMap();
- * filters.put(BasicCacheVisitor.Key.OWNER, owner);
- * filters.put(BasicCachevisitor.Key.RESIDENT, Boolean.TRUE);
- *
- * BasicCacheVisitor visitor = new BasicCacheVisitor();
- * visitor.setFilters(filters);
- * cache.accept(visitor);
- *
- * for (DiskCachedTile tile : visitor.getTiles()) {
- *     System.out.println(String.format("tile %d,%d", tile.getTileX(), tile.getTileY()));
- * }
- *
- * - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class BasicCacheVisitor implements DiskMemTileCacheVisitor { - - /** - * Defines keys to filter the tles visited - */ - public enum Key { - /** - * Filter on the image that owns the tile. The value - * provided for this key must be a RenderedImage. - */ - OWNER(RenderedImage.class), - - /** - * Filter on whether the tile is currently resident - * in memory. The value provided for this key must - * be of type Boolean. - */ - RESIDENT(Boolean.class); - - - private Class clazz; - - private Key(Class clazz) { - this.clazz = clazz; - } - - /** - * Gets the value class. - * - * @return the class - */ - public Class getValueClass() { - return clazz; - } - } - - private List tiles = new ArrayList(); - private Map filters = new HashMap(); - - - /** - * Set filter conditions. Any existing conditions are cleared. - * - * @param params a Map where each key is one of the constants - * defined by BasicCacheVisitor.Key and each value is the - * corresponding value to match tiles against - */ - public void setFilters(Map params) { - filters.clear(); - for (Entry e : params.entrySet()) { - addFilter(e.getKey(), e.getValue()); - } - } - - /** - * Set a single filter condition. Any existing conditions - * are cleared - * - * @param key one of the constants defined by BasicCacheVisitor.Key - * @param value corresponding value to match tiles against - */ - public void setFilter(Key key, Object value) { - filters.clear(); - addFilter(key, value); - } - - /** - * Clear all existing filter conditions - */ - public void clearFilters() { - filters.clear(); - } - - /** - * Returns an unmodifiable collection of tiles retrieved from - * the cache. - * - * @return tiles retrieved from the cache - */ - public Collection getTiles() { - return Collections.unmodifiableCollection(tiles); - } - - - /** - * This method is called by the cache for each tile in turn. Tiles that - * pass this visitor's filter conditions, if any, will be added to the - * visitor's tile collection. - * - * @param tile the cached tile being visited - * @param isResident set by the cache to indicate whether the tile is - * currently resident in memory - */ - public void visit(DiskCachedTile tile, boolean isResident) { - if (filters.isEmpty()) { - tiles.add(tile); - - } else { - boolean pass = true; - if (filters.containsKey(Key.OWNER)) { - if ( !(tile.getOwner().equals(filters.get(Key.OWNER))) ) { - pass = false; - } - } - - if (pass && filters.containsKey(Key.RESIDENT)) { - if ( ((Boolean)filters.get(Key.RESIDENT)).booleanValue() != isResident) { - pass = false; - } - } - - if (pass) { - tiles.add(tile); - } - } - } - - /** - * Clear collected tile data. Useful if an instance of this - * class makes multiple visits to a cache. - */ - public void clear() { - tiles.clear(); - } - - /** - * Helper method for setFilters and setFilter - * @param key - * @param value - */ - private void addFilter(Key key, Object value) { - if (key.getValueClass().isAssignableFrom(value.getClass())) { - filters.put(key, value); - } else { - throw new IllegalArgumentException( - String.format("Object of type %s cannot be used as value for %s", - value.getClass().getName(), key.toString())); - } - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java deleted file mode 100644 index 8ed3ec1df..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/DiskCacheFailedException.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.awt.image.RenderedImage; - -/** - * Exception thrown when by (@code DiskMemTileCache} when a an attempt - * to cache a tile to disk has failed. - * - * @see DiskMemTileCache - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class DiskCacheFailedException extends DiskMemCacheException { - - /** - * Creates a new instance. - * - * @param owner the owning image - * @param tileX tile X ordinate - * @param tileY tile Y ordinate - */ - public DiskCacheFailedException(RenderedImage owner, int tileX, int tileY) { - super(owner, tileX, tileY, "could not be written to disk"); - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java deleted file mode 100644 index 201c3876f..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/DiskCachedTile.java +++ /dev/null @@ -1,700 +0,0 @@ -/* - * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.awt.Point; -import java.awt.image.DataBuffer; -import java.awt.image.DataBufferByte; -import java.awt.image.DataBufferDouble; -import java.awt.image.DataBufferFloat; -import java.awt.image.DataBufferInt; -import java.awt.image.DataBufferShort; -import java.awt.image.DataBufferUShort; -import java.awt.image.Raster; -import java.awt.image.RenderedImage; -import java.awt.image.WritableRaster; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.imageio.ImageIO; -import javax.imageio.stream.ImageInputStream; -import javax.imageio.stream.ImageOutputStream; -import org.eclipse.imagen.CachedTile; - -/** - * A managed tile class for {@code DiskMemTileCache}. Represents an image tile - * that can be cached on disk and/or in memory. - * - * @see DiskMemTileCache - * - * @author Michael Bedward - * @author Simone Giannecchini, GeoSolutions SAS - * @since 1.0 - * @version $Id$ - */ -public final class DiskCachedTile implements CachedTile { - - private static final Logger LOGGER = Logger.getLogger("org.jaitools.tilecache"); - - /** - * Constants identifying tile actions. - */ - public enum TileAction { - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile has been added to the cache - */ - ACTION_ADDED(0, "added to cache"), - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile has been added to the cache and immediately loaded - * into memory - */ - ACTION_ADDED_RESIDENT(1, "added to cache and placed into memory"), - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile becomes resident in memory - */ - ACTION_RESIDENT(2, "placed into memory"), - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile is removed from memory - */ - ACTION_NON_RESIDENT(3, "removed from memory"), - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile is removed from the cache entirely - */ - ACTION_REMOVED(4, "removed from the cache"), - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile is accessed via the cache - */ - ACTION_ACCESSED(5, "accessed"), - - /** - * Value that will be returned by {@linkplain #getAction()} when - * the tile's raster has been garbage collected - */ - ACTION_GARBAGE_COLLECTED(6, "garbage collected"); - - /** - * Map for the reverse lookup facility - */ - private static final Map lookup; - static { - lookup = new HashMap(); - for (TileAction t : EnumSet.allOf(TileAction.class)) { - lookup.put(t.getAction(), t); - } - } - /** An int associated to this action.*/ - private final int action; - /** Description of the action. */ - private final String desc; - - /** - * Private constructor to have maximum control over the values we use for this action. - * @param action an int associated to this action for interoperability with {@link CachedTile} interface. - */ - private TileAction(final int action, final String desc) { - this.action = action; - this.desc = desc; - } - - /** - * Retrieves an int associated to this action for interoperability with {@link CachedTile} interface. - * @return an int associated to this action for interoperability with {@link CachedTile} interface. - */ - public int getAction() { - return action; - } - - /** - * Retrieves a description of this action - * @return a description of this action - */ - public String getDescription() { - return desc; - } - - /** - * Reverse lookup. - * @param actionValue integer value of the action - * @return the corresponding TileAction - */ - public static TileAction get(int actionValue) { - return lookup.get(actionValue); - } - - /** - * The default action. - * @return the default action. - */ - public static TileAction getDefault() { - return ACTION_ACCESSED; - } - } - - /** - * The prefix used for temporary cache files - * data - */ - public static final String FILE_PREFIX = "tile"; - - /** - * The suffix used for temporary cache files - */ - public static final String FILE_SUFFIX = ".tmp"; - - /** - * The folder used to store tiles that are being newly cached to disk. - * If {@code null} (the default) the system's default folder is used. - * It is safe (though not necessarily sensible) to change this property - * after some tiles have already been cached to disk. - */ - private static File cacheFolder = null; - private static final Object folderLock = new Object(); - - private final Object id; - private final WeakReference ownerRef; - private final int tileX; - private final int tileY; - private final Object tileCacheMetric; - private long timeStamp; - private final int numBanks; - private final int dataLen; - private final long memorySize; - private File file; - private final Point location; - private final boolean isWritable; - - private TileAction action =TileAction.getDefault(); - - /** - * Get the current cache folder. This is the folder in which - * newly created temporary files holding tile data will be created. - * - * @return a new {@code File} object for the current cache folder - * or {@code null} if the default system folder is being - * used (ie. the folder corresponding to the {@code java.io.file} - * System property - */ - public static File getCacheFolder() { - File file = null; - if (cacheFolder != null) { - file = new File(cacheFolder.toURI()); - } - - return file; - } - - /** - * Set the current cache folder. This is the folder in which - * newly created temporary files holding tile data will be created. - *

- * It is safe, though not necessarily sensible, to change the - * cache folder while one or more caches are running. - * - * @param folder the new cache folder or {@code null} to use - * the System's default temporary file folder - */ - public static void setCacheFolder(File folder) { - synchronized(folderLock) { - if (folder == null) { - cacheFolder = null; - } else { - cacheFolder = new File(folder.toURI()); - } - } - } - - /** - * Package-private constructor - * - * @param id the tile's unique identifier, allocated by the cache - * @param owner the image to which this tile belongs - * @param tileX the column index for this tile in the image's tile grid - * @param tileY the row index for this tile in the image's tile grid - * @param raster the image data for this tile - * @param writeToFile if true, the tile's data will be cached to disk - * immediately; if false, disk caching is deferred - * @param tileCacheMetric optional tile cache metric for use in scheduling - * (may be {@code null}) - * - * @throws IOException if an attempt to write the tile to disk fails - */ - DiskCachedTile(Object id, - RenderedImage owner, - int tileX, - int tileY, - Raster raster, - boolean writeToFile, - Object tileCacheMetric) throws IOException { - - if (owner == null || raster == null) { - throw new IllegalArgumentException( - "All of owner, tile and file args must be non-null"); - } - - this.id = id; - this.ownerRef = new WeakReference(owner); - this.tileX = tileX; - this.tileY = tileY; - this.tileCacheMetric = tileCacheMetric; - this.location = raster.getBounds().getLocation(); - this.isWritable = (raster instanceof WritableRaster); - - DataBuffer db = raster.getDataBuffer(); - numBanks = db.getNumBanks(); - dataLen = db.getSize(); - memorySize = DataBuffer.getDataTypeSize(db.getDataType()) / 8L * dataLen * numBanks; - - if (writeToFile ) { - writeData(raster); - } - - setTileTimeStamp(System.currentTimeMillis()); - } - - /** - * Returns a string representation of this cached tile - */ - @Override - public String toString() { - RenderedImage o = getOwner(); - String ostring = o == null ? "null" : o.toString(); - - Raster t = getTile(); - String tstring = t == null ? "null" : t.toString(); - - return getClass().getName() + "@" + Integer.toHexString(hashCode()) + - ": owner = " + ostring + - " tileX = " + Integer.toString(tileX) + - " tileY = " + Integer.toString(tileY) + - " tile = " + tstring + - " id = " + ((id instanceof Long)? Long.toHexString(((Long)id).longValue()) : id.toString()) + - " memorySize = " + Long.toString(memorySize) + - " timeStamp = " + Long.toString(timeStamp) + - " file = " + file.getPath(); - } - - /** - * Do not use this method. - *

- * It is implemented to satisfy the CAchedTile interface but calling it will - * provoke an UnsupportedOperationException. - *

- * To get a tile, use {@linkplain DiskBasedTileCache#getTile(java.awt.image.RenderedImage, int, int) } - */ - public Raster getTile() { - throw new UnsupportedOperationException("Can't get a tile directly from a DiskCachedTile object"); - } - - /** - * Gets the image that owns this tile. - * - * @return the owning image - */ - public RenderedImage getOwner() { - return ownerRef.get(); - } - - /** - * Gets the last time of access for this tile. - * - * @return the time stamp - */ - public long getTileTimeStamp() { - return timeStamp; - } - - /** - * Gets the tile cache metric for this tile. - * - * @return the metric - */ - public Object getTileCacheMetric() { - return tileCacheMetric; - } - - /** - * Gets the tile size in bytes. This is actually the - * size of the raster associated with this cached tile instance. - * - * @return tile size in bytes - */ - public long getTileSize() { - return memorySize; - } - - /** - * Gets the most recent action for this tile. More information can be - * retrieved by getting the corresponding {@linkplain DiskCachedTile.TileAction} - * constant as shown here: - * 


-     * int code = tile.getAction();
-     * TileAction action = TileAction.get(code);
-     * System.out.println("tile action: " + action.getDescription());
-     *
- * - * @return the most recent action - */ - public int getAction() { - return action.ordinal(); - } - - /** - * Queries if this tile has been cached to disk. This method is - * a short-cut for: {@code getFile() != null}. - * - * @return {@code true} if the tile is cached on disk; {@code false} otherwise - */ - public boolean cachedToDisk() { - return file != null; - } - - /** - * Gets this tile's disk cache file. Returns {@code null} if the tile has not - * been cached to disk. - * - * @return the disk cache file for this tile or {@code null} - */ - public File getFile() { - return file; - } - - /** - * Deletes this tile's disk cache file. If the file could not be deleted - * a warning is logged. - */ - public void deleteDiskCopy() { - if (file != null) { - if (!file.delete()) { - LOGGER.log(Level.WARNING, - "Unable to delete cached image tile file: {0}", file.getPath()); - } - } - } - - /** - * Gets the unique ID for this tile. The ID is a combination of - * either the JAI-generated unique ID of the owning image or the image's - * hash key if a unique ID was not available plus the tile index. - * The returned object will be either Long or BigInteger. - * - * @return tile ID - */ - public Object getTileId() { - return id; - } - - /** - * Gets this tile's location in its parent image in pixel coordinates - * - * @return tile origin expressed in parent image coordinates - */ - public Point getLocation() { - return new Point(location); - } - - /** - * Gets the X ordinate of this tile in the parent image tile grid. - * - * @return tile grid X ordinate - */ - public int getTileX() { - return tileX; - } - - /** - * Gets the Y ordinate of this tile in the parent image tile grid. - * - * @return tile grid Y ordinate - */ - public int getTileY() { - return tileY; - } - - /** - * Queries if this tile is writable. - * - * @return {@code true} if writable; {@code false} otherwise - */ - public boolean isWritable() { - return isWritable; - } - - /** - * Package-private method called by the controlling {@linkplain DiskBasedTileCache} - * object when the tile is added to, or removed from, the cache. - */ - void setAction( TileAction action ) { - this.action = action; - } - - /** - * Package-private method called by the controlling {@linkplain DiskBasedTileCache} - * object when the tile is accessed - */ - void setTileTimeStamp(long time) { - this.timeStamp = time; - } - - /** - * Package-private method that reads data for the raster associated with this tile - * from disk - * - * @return a new instance of Raster or WritableRaster - */ - Raster readData() { - ImageInputStream strm = null; - DataBuffer dataBuf = null; - RenderedImage img = ownerRef.get(); - Raster raster = null; - - if (file != null && img != null) { - try { - strm = ImageIO.createImageInputStream(file); - - switch (img.getSampleModel().getDataType()) { - case DataBuffer.TYPE_BYTE: { - byte[][] bankData = new byte[numBanks][dataLen]; - for (int i = 0; i < numBanks; i++) { - int numRead = strm.read(bankData[i], 0, dataLen); - if (numRead < numBanks) { - throw new RuntimeException( - "Cached tile file appears to be truncated"); - } - } - dataBuf = new DataBufferByte(bankData, dataLen); - } - break; - - case DataBuffer.TYPE_DOUBLE: { - double[][] bankData = new double[numBanks][dataLen]; - for (int i = 0; i < numBanks; i++) { - strm.readFully(bankData[i], 0, dataLen); - } - dataBuf = new DataBufferDouble(bankData, dataLen); - } - break; - - case DataBuffer.TYPE_FLOAT: { - float[][] bankData = new float[numBanks][dataLen]; - for (int i = 0; i < numBanks; i++) { - strm.readFully(bankData[i], 0, dataLen); - } - dataBuf = new DataBufferFloat(bankData, dataLen); - } - break; - - case DataBuffer.TYPE_INT: { - int[][] bankData = new int[numBanks][dataLen]; - for (int i = 0; i < numBanks; i++) { - strm.readFully(bankData[i], 0, dataLen); - } - dataBuf = new DataBufferInt(bankData, dataLen); - } - break; - - case DataBuffer.TYPE_SHORT: { - short[][] bankData = new short[numBanks][dataLen]; - for (int i = 0; i < numBanks; i++) { - strm.readFully(bankData[i], 0, dataLen); - } - dataBuf = new DataBufferShort(bankData, dataLen); - } - break; - - case DataBuffer.TYPE_USHORT: { - short[][] bankData = new short[numBanks][dataLen]; - for (int i = 0; i < numBanks; i++) { - strm.readFully(bankData[i], 0, dataLen); - } - dataBuf = new DataBufferUShort(bankData, dataLen); - } - break; - - default: - throw new UnsupportedOperationException("Unsupported image data type"); - } - - } catch (FileNotFoundException ex) { - LOGGER.log(Level.SEVERE, "Failed to read image tile data", ex); - return null; - - } catch (IOException ex) { - LOGGER.log(Level.SEVERE, "Failed to read image tile data", ex); - return null; - } - finally{ - if(strm!=null) - try{ - strm.close(); - }catch (Throwable e) { - // chew me - } - } - } - - if (dataBuf != null) { - if (isWritable) { - raster = Raster.createWritableRaster(img.getSampleModel(), dataBuf, location); - } else { - raster = Raster.createRaster(img.getSampleModel(), dataBuf, location); - } - } - - return raster; - } - - - /** - * Write data for the raster associated with this tile to - * disk. This may be called by DiskMemTileCache - * as well as be the tile itself. - */ - void writeData(Raster raster) throws IOException { - ImageOutputStream strm = null; - DataBuffer dataBuf = raster.getDataBuffer(); - - if (file == null) { - // first time this tile has been written to disk - file = createFile(); - } - - try { - strm = ImageIO.createImageOutputStream(file); - - switch (dataBuf.getDataType()) { - case DataBuffer.TYPE_BYTE: - { - byte[] bankData; - for (int i = 0; i < numBanks; i++) { - bankData = ((DataBufferByte) dataBuf).getData(i); - strm.write(bankData); - } - } - break; - - case DataBuffer.TYPE_DOUBLE: - { - double[] bankData; - for (int i = 0; i < numBanks; i++) { - bankData = ((DataBufferDouble) dataBuf).getData(i); - strm.writeDoubles(bankData, 0, dataLen); - } - } - break; - - case DataBuffer.TYPE_FLOAT: - { - float[] bankData; - for (int i = 0; i < numBanks; i++) { - bankData = ((DataBufferFloat) dataBuf).getData(i); - strm.writeFloats(bankData, 0, dataLen); - } - } - break; - - case DataBuffer.TYPE_INT: - { - int[] bankData; - for (int i = 0; i < numBanks; i++) { - bankData = ((DataBufferInt) dataBuf).getData(i); - strm.writeInts(bankData, 0, dataLen); - } - } - break; - - case DataBuffer.TYPE_SHORT: - { - short[] bankData; - for (int i = 0; i < numBanks; i++) { - bankData = ((DataBufferShort) dataBuf).getData(i); - strm.writeShorts(bankData, 0, dataLen); - } - } - break; - - case DataBuffer.TYPE_USHORT: - { - short[] bankData; - for (int i = 0; i < numBanks; i++) { - bankData = ((DataBufferUShort) dataBuf).getData(i); - strm.writeShorts(bankData, 0, dataLen); - } - } - break; - - default: - throw new UnsupportedOperationException("Unsupported image data type"); - } - - - } catch (FileNotFoundException ex) { - LOGGER.log(Level.SEVERE, "Failed to write image tile data", ex); - - } catch (IOException ex) { - LOGGER.log(Level.SEVERE, "Failed to write image tile data", ex); - } - finally{ - if(strm!=null) - try{ - strm.close(); - }catch (Throwable e) { - // chew me - } - } - } - - /** - * Create a file to cache the given tile on disk. Presently - * this method does nothing more than delegate to File.createTempFile - * - * @throws java.io.IOException - */ - private File createFile() throws IOException { - synchronized(folderLock) { - return File.createTempFile(FILE_PREFIX, FILE_SUFFIX, cacheFolder); - } - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java deleted file mode 100644 index a0867f145..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemCacheException.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.awt.image.RenderedImage; - -/** - * Base exception class for {@code DiskMemTileCache} errors. - * - * @see DiskMemTileCache - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class DiskMemCacheException extends Exception { - - /** - * Creates a new instance. - * - * @param owner the owning image - * @param tileX tile X ordinate - * @param tileY tile Y ordinate - * @param msg error message - */ - public DiskMemCacheException(RenderedImage owner, int tileX, int tileY, String msg) { - super(String.format("tile at %d,%d of image %s %s", - tileX, tileY, owner.toString(), msg)); - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java deleted file mode 100644 index 898fd8488..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCache.java +++ /dev/null @@ -1,1277 +0,0 @@ -/* - * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.awt.Point; -import java.awt.image.Raster; -import java.awt.image.RenderedImage; -import java.io.IOException; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Observable; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.eclipse.imagen.CachedTile; -import org.eclipse.imagen.PlanarImage; -import org.eclipse.imagen.TileCache; - -import org.jaitools.CollectionFactory; -import org.jaitools.DaemonThreadFactory; - - -/** - * This class implements ImageN {@linkplain org.eclipse.imagen.TileCache}. It can store - * cached tiles on disk to allow applications to work with very large volumes of - * tiled image data without being limited by available memory. A subset of tiles - * (by default, the most recently accessed) are cached in memory to reduce access - * time. - *

- * - * The default behaviour is to cache newly added tiles into memory. If the cache - * needs to free memory to accommodate a tile, it does so by removing lowest priority - * tiles from memory and caching them to disk. Optionally, the user can specify - * that newly added tiles are cached to disk immediately. - *

- * - * Unlike the standard ImageN {@code TileCache} implementation, resident tiles are cached - * using strong references. This is to support the use of this class with - * {@linkplain org.jaitools.tiledimage.DiskMemImage} as well as operations that need to - * cache tiles that are expensive to create (e.g. output of a time-consuming analysis). - * A disadvantage of this design is that when the cache is being used for easily - * generated tiles it can end up unnecessarily holding memory that is more urgently - * required by other parts of an application. To avoid this happening, the cache can - * be set to auto-flush resident tiles at regular intervals. - *

- * - *

Implementation note

- * Tile polling and auto-flushing of memory resident tiles (if enabled) both run - * on low-priority background threads. These are marked as daemon threads to - * avoid these services blocking application shutdown. - * - * @author Michael Bedward - * @author Simone Giannecchini, GeoSolutions SAS - * - * @since 1.0 - * @version $Id$ - * - * @see DiskCachedTile - * @see TileAccessTimeComparator - */ -public class DiskMemTileCache extends Observable implements TileCache { - - private static final Logger LOGGER = Logger.getLogger("org.jaitools.tilecache"); - - /** - * The default memory capacity in bytes (64 * 2^20 = 64Mb) - * @see #setMemoryCapacity(long) - */ - public static final long DEFAULT_MEMORY_CAPACITY = 64L * 1024L * 1024L; - - /** - * The default memory threshold value (0.75) - * - * @see #setMemoryThreshold(float) - */ - public static final float DEFAULT_MEMORY_THRESHOLD = 0.75F; - - /** - * The default minimum period (2.5 seconds) of cache inactivity that - * must elapse before memory-resident tiles are automatically flushed. - * - * @see #setAutoFlushMemoryInterval(long) - */ - public static final long DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL = 2500; - - /** - * The default interval (2 seconds) for polling each tile to check if - * its owning image has been garbage collected. - * - * @see #setTilePollingInterval(long) - */ - public static final long DEFAULT_TILE_POLLING_INTERVAL = 2000L; - - - // @todo use ImageN ParameterList or some other ready-made class for this ? - private static class ParamDesc { - String key; - Class clazz; - Object defaultValue; - - ParamDesc(String key, Class clazz, Object defaultValue) { - this.key = key; this.clazz = clazz; this.defaultValue = defaultValue; - } - - boolean typeOK (Object value) { - if (Number.class.isAssignableFrom(clazz)) { - return Number.class.isAssignableFrom(value.getClass()); - - } else { - return clazz.isAssignableFrom(value.getClass()); - } - } - } - - /** - * Key for the parameter controlling initial memory capacity of the - * tile cache. This determines the maximum number of tiles that can - * be resident concurrently. The value must be numeric and will be - * treated as Long. - * @see #setMemoryCapacity(long) - * @see #DEFAULT_MEMORY_CAPACITY - */ - public static final String KEY_INITIAL_MEMORY_CAPACITY = "memcapacity"; - - /** - * Key for the parameter controlling whether newly added tiles - * are immediately cached to disk as well as in memory. The value - * must be one Boolean. If the value is {@code Boolean.FALSE} (the default), - * disk caching of tiles is deferred until required (ie. when - * memory needs to be freed for other tiles). - */ - public static final String KEY_ALWAYS_DISK_CACHE = "diskcache"; - - /** - * Key for the parameter controlling whether the cache will auto-flush - * memory-resident tiles. The value must be Boolean. If the value is - * {@code Boolean.TRUE}, auto-flushing of resident tiles will be enabled - * when the cache is created. The default is {@code Boolean.FALSE}. - * @see #setAutoFlushMemoryEnabled(boolean) - */ - public static final String KEY_AUTO_FLUSH_MEMORY_ENABLED = "enableautoflush"; - - /** - * Key for the cache auto-flush interval parameter. The value must be numeric - * and represents the interval, in milliseconds, between auto-flushes of - * resident tiles. Values less than or equal to zero are ignored. - * @see #setAutoFlushMemoryInterval(long) - * @see #DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL - */ - public static final String KEY_AUTO_FLUSH_MEMORY_INTERVAL = "autoflushinterval"; - - private static final Map paramDescriptors; - static { - ParamDesc desc; - paramDescriptors = new HashMap(); - - desc = new ParamDesc(KEY_INITIAL_MEMORY_CAPACITY, Number.class, DEFAULT_MEMORY_CAPACITY); - paramDescriptors.put( desc.key, desc ); - - desc = new ParamDesc(KEY_ALWAYS_DISK_CACHE, Boolean.class, Boolean.FALSE); - paramDescriptors.put( desc.key, desc ); - - desc = new ParamDesc(KEY_AUTO_FLUSH_MEMORY_ENABLED, Boolean.class, Boolean.FALSE); - paramDescriptors.put( desc.key, desc ); - - desc = new ParamDesc(KEY_AUTO_FLUSH_MEMORY_INTERVAL, Number.class, DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL); - paramDescriptors.put( desc.key, desc ); - } - - // maximum memory available for resident tiles - private long memCapacity; - - // current memory used for resident tiles - private long curMemory; - - /* - * A value between 0.0 and 1.0 that may be used for memory control - * if the param KEY_USE_MEMORY_THRESHOLD is TRUE. - */ - private float memThreshold; - - private boolean writeNewTilesToDisk; - - /** - * Map of all cached tiles. - */ - protected Map tiles; - - /** - * Memory-resident tiles. - */ - protected Map residentTiles; - - /** - * A tile comparator used to determine the priority of tiles for - * storage in memory. - */ - private Comparator comparator; - - /* List of tile references that is sorted into tile priority order when - * required for memory swapping. - *

- * Implementation note: we use this in preference to a SortedSet or similar - * because of the complications of using the remove(obj) method with a sorted - * collection, where the comparator is used rather than the equals method. - */ - - /** - * Tiles sorted according to the current tile priority comparator. - */ - protected List sortedResidentTiles; - - // whether to send cache diagnostics to observers - private boolean diagnosticsEnabled; - - // Lock for tile access - private final ReentrantLock tileLock = new ReentrantLock(); - - // Variables used for auto-flushing of resident tiles - private ScheduledExecutorService flushService; - private ScheduledFuture flushFuture; - private long autoFlushInterval = DEFAULT_AUTO_FLUSH_MEMORY_INTERVAL; - private AtomicBoolean okToFlush = new AtomicBoolean(false); - - // Variables used for polling each tile to check if its owning image has been - // garbage collected - private final ScheduledExecutorService tilePollingService; - private ScheduledFuture tilePollingFuture; - private long tilePollingInterval = DEFAULT_TILE_POLLING_INTERVAL; - - - /** - * Creates a new cache with all parameters set to their default values. - */ - public DiskMemTileCache() { - this(null); - } - - /** - * Creates a new cache. - * - * @param params an optional map of parameters (may be empty or {@code null}) - */ - public DiskMemTileCache(Map params) { - if (params == null) { - params = Collections.emptyMap(); - } - - diagnosticsEnabled = false; - tiles = new HashMap(); - residentTiles = CollectionFactory.map(); - curMemory = 0L; - memThreshold = DEFAULT_MEMORY_THRESHOLD; - - Object o; - ParamDesc desc; - - desc= paramDescriptors.get(KEY_INITIAL_MEMORY_CAPACITY); - memCapacity = (Long)desc.defaultValue; - o = params.get(desc.key); - if (o != null) { - if (desc.typeOK(o)) { - memCapacity = ((Number)o).longValue(); - } - } - - desc = paramDescriptors.get(KEY_ALWAYS_DISK_CACHE); - writeNewTilesToDisk = (Boolean)desc.defaultValue; - o = params.get(desc.key); - if (o != null) { - if (desc.typeOK(o)) { - writeNewTilesToDisk = (Boolean)o; - } - } - - desc = paramDescriptors.get(KEY_AUTO_FLUSH_MEMORY_INTERVAL); - autoFlushInterval = ((Number)desc.defaultValue).longValue(); - o = params.get(desc.key); - if (o != null) { - if (desc.typeOK(o)) { - long lval = ((Number)o).longValue(); - if (lval > 0) { - autoFlushInterval = lval; - } - } - } - - desc = paramDescriptors.get(KEY_AUTO_FLUSH_MEMORY_ENABLED); - o = params.get(desc.key); - if (o != null) { - if (desc.typeOK(o)) { - setAutoFlushMemoryEnabled((Boolean)o); - } - } - - comparator = new TileAccessTimeComparator(); - sortedResidentTiles = new ArrayList(); - - tilePollingService = Executors.newSingleThreadScheduledExecutor( - new DaemonThreadFactory(Thread.MIN_PRIORITY, "cache-polling")); - - startTilePolling(); - } - - /** - * Adds a tile to the cache if not already present. - * - * @param owner the image that this tile belongs to - * @param tileX the tile column - * @param tileY the tile row - * @param data the tile data - */ - public void add(RenderedImage owner, int tileX, int tileY, Raster data) { - add(owner, tileX, tileY, data, null); - } - - /** - * Adds a tile to the cache if not already present. - * - * @param owner the image that this tile belongs to - * @param tileX the tile column - * @param tileY the tile row - * @param data the tile data - * @param tileCacheMetric optional tile cache metric (may be {@code null} - */ - public void add(RenderedImage owner, - int tileX, - int tileY, - Raster data, - Object tileCacheMetric) { - - tileLock.lock(); - - try { - okToFlush.set(false); - Object key = getTileId(owner, tileX, tileY); - if (tiles.containsKey(key)) { - // tile is already cached - return; - } - - DiskCachedTile tile = new DiskCachedTile( - key, owner, tileX, tileY, data, writeNewTilesToDisk, tileCacheMetric); - tiles.put(key, tile); - - if ( makeResident(tile, data) ) { - tile.setAction(DiskCachedTile.TileAction.ACTION_ADDED_RESIDENT); - } else { - tile.setAction(DiskCachedTile.TileAction.ACTION_ADDED); - } - - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - - } catch (IOException ex) { - LOGGER.log(Level.SEVERE, "Unable to cache this tile on disk", ex); - - } finally { - tileLock.unlock(); - } - } - - /** - * Removes a tile from the cache. - * - * @param owner the image that this tile belongs to - * @param tileX the tile column - * @param tileY the tile row - */ - public void remove(RenderedImage owner, int tileX, int tileY) { - tileLock.lock(); - - try { - okToFlush.set(false); - Object key = getTileId(owner, tileX, tileY); - DiskCachedTile tile = tiles.get(key); - - if (tile == null) { - return; - } - - if (residentTiles.containsKey(key)) { - try { - removeResidentTile(key, false); - - } catch (DiskCacheFailedException ex) { - /* - * It would be nicer to just throw this exception - * upwards but we can't in the overidden method - */ - LOGGER.log(Level.SEVERE, null, ex); - } - } - - tile.deleteDiskCopy(); - - tile.setAction(DiskCachedTile.TileAction.ACTION_REMOVED); - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - - tiles.remove(key); - - } finally { - tileLock.unlock(); - } - } - - /** - * Gets the specified tile from the cache if present. If the tile is - * cached but not resident in memory it will be read from the cache's - * disk storage and made resident. - * - * @param owner the image that the tile belongs to - * @param tileX the tile column - * @param tileY the tile row - * @return the requested tile or {@code null} if the tile was not cached - */ - public Raster getTile(RenderedImage owner, int tileX, int tileY) { - tileLock.lock(); - - try { - okToFlush.set(false); - Raster r = null; - Object key = getTileId(owner, tileX, tileY); - - DiskCachedTile tile = tiles.get(key); - if (tile != null) { - - // is the tile resident ? - r = residentTiles.get(key); - if (r == null) { - /* - * The tile is not resident. Attempt - * to read it from the disk. - */ - r = tile.readData(); - if (r == null) { - /* The tile was not cached on disk. It may have - * been resident only, and then flushed. - */ - return null; - } - - if (makeResident(tile, r)) { - tile.setAction(DiskCachedTile.TileAction.ACTION_RESIDENT); - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - } - } - - tile.setAction(DiskCachedTile.TileAction.ACTION_ACCESSED); - tile.setTileTimeStamp(System.currentTimeMillis()); - - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - } - - return r; - } finally { - tileLock.unlock(); - } - } - - /** - * Gets all cached tiles associated with the given image. - * The tiles will be loaded into memory as space allows. - * - * @param owner the image for which tiles are requested - * @return an array of tile Rasters - */ - public Raster[] getTiles(RenderedImage owner) { - tileLock.lock(); - - try { - okToFlush.set(false); - int minX = owner.getMinTileX(); - int minY = owner.getMinTileY(); - int numX = owner.getNumXTiles(); - int numY = owner.getNumYTiles(); - - List keys = new ArrayList(); - for (int y = minY, ny = 0; ny < numY; y++, ny++) { - for (int x = minX, nx = 0; nx < numX; x++, nx++) { - Object key = getTileId(owner, x, y); - if (tiles.containsKey(key)) { - keys.add(key); - } - } - } - - Raster[] rasters = new Raster[keys.size()]; - int k = 0; - for (Object key : keys) { - DiskCachedTile tile = tiles.get(key); - Raster r = residentTiles.get(tile.getTileId()); - if (r == null) { - r = tile.readData(); - makeResident(tile, r); - } - - rasters[k++] = r; - - tile.setTileTimeStamp(System.currentTimeMillis()); - tile.setAction(DiskCachedTile.TileAction.ACTION_ACCESSED); - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - } - - return rasters; - - } finally { - tileLock.unlock(); - } - } - - /** - * Removes all tiles that belong to the given image from the cache. - * - * @param owner the image owning the tiles to be removed - */ - public void removeTiles(RenderedImage owner) { - tileLock.lock(); - try { - for (int y = owner.getMinTileY(), ny = 0; ny < owner.getNumYTiles(); y++, ny++) { - for (int x = owner.getMinTileX(), nx = 0; nx < owner.getNumXTiles(); x++, nx++) { - remove(owner, x, y); - } - } - } finally { - tileLock.unlock(); - } - } - - /** - * Sets the interval between polling each tile to check if its owning image - * has been garbage collected. Any such tiles are removed from the - * cache. - * - * @param interval interval in milliseconds - * (values less than or equal to zero are ignored) - */ - public void setTilePollingInterval(long interval) { - if (interval > 0 && interval != tilePollingInterval) { - stopTilePolling(); - tilePollingInterval = interval; - startTilePolling(); - } - } - - /** - * Sets the interval between polling each tile to check if its owning image - * has been garbage collected. Any such tiles are removed from the - * cache. * - * @return interval in milliseconds - */ - public long getTilePollingInterval() { - return tilePollingInterval; - } - - /** - * Starts the tile polling task which calls {@link #removeNullTiles()} - * at a fixed interval. - */ - private void startTilePolling() { - if (!isPollingTiles()) { - tilePollingFuture = tilePollingService.scheduleAtFixedRate( - new Runnable() { - public void run() { - removeNullTiles(); - } - }, - tilePollingInterval, - tilePollingInterval, - TimeUnit.MILLISECONDS); - } - } - - /** - * Stops the tile polling task. - */ - private void stopTilePolling() { - if (isPollingTiles()) { - tilePollingFuture.cancel(true); - tilePollingFuture = null; - } - } - - private boolean isPollingTiles() { - return tilePollingFuture != null && !tilePollingFuture.isDone(); - } - - /** - * Checks if any tiles have a {@code null} owner (e.g. owning image has been - * garbage collected) and, if so, removes them from the cache. - */ - private void removeNullTiles() { - if (!tileLock.tryLock()) { // jumps the queue of waiting threads - return; - } - - try { - Set nullTileKeys = CollectionFactory.set(); - for (Object key : tiles.keySet()) { - DiskCachedTile tile = tiles.get(key); - if (tile.getOwner() == null) { - nullTileKeys.add(key); - } - } - - for (Object key : nullTileKeys) { - DiskCachedTile tile = tiles.get(key); - tile.deleteDiskCopy(); - if (residentTiles.containsKey(key)) { - residentTiles.remove(key); - sortedResidentTiles.remove(tile); - curMemory -= tile.getTileSize(); - } - tiles.remove(key); - } - - } finally { - tileLock.unlock(); - } - } - - /** - * Adds all tiles for the given image to the cache. - * - * @param owner the image that the tiles belong to - * @param tileIndices an array of Points specifying the column-row coordinates - * of each tile - * @param tiles tile data in the form of Raster objects - * @param tileCacheMetric optional metric (may be {@code null}) - */ - public void addTiles(RenderedImage owner, - Point[] tileIndices, - Raster[] tiles, - Object tileCacheMetric) { - - if (tileIndices.length != tiles.length) { - throw new IllegalArgumentException( - "tileIndices and tiles args must be the same length"); - } - - tileLock.lock(); - try { - for (int i = 0; i < tiles.length; i++) { - add(owner, tileIndices[i].x, tileIndices[i].y, tiles[i], tileCacheMetric); - } - - } finally { - tileLock.unlock(); - } - } - - /** - * Gets the specified tiles for the given image. - * - * @param owner the image that the tiles belong to - * @param tileIndices an array of Points specifying the column-row coordinates - * of each tile - * @return data for the requested tiles as Raster objects - */ - public Raster[] getTiles(RenderedImage owner, Point[] tileIndices) { - tileLock.lock(); - try { - Raster[] r = null; - - if (tileIndices.length > 0) { - r = new Raster[tileIndices.length]; - for (int i = 0; i < tileIndices.length; i++) { - r[i] = getTile(owner, tileIndices[i].x, tileIndices[i].y); - } - } - - return r; - - } finally { - tileLock.unlock(); - } - } - - /** - * Removes ALL tiles from the cache: all resident tiles will be - * removed from memory and all files for disk-cached tiles will - * be discarded. - *

- * The update action of each tile will be set to {@linkplain DiskCachedTile#ACTION_REMOVED}. - */ - public void flush() { - tileLock.lock(); - - try { - flushMemory(); - - for (DiskCachedTile tile : tiles.values()) { - tile.deleteDiskCopy(); - tile.setAction(DiskCachedTile.TileAction.ACTION_REMOVED); - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - } - tiles.clear(); - - } finally { - tileLock.unlock(); - } - } - - /** - * Removes all resident tiles from memory. No rewriting of tile data - * to disk is done. - */ - public void flushMemory() { - tileLock.lock(); - try { - residentTiles.clear(); - sortedResidentTiles.clear(); - curMemory = 0; - - } finally { - tileLock.unlock(); - } - } - - /** - * Frees memory for resident tiles so that the fraction of memory occupied is - * no more than the current value of the mamory threshold. - * - * @see DiskMemTileCache#setMemoryThreshold(float) - */ - public void memoryControl() { - tileLock.lock(); - try { - long maxUsed = (long) (memThreshold * memCapacity); - long toFree = curMemory - maxUsed; - if (toFree > 0) { - defaultMemoryControl(toFree); - } - } finally { - tileLock.unlock(); - } - } - - /** - * Makes the requested amount of memory cache available, removing - * resident tiles as necessary. - * - * @param memRequired memory requested (bytes) - */ - private void defaultMemoryControl( long memRequired ) { - if (memRequired > memCapacity) { - // @todo something better than this... - throw new RuntimeException("space required is greater than cache memory capacity"); - } - - /* - * Remove one or more lowest priority tiles to free - * space - */ - Collections.sort(sortedResidentTiles, comparator); - while (memCapacity - curMemory < memRequired && !sortedResidentTiles.isEmpty()) { - Object key = sortedResidentTiles.get(sortedResidentTiles.size()-1).getTileId(); - - try { - removeResidentTile(key, true); - } catch (DiskCacheFailedException ex) { - /* - * It would be nicer to just throw this exception - * upwards be we can't in the overidden method - */ - LOGGER.log(Level.SEVERE, null, ex); - } - } - } - - - /** - * Does nothing. - * - * @deprecated Please do not use this method - */ - @Deprecated - public void setTileCapacity(int arg0) { - } - - /** - * Always returns 0. - * - * @deprecated Please do not use this method - */ - public int getTileCapacity() { - return 0; - } - - /** - * Resets the memory capacity of the cache. Setting capacity to 0 will - * flush all resident tiles from memory. Setting a capacity less than the - * current capacity could cause some memory-resident tiles being - * removed from memory. - * - * @param newCapacity requested memory capacity for resident tiles - */ - public void setMemoryCapacity(long newCapacity) { - tileLock.lock(); - - try { - okToFlush.set(false); - if (newCapacity < 0) { - throw new IllegalArgumentException("memory capacity must be >= 0"); - } - - long oldCapacity = memCapacity; - memCapacity = newCapacity; - - if (newCapacity == 0) { - flushMemory(); - - } else if (newCapacity < oldCapacity && curMemory > newCapacity) { - /* - * Note: we free memory here directly rather than using - * memoryControl or defaultMemoryControl methods because - * they will fail when memCapacity has been reduced - */ - Collections.sort(sortedResidentTiles, comparator); - while (curMemory > newCapacity) { - Object key = sortedResidentTiles.get(sortedResidentTiles.size() - 1).getTileId(); - try { - removeResidentTile(key, true); - } catch (DiskCacheFailedException ex) { - /* - * It would be nicer to just throw this exception - * upwards be we can't in the overidden method - */ - LOGGER.log(Level.SEVERE, null, ex); - } - } - } - } finally { - tileLock.unlock(); - } - } - - /** - * Gets the amount of memory, in bytes, allocated for storage of - * resident tiles. - * - * @return resident tile memory capacity in bytes - */ - public long getMemoryCapacity() { - return memCapacity; - } - - /** - * Gets the amount of memory currently being used for storage of - * memory-resident tiles. - * - * @return current memory use in bytes - */ - public long getCurrentMemory() { - return curMemory; - } - - /** - * Sets the memoryThreshold value to a floating point number that ranges from - * 0.0 to 1.0. When the cache memory is full, the memory usage will be reduced - * to this fraction of the total cache memory capacity. For example, a value - * of .75 will cause 25% of the memory to be cleared, while retaining 75%. - * - * @param newThreshold the new memory threshold between 0 and 1 - * - * @throws IllegalArgumentException if the memoryThreshold is less than 0.0 or greater than 1.0 - */ - public void setMemoryThreshold(float newThreshold) { - if (newThreshold < 0.0F) { - memThreshold = 0.0F; - } else if (newThreshold > 1.0F) { - memThreshold = 1.0F; - } else { - memThreshold = newThreshold; - } - - memoryControl(); - } - - /** - * Returns the memory threshold, which is the fractional amount of cache memory - * to retain during tile removal. This only applies if memory thresholding has - * been enabled by passing the parameter {@linkplain #KEY_USE_MEMORY_THRESHOLD} to - * the constructor with a value of {@code Boolean.TRUE}. - * - * @return the retained fraction of memory - */ - public float getMemoryThreshold() { - return memThreshold; - } - - /** - * Sets the comparator to use to assign memory-residence priority to - * tiles. If {@code comp} is {@code null} the default comparator - * ({@link TileAccessTimeComparator}) will be used. - * - * @param comp the comparator or {@code null} for the default - */ - public void setTileComparator(Comparator comp) { - tileLock.lock(); - try { - if (comp == null) { - // switch to default comparator based on tile access time - comparator = new TileAccessTimeComparator(); - } else { - comparator = comp; - } - - sortedResidentTiles = new ArrayList(); - for (Object key : residentTiles.keySet()) { - sortedResidentTiles.addAll(tiles.values()); - } - Collections.sort(sortedResidentTiles, comparator); - - } finally { - tileLock.unlock(); - } - } - - /** - * Gets the comparator currently used to assign memory-residence - * priority to tiles. - * - * @return the current comparator - */ - public Comparator getTileComparator() { - return comparator; - } - - /** - * Gets the total number of tiles currently in the cache. - * - * @return number of cached tiles - */ - public int getNumTiles() { - return tiles.size(); - } - - /** - * Gets the number of tiles currently residing in the - * cache's memory storage. - * - * @return number of memory-resident tiles - */ - public int getNumResidentTiles() { - return residentTiles.size(); - } - - /** - * Checks whether a given tile is in this cache. - * - * @param owner the owning image - * @param tileX tile column - * @param tileY tile row - * @return {@code true} if the cache contains the tile; {@code false} otherwise - */ - public boolean containsTile(RenderedImage owner, int tileX, int tileY) { - Object key = getTileId(owner, tileX, tileY); - return tiles.containsKey(key); - } - - /** - * Checks whether a given tile is in this cache's memory storage. - * - * @param owner the owning image - * @param tileX tile column - * @param tileY tile row - * @return {@code true} if the tile is in cache memory; {@code false} otherwise - */ - public boolean containsResidentTile(RenderedImage owner, int tileX, int tileY) { - Object key = getTileId(owner, tileX, tileY); - return residentTiles.containsKey(key); - } - - /** - * Informs the cache that a tile's data have changed. The tile should - * be resident in memory as the result of a previous {@code getTile} - * request. If this is the case and the tile was previously written to - * disk, then the cache's disk copy of the tile will be refreshed. - *

- * If the tile is not resident in memory, for instance - * because of memory swapping for other tile accesses, the disk copy - * will not be refreshed and a {@code TileNotResidentException} is - * thrown. - * - * @param owner the owning image - * @param tileX tile column - * @param tileY tile row - * @throws TileNotResidentException if the tile is not resident - * @throws DiskCacheFailedException if the tile is cached to disk but its data could - * not be updated - */ - public void setTileChanged(RenderedImage owner, int tileX, int tileY) - throws TileNotResidentException, DiskCacheFailedException { - - tileLock.lock(); - try { - okToFlush.set(false); - Object tileId = getTileId(owner, tileX, tileY); - Raster r = residentTiles.get(tileId); - if (r == null) { - throw new TileNotResidentException(owner, tileX, tileY); - } - - DiskCachedTile tile = tiles.get(tileId); - if (tile.cachedToDisk()) { - try { - tile.writeData(r); - } catch (IOException ioEx) { - throw new DiskCacheFailedException(owner, tileX, tileY); - } - } - - } finally { - tileLock.unlock(); - } - } - - /** - * Enables or disables auto-flushing of memory resident with the - * currently set minimum interval. - * - * @param enable {@code true} to enable auto-flushing; {@code false} to disable - * @see #setAutoFlushMemoryInterval(long) - */ - public final void setAutoFlushMemoryEnabled(boolean enable) { - if (enable) { - if (!isAutoFlushMemoryEnabled()) { - if (flushService == null) { - flushService = Executors.newSingleThreadScheduledExecutor( - new DaemonThreadFactory(Thread.MIN_PRIORITY, "cache-flush")); - } - - flushFuture = flushService.scheduleWithFixedDelay( - new Runnable() { - public void run() { - if (okToFlush.getAndSet(true)) { - flushMemory(); - } - } - }, - autoFlushInterval, - autoFlushInterval, - TimeUnit.MILLISECONDS); - } - - } else if (isAutoFlushMemoryEnabled()) { - flushFuture.cancel(true); - } - } - - /** - * Checks whether auto-flushing of memory-resident tiles is currently enabled. - * - * @return {@code true} if the cache is auto-flushing; {@code false} otherwise - */ - public boolean isAutoFlushMemoryEnabled() { - return (flushFuture != null && !flushFuture.isDone()); - } - - /** - * Sets the minimum period of cache inactivity, in milliseconds, that must - * elapse before automatically flushing memory-resident tiles. - * - * @param interval interval in milliseconds - * (values less than or equal to zero are ignored) - */ - public void setAutoFlushMemoryInterval(long interval) { - if (interval > 0 && interval != autoFlushInterval) { - if (isAutoFlushMemoryEnabled()) { - setAutoFlushMemoryEnabled(false); - } - autoFlushInterval = interval; - setAutoFlushMemoryEnabled(true); - } - } - - /** - * Gets the current auto-flush interval. This is the minimum period of - * cache inactivity, in milliseconds, that must elapse before - * automatically flushing tiles. - * - * @return interval in milliseconds - */ - public long getAutoFlushMemoryInterval() { - return autoFlushInterval; - } - - /** - * Enables or disables the publishing of cache messages to Observers. - * - * @param state {@code true} to publish diagnostic messages; {@code false} to suppress them - */ - public void setDiagnostics(boolean state) { - diagnosticsEnabled = state; - } - - /** - * Accepts a {@code DiskMemCacheVisitor} object and calls its - * {@code visit} method for each tile in the cache. - * - * @param visitor the visitor - */ - public void accept(DiskMemTileCacheVisitor visitor) { - tileLock.lock(); - try { - okToFlush.set(false); - for (Object key : tiles.keySet()) { - visitor.visit(tiles.get(key), residentTiles.containsKey(key)); - } - } finally { - tileLock.unlock(); - } - } - - /** - * Adds a raster to those resident in memory. - */ - private boolean makeResident(DiskCachedTile tile, Raster data) { - if (tile.getTileSize() > memCapacity) { - return false; - } - - if (tile.getTileSize() > memCapacity - curMemory) { - memoryControl(); - - /* - * It is possible that the threshold rule fails to - * free enough memory for the tile - */ - if (tile.getTileSize() > memCapacity - curMemory) { - defaultMemoryControl(tile.getTileSize()); - } - } - - residentTiles.put(tile.getTileId(), data); - curMemory += tile.getTileSize(); - - /* - * We don't bother about sort order here. Instead, the list - * will be sorted by tile priority when resident tiles are - * being removed - */ - sortedResidentTiles.add(tile); - - return true; - } - - - /** - * Removes a tile from the cache's memory storage. This may be to free - * space for other tiles, in which case {@code writeData} will be - * set to {@code true} and, if the tile is writable, a request is made to write - * its data to disk again. If the tile is being removed from the cache - * entirely, this method will be called with {@code writeData} set - * to {@code false}. - * - * @param tileId the tile's unique id - * @param writeData if {@code true}, and the tile is writable, its data will be - * written to disk again; otherwise no writing is done. - */ - private void removeResidentTile(Object tileId, boolean writeData) throws DiskCacheFailedException { - DiskCachedTile tile = tiles.get(tileId); - Raster raster = residentTiles.remove(tileId); - sortedResidentTiles.remove(tile); - curMemory -= tile.getTileSize(); - - /** - * If the tile is writable, ie. its data are represented - * by a WritableRaster, we cache it to disk - */ - if (writeData && tile.isWritable()) { - try { - tile.writeData(raster); - } catch (IOException ioEx) { - throw new DiskCacheFailedException(tile.getOwner(), tile.getTileX(), tile.getTileY()); - } - } - - tile.setAction(DiskCachedTile.TileAction.ACTION_NON_RESIDENT); - if (diagnosticsEnabled) { - setChanged(); - notifyObservers(tile); - } - } - - - /** - * Generates a unique ID for this tile. This uses the same technique as the - * Sun memory cache implementation: putting the id of the owning image - * into the upper bytes of a long or BigInteger value and the tile index into - * the lower bytes. - * @param owner the owning image - * @param tileX tile column - * @param tileY tile row - * @return the ID as an Object which will be an instance of either Long or BigInteger - */ - private Object getTileId(RenderedImage owner, - int tileX, - int tileY) { - - long tileId = tileY * (long)owner.getNumXTiles() + tileX; - - BigInteger imageId = null; - - if (owner instanceof PlanarImage) { - imageId = (BigInteger)((PlanarImage)owner).getImageID(); - } - - if (imageId != null) { - byte[] buf = imageId.toByteArray(); - int length = buf.length; - byte[] buf1 = new byte[buf.length + 8]; - System.arraycopy(buf, 0, buf1, 0, length); - for (int i = 7, j = 0; i >= 0; i--, j += 8) - buf1[length++] = (byte)(tileId >> j); - return new BigInteger(buf1); - - } else { - tileId &= 0x00000000ffffffffL; - return Long.valueOf(((long)owner.hashCode() << 32) | tileId); - } - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java b/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java deleted file mode 100644 index 8f96f6791..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/DiskMemTileCacheVisitor.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -/** - * A visitor to collect information about tiles in a {@linkplain DiskMemTileCache}. - * This can be used to examine cache performance in more detail than with - * the cache's diagnostic methods and observer events. - * - * @see DiskMemTileCache - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public interface DiskMemTileCacheVisitor { - - /** - * Called by the cache once for each tile - * - * @param tile the tile being visited - * @param isResident set by the cache to indicate whether the tile is - * currently resident in memory - */ - public void visit(DiskCachedTile tile, boolean isResident); - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java b/klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java deleted file mode 100644 index 5042bca90..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/TileAccessTimeComparator.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.io.Serializable; -import java.util.Comparator; -import org.eclipse.imagen.CachedTile; - -/** - * This is the default comparator for CachedTile objects used by - * {@linkplain DiskMemTileCache} to determine priority of tiles for - * storage in memory. It orders tiles from most recent to least - * recent access time. - * - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class TileAccessTimeComparator implements Comparator, Serializable { - - public int compare(CachedTile t1, CachedTile t2) { - long time1 = t1.getTileTimeStamp(); - long time2 = t2.getTileTimeStamp(); - - // t1 > t2 if time1 < time2 - return (time1 > time2 ? -1 : (time1 == time2 ? 0 : 1)); - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java b/klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java deleted file mode 100644 index 816dbe8d7..000000000 --- a/klab.engine/src/main/java/org/jaitools/tilecache/TileNotResidentException.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tilecache; - -import java.awt.image.RenderedImage; - -/** - * Exception thrown when by DiskMemTileCache when a tile was - * not resident in cache memory when required to be. - * - * @see DiskMemTileCache - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class TileNotResidentException extends DiskMemCacheException { - - TileNotResidentException(RenderedImage owner, int tileX, int tileY) { - super(owner, tileX, tileY, " is not resident in memory"); - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java b/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java deleted file mode 100644 index 7abbfa13c..000000000 --- a/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImage.java +++ /dev/null @@ -1,818 +0,0 @@ -/* - * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tiledimage; - -import java.awt.Graphics2D; -import java.awt.Point; -import java.awt.Rectangle; -import java.awt.image.ColorModel; -import java.awt.image.DataBuffer; -import java.awt.image.Raster; -import java.awt.image.SampleModel; -import java.awt.image.TileObserver; -import java.awt.image.WritableRaster; -import java.awt.image.WritableRenderedImage; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.eclipse.imagen.ImageLayout; -import org.eclipse.imagen.PlanarImage; - -import org.jaitools.tilecache.DiskMemTileCache; -import org.jaitools.CollectionFactory; - - -/** - * A tiled image class similar to JAI's standard {@code TiledImage} that uses a - * {@code DiskMemTileCache} to manage its data. Image tiles are cached in memory - * and, if the volume of image data is greater than available memory, also on disk - * as temporary files. When a tile is accessed it is swapped into memory if not - * already resident. Thus, very large images can be handled, albeit at the cost of - * disk I/O. - *

- * By default, each {@code DiskMemImage} uses its own tile cache but it is also - * possible for images to share a common tile cache as shown here: - * 


- *     DiskMemImage image1 = new DiskMemImage(...);
- *     image1.useCommonTileCache(true);
- *
- *     DiskMemImage image2 = new DiskMemImage(...);
- *     image2.useCommonTileCache(true);
- *
- *     DiskMemImage image3 = new DiskMemImage(...);
- *
- * In the example above, image1 and image2 will share a common tile cache while - * image3 will use a separate, exclusive tile cache. You can test whether an - * image is using the common cache like this: - * 

- *     boolean usingCommonCache = image.isUsingCommonTileCache();
- *
- * The memory capacity of the common cache can be set like this: - * 

- *     long memCapacity = 128 * 1024 * 1024; // 128 Mb
- *     DiskMemImage.getCommonCache().setMemoryCapacity(memCapacity);
- *
- * - * @see DiskMemTileCache - * - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class DiskMemImage - extends PlanarImage - implements WritableRenderedImage { - - /** - * The default memory capacity of the common tile cache in bytes - * (256 * 1024 * 1024 = 256 Mb) - */ - public static final long DEFAULT_COMMON_CACHE_SIZE = 256L * 1024 * 1024; - - /** TileCache which will be shared bewtween all Images */ - private static DiskMemTileCache commonCache = null; - - /** - * Get the common tile cache. This method has the side effect of - * creating the common cache if it does not already exist (ie. it - * never returns {@code null}). - * - * @return the common tile cache - * - * @see #isUsingCommonCache() - * @see #setUseCommonCache(boolean) - */ - public static DiskMemTileCache getCommonTileCache() { - if (commonCache == null) { - commonCache = createNewCache(); - } - return commonCache; - } - - /** - * Sets a new common cache. If a common cache has already been - * initialized (either with this method or via {@linkplain #setUseCommonCache}) - * this method does nothing. - * - * @param newCache the cache instance to use as the common tile - * cache for {@code DiskMemImage} objects ({@code null} - * arguments are ignored) - */ - public static void setCommonTileCache(DiskMemTileCache newCache) { - if (newCache != null && commonCache == null) { - commonCache = newCache; - } - } - - /** - * Creates a new tile cache. - * - * @return a new instance of {@code DiskMemTileCache} - */ - private static DiskMemTileCache createNewCache() { - Map cacheParams = new HashMap(); - cacheParams.put(DiskMemTileCache.KEY_INITIAL_MEMORY_CAPACITY, Long.valueOf(DEFAULT_COMMON_CACHE_SIZE)); - cacheParams.put(DiskMemTileCache.KEY_ALWAYS_DISK_CACHE, Boolean.FALSE); - DiskMemTileCache cache = new DiskMemTileCache(cacheParams); - cache.setDiagnostics(false); - - return cache; - } - - /** - * The tile cache in use by this image. It may or may not be the - * common tile cache. Ahthough this is a {@code protected} field, - * it is recommended that sub-classes use {@linkplain #getTileCache()} - * for general purposes. - */ - protected DiskMemTileCache tileCache; - - /** - * Defines the valid range of tile x and y coordinates. - */ - protected Rectangle tileGrid; - - /** - * A 2D array with dimensions corresponding to the tile grid width - * and height that records, for each tile, the number of writers - * that have the tile checked out currently - */ - protected int[][] numWriters; - - /** - * The number of tiles that are currently checked out - * for writing - * - * @see #getWritableTileIndices() - */ - protected int numTilesInUse; - - /** - * The amount of memory (in bytes) required to hold - * the data for an image tile - */ - protected long tileMemorySize; - - /** - * The set of tile observers currently registered with - * this image - */ - protected Set tileObservers; - - /** - * Creates a new image with default values for origin (0, 0), - * x and y tile offsets (0) and {@code ColorModel} - * ({@code null}). - * - * @param width image width - * @param height image height - * @param tileSampleModel a {@code SampleModel} specifying the dimensions - * data type etc. for image tiles - */ - public DiskMemImage( - int width, int height, - SampleModel tileSampleModel) { - - this(0, 0, // minX, minY - width, height, - 0, 0, // tileGridXOffset, tileGridYOffset - tileSampleModel, - null // ColorModel - ); - } - - - /** - * Creates a new image with default values for origin (0, 0) and - * tile offsets (0). - * - * @param width image width - * @param height image height - * @param tileSampleModel the {@code SampleModel} - * @param colorModel the {@code ColorModel} - */ - public DiskMemImage( - int width, int height, - SampleModel tileSampleModel, - ColorModel colorModel) { - - this(0, 0, // minX, minY - width, height, - 0, 0, // tileGridXOffset, tileGridYOffset - tileSampleModel, - colorModel - ); - } - - - /** - * Creates a new image with default values for tile offsets (0) - * and {@code ColorModel} ({@code null}) - * - * @param minX x coordinate of the upper-left image pixel - * @param minY y coordinate of the upper-left image pixel - * @param width image width - * @param height image height - * @param tileSampleModel the {@code SampleModel} - */ - public DiskMemImage( - int minX, int minY, - int width, int height, - SampleModel tileSampleModel) { - - this(minX, minY, - width, height, - 0, 0, // tileGridXOffset, tileGridYOffset - tileSampleModel, - null // ColorModel - ); - } - - - /** - * Creates a new image. Sets the {@code ColorModel} to {@code null} - * - * @param minX x coordinate of the upper-left image pixel - * @param minY y coordinate of the upper-left image pixel - * @param width image width - * @param height image height - * data type etc. for image tiles - * @param tileGridXOffset x coordinate of the upper-left pixel of the upper-left tile - * @param tileGridYOffset y coordinate of the upper-left pixel of the upper-left tile - * @param tileSampleModel the {@code SampleModel} - */ - public DiskMemImage( - int minX, int minY, - int width, int height, - int tileGridXOffset, int tileGridYOffset, - SampleModel tileSampleModel) { - - this(minX, minY, - width, height, - tileGridXOffset, tileGridYOffset, - tileSampleModel, - null // ColorModel - ); - } - - /** - * Creates a new image. - * - * @param minX x coordinate of the upper-left image pixel - * @param minY y coordinate of the upper-left image pixel - * @param width image width - * @param height image height - * data type etc. for image tiles - * @param tileGridXOffset x coordinate of the upper-left pixel of the upper-left tile - * @param tileGridYOffset y coordinate of the upper-left pixel of the upper-left tile - * @param tileSampleModel the {@code SampleModel} - * @param colorModel the {@code ColorModel} - */ - public DiskMemImage( - int minX, int minY, - int width, int height, - int tileGridXOffset, int tileGridYOffset, - SampleModel tileSampleModel, ColorModel colorModel) { - - super(new ImageLayout( - minX, minY, - width, height, - tileGridXOffset, tileGridYOffset, - tileSampleModel.getWidth(), tileSampleModel.getHeight(), - tileSampleModel, colorModel), - - null, null); // sources, properties - - tileGrid = new Rectangle( - getMinTileX(), - getMinTileY(), - getMaxTileX() - getMinTileX() + 1, - getMaxTileY() - getMinTileY() + 1); - - numWriters = new int[tileGrid.width][tileGrid.height]; - numTilesInUse = 0; - - DataBuffer db = tileSampleModel.createDataBuffer(); - tileMemorySize = DataBuffer.getDataTypeSize(db.getDataType()) / 8L * - db.getSize() * db.getNumBanks(); - - tileObservers = CollectionFactory.set(); - - // just to remind us that we are deferring creation of - // the tile cache - tileCache = null; - } - - /** - * Gets a tile for reading. Any changes to the tile's data - * will not be preserved by the cache. - * - * @param tileX the tile's column in the tile grid - * @param tileY the tile's row in the tile grid - * @return the tile data for reading - */ - @Override - public Raster getTile(int tileX, int tileY) { - Raster r = null; - if (tileGrid.contains(tileX, tileY)) { - r = getTileCache().getTile(this, tileX, tileY); - if (r == null) { - r = createTile(tileX, tileY); - getTileCache().add(this, tileX, tileY, r); - } - } - - return r; - } - - /** - * {@inheritDoc} - */ - public void addTileObserver(TileObserver to) { - tileObservers.add(to); - } - - /** - * {@inheritDoc} - */ - public void removeTileObserver(TileObserver to) { - tileObservers.remove(to); - } - - /** - * Checks out a tile for writing. The tile will be unavailable to other - * callers through this method until it is released via - * {@linkplain #releaseWritableTile(int, int)}. If this method is - * called for the same time prior to the tile being released - * it returns {@code null} and a warning message is logged. - * - * @param tileX tile X ordinate - * @param tileY tile Y ordinate - * @return the tile data for writing, or {@code null} if the tile - * is already checked-out - */ - public WritableRaster getWritableTile(int tileX, int tileY) { - WritableRaster r = null; - if (tileGrid.contains(tileX, tileY)) { - numWriters[tileX - tileGrid.x][tileY - tileGrid.y]++ ; - if (numWriters[tileX - tileGrid.x][tileY - tileGrid.y] == 1) { - numTilesInUse++ ; - } - - r = (WritableRaster) getTileCache().getTile(this, tileX, tileY); - if (r == null) { - r = createTile(tileX, tileY); - getTileCache().add(this, tileX, tileY, r); - } - - for (TileObserver obs : tileObservers) { - obs.tileUpdate(this, tileX, tileY, true); - } - } - return r; - } - - /** - * Releases a tile that was checked out for writing. The cache's disk copy of - * the tile's data will be refreshed. - *

- * If the cache no longer has the tile in its memory storage, e.g. - * because of memory swapping for other tile accesses, the cache - * will be unable to refresh the tile's data on disk. In this case - * a warning message is logged. - *

- * If the tile was not previously checked-out via - * {@linkplain #getWritableTile(int, int)} a warning message is - * logged. - * - * @param tileX tile X ordinate - * @param tileY tile Y ordinate - */ - public void releaseWritableTile(int tileX, int tileY) { - if (tileGrid.contains(tileX, tileY)) { - numWriters[tileX - tileGrid.x][tileY - tileGrid.y]-- ; - if (numWriters[tileX - tileGrid.x][tileY - tileGrid.y] < 0) { - Logger.getLogger( - DiskMemImage.class.getName()).log(Level.SEVERE, - String.format("Tile %d,%d released more times than it has been checked out", - tileX, tileY)); - - } else if (numWriters[tileX - tileGrid.x][tileY - tileGrid.y] == 0) { - numTilesInUse--; - } - - /* - * TODO: Consider skipping this step. It is mostly here as a - * precaution against the cached tile being garbage collected - * if the system runs very low on memory. - */ - try { - getTileCache().setTileChanged(this, tileX, tileY); - - } catch (Exception ex) { - Logger.getLogger(DiskMemImage.class.getName()). - log(Level.SEVERE, null, ex); - } - - for (TileObserver obs : tileObservers) { - obs.tileUpdate(this, tileX, tileY, false); - } - } - } - - /** - * Tests if a tile is currently checked-out for writing (via - * a call to {@linkplain #getWritableTile(int, int)}. - * - * @param tileX tile X ordinate - * @param tileY tile Y ordinate - * @return {@code true} if the tile is currently checked-out for - * writing; {@code false} otherwise. - */ - public boolean isTileWritable(int tileX, int tileY) { - return numWriters[tileX - tileGrid.x][tileY - tileGrid.y] > 0; - } - - /** - * Gets the indices (tile grid col,row) of tiles that are - * currently checked out for writing. - * - * @return tile indices - */ - public Point[] getWritableTileIndices() { - Point[] indices = null; - - if (numTilesInUse > 0) { - indices = new Point[numTilesInUse]; - int k = 0; - for (int y = tileGrid.y, ny = 0; ny < tileGrid.height; y++, ny++) { - for (int x = tileGrid.x, nx = 0; nx < tileGrid.width; x++, nx++) { - if (numWriters[nx][ny] > 0) { - indices[k++] = new Point(x, y); - } - } - } - } - - return indices; - } - - /** - * Tests if any tiles are currently checked out for writing. - * @return {@code true} if any tiles are currently checked out; - * {@code false} otherwise - */ - public boolean hasTileWriters() { - return numTilesInUse > 0; - } - - - /** - * Gets the image value for the given image position as an integer. - * - * @param x X ordinate - * @param y Y ordinate - * @param b band index - * - * @return image value - * @throws PixelOutsideImageException - */ - public int getSample(int x, int y, int b) throws PixelOutsideImageException { - int tileX = XToTileX(x); - int tileY = YToTileY(y); - Raster t = getTile(tileX, tileY); - if (t == null) { - throw new PixelOutsideImageException(x, y, b); - } - return t.getSample(x, y, b); - } - - /** - * Gets the image value for the given image position as a float. - * - * @param x X ordinate - * @param y Y ordinate - * @param b band index - * - * @return image value - * @throws PixelOutsideImageException - */ - public float getSampleFloat(int x, int y, int b) throws PixelOutsideImageException { - int tileX = XToTileX(x); - int tileY = YToTileY(y); - Raster t = getTile(tileX, tileY); - if (t == null) { - throw new PixelOutsideImageException(x, y, b); - } - return t.getSampleFloat(x, y, b); - } - - /** - * Gets the image value for the given image position as a double. - * - * @param x X ordinate - * @param y Y ordinate - * @param b band index - * - * @return image value - * @throws PixelOutsideImageException - */ - public double getSampleDouble(int x, int y, int b) throws PixelOutsideImageException { - int tileX = XToTileX(x); - int tileY = YToTileY(y); - Raster t = getTile(tileX, tileY); - if (t == null) { - throw new PixelOutsideImageException(x, y, b); - } - return t.getSampleDouble(x, y, b); - } - - /** - * Sets the image value for the given image position as an integer. - * - * @param x X ordinate - * @param y Y ordinate - * @param b band index - * @param value the new value - * - * @throws PixelOutsideImageException - */ - public void setSample(int x, int y, int b, int value) throws PixelOutsideImageException { - int tileX = XToTileX(x); - int tileY = YToTileY(y); - try { - WritableRaster t = getWritableTile(tileX, tileY); - if (t == null) { - throw new PixelOutsideImageException(x, y, b); - } - t.setSample(x, y, b, value); - - } finally { - releaseWritableTile(tileX, tileY); - } - } - - /** - * Sets the image value for the given image position as a float. - * - * @param x X ordinate - * @param y Y ordinate - * @param b band index - * @param value the new value - * @throws PixelOutsideImageException - * - */ - public void setSample(int x, int y, int b, float value) throws PixelOutsideImageException { - int tileX = XToTileX(x); - int tileY = YToTileY(y); - try { - WritableRaster t = getWritableTile(tileX, tileY); - if (t == null) { - throw new PixelOutsideImageException(x, y, b); - } - t.setSample(x, y, b, value); - - } finally { - releaseWritableTile(tileX, tileY); - } - } - - /** - * Sets the image value for the given image position as a double. - * - * @param x X ordinate - * @param y Y ordinate - * @param b band index - * @param value the new value - * @throws PixelOutsideImageException - * - */ - public void setSample(int x, int y, int b, double value) throws PixelOutsideImageException { - int tileX = XToTileX(x); - int tileY = YToTileY(y); - try { - WritableRaster t = getWritableTile(tileX, tileY); - if (t == null) { - throw new PixelOutsideImageException(x, y, b); - } - t.setSample(x, y, b, value); - - } finally { - releaseWritableTile(tileX, tileY); - } - } - - /** - * Copies data from the given {@code Raster} object into this - * image. The bounds of {@code data} will be used to - * place the data and only that portion of {@code data} - * within this image's bounds will be copied. - * - * @param data the data to copy - * @throws IllegalArgumentException if {@code data} is {@code null} - */ - public void setData(Raster data) { - if (data == null) { - throw new IllegalArgumentException("The data argument must not be null"); - } - - Rectangle rBounds = data.getBounds(); - Rectangle common = rBounds.intersection(getBounds()); - if (common.isEmpty()) { - return; - } - - int minTileX = XToTileX(common.x); - int maxTileX = XToTileX(common.x + common.width - 1); - int minTileY = YToTileY(common.y); - int maxTileY = YToTileY(common.y + common.height - 1); - - for (int y = minTileY; y <= maxTileY; y++) { - for (int x = minTileX; x <= maxTileX; x++) { - WritableRaster tile = getWritableTile(x, y); - Rectangle tileOverlap = tile.getBounds().intersection(common); - - Raster dataChild = data.createChild( - tileOverlap.x, tileOverlap.y, - tileOverlap.width, tileOverlap.height, - tileOverlap.x, tileOverlap.y, - null); - - WritableRaster tChild = tile.createWritableChild( - tileOverlap.x, tileOverlap.y, - tileOverlap.width, tileOverlap.height, - tileOverlap.x, tileOverlap.y, - null); - - tChild.setRect(dataChild); - } - } - } - - /** - * Creates a Graphics2D object for drawing operations with this image. - * The graphics object will be an instance of {@code DiskMemImageGraphics}. - *

- * Note that only images of integral data type support graphics operations. - * - * @return a new Graphics2D object - * @throws UnsupportedOperationException if the image is not of integral data type - * @see DiskMemImageGraphics - */ - public Graphics2D createGraphics() { - int dataType = getSampleModel().getDataType(); - if (dataType == DataBuffer.TYPE_BYTE || - dataType == DataBuffer.TYPE_INT || - dataType == DataBuffer.TYPE_SHORT || - dataType == DataBuffer.TYPE_USHORT) - { - return new DiskMemImageGraphics(this); - - } else { - throw new UnsupportedOperationException("Image must have an integral data type"); - } - } - - /** - * Gets the maximum amount of memory, in bytes, that this - * image will use for in-memory storage of its data. Since - * this class uses a {@code DiskMemTileCache} this is not - * the limit of image size. - * - * @return max memory usage - */ - public long getMemoryCapacity() { - return getTileCache().getMemoryCapacity(); - } - - /** - * Gets the amount of memory (bytes) required to store a single - * tile's data. - * - * @return tile memory size in bytes - */ - public long getTileMemorySize() { - return tileMemorySize; - } - - /** - * Sets whether this image will use the common tile cache. Any tiles - * belonging to this image that are already cached will be transferred - * from the image's current tile cache to the common cache (if {@code useCommon} - * is {@code true}) or to a new exclusive tile cache ((if {@code useCommon} - * is {@code false}). - *

- * By default, the image will use an exclusive cache. - * - * @param useCommon true to set this image to use the common tile cache; false - * for the image to use its own exclusive cache. - * - * @see #getCommonTileCache() - * @see #isUsingCommonCache() - */ - public void setUseCommonCache(boolean useCommon) { - if (useCommon && !isUsingCommonCache()) { - /* - * transfer any existing tiles to the common cache - */ - if (tileCache != null) { - for (int y = getMinTileY(), ny = 0; ny < getNumYTiles(); y++, ny++) { - for (int x = getMinTileX(), nx = 0; nx < getNumXTiles(); x++, nx++) { - Raster tile = tileCache.getTile(this, x, y); - if (tile != null) { - getCommonTileCache().add(this, x, y, (WritableRaster)tile); - tileCache.remove(this, x, y); - } - } - } - } - tileCache = getCommonTileCache(); - - } else if (isUsingCommonCache()) { - /* - * transfer any existing tiles from the common cache to - * this images new exclusive cache - */ - DiskMemTileCache newCache = createNewCache(); - for (int y = getMinTileY(), ny = 0; ny < getNumYTiles(); y++, ny++) { - for (int x = getMinTileX(), nx = 0; nx < getNumXTiles(); x++, nx++) { - Raster tile = getCommonTileCache().getTile(this, x, y); - if (tile != null) { - newCache.add(this, x, y, (WritableRaster)tile); - getCommonTileCache().remove(this, x, y); - } - } - } - tileCache = newCache; - } - } - - /** - * Gets a reference to the {@code DiskMemTileCache} instance - * used by this image. This method is intended for client - * code that wishes to query cache state or receive cache diagnostic - * messages (via the {@code Observer} interface). It is probably not - * a good idea to manipulate the cache state directly. - * - * @return a live reference to the cache being used by this image - * @see #isUsingCommonCache() - * @see DiskMemTileCache - */ - public DiskMemTileCache getTileCache() { - if (tileCache == null) { - tileCache = createNewCache(); - } - - return tileCache; - } - - /** - * Tests if this image is using the common tile cache. - * - * @return {@code true} if using the common tile cache; {@code false} otherwise - * @see #setUseCommonCache(boolean) - */ - public boolean isUsingCommonCache() { - return tileCache != null && tileCache == commonCache; - } - - /** - * Creates a new image tile - * @param tileX tile X ordinate - * @param tileY tile Y ordinate - * @return the new tile - */ - private WritableRaster createTile(int tileX, int tileY) { - assert(getTileCache().getTile(this, tileX, tileY) == null); - - Point location = new Point(tileXToX(tileX), tileYToY(tileY)); - return createWritableRaster(getSampleModel(), location); - } - -} diff --git a/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java b/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java deleted file mode 100644 index c792c6886..000000000 --- a/klab.engine/src/main/java/org/jaitools/tiledimage/DiskMemImageGraphics.java +++ /dev/null @@ -1,1027 +0,0 @@ -/* - * Copyright (c) 2009-2011, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tiledimage; - -import java.awt.Color; -import java.awt.Composite; -import java.awt.Font; -import java.awt.FontMetrics; -import java.awt.Graphics; -import java.awt.Graphics2D; -import java.awt.GraphicsConfiguration; -import java.awt.Image; -import java.awt.Paint; -import java.awt.Point; -import java.awt.Rectangle; -import java.awt.RenderingHints; -import java.awt.RenderingHints.Key; -import java.awt.Shape; -import java.awt.Stroke; -import java.awt.font.FontRenderContext; -import java.awt.font.GlyphVector; -import java.awt.geom.AffineTransform; -import java.awt.geom.Area; -import java.awt.geom.NoninvertibleTransformException; -import java.awt.geom.Point2D; -import java.awt.geom.Rectangle2D; -import java.awt.image.BufferedImage; -import java.awt.image.BufferedImageOp; -import java.awt.image.ColorModel; -import java.awt.image.ImageObserver; -import java.awt.image.Raster; -import java.awt.image.RenderedImage; -import java.awt.image.SampleModel; -import java.awt.image.WritableRaster; -import java.awt.image.renderable.RenderableImage; -import java.lang.reflect.Method; -import java.text.AttributedCharacterIterator; -import java.util.Hashtable; -import java.util.Map; - -import org.eclipse.imagen.PlanarImage; - -/** - * A Graphics class for drawing into a DiskMemImage. - * As with JAI's TiledImageGraphics class, java.awt - * routines do the work and the purpose of this class is to - * serve the image data in a form that those routines can handle. - *

- * Most of the methods in this class are identical in function to - * those in Graphics2D; these have not been documented here. - * - * @see DiskMemImage - * - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class DiskMemImageGraphics extends Graphics2D { - - private DiskMemImage targetImage; - private ColorModel colorModel; - private Hashtable properties; - private RenderingHints renderingHints; - - /** - * Constants for paint mode: PAINT or XOR. - */ - public static enum PaintMode { - /** Simple paint mode. */ - PAINT, - /** XOR paint mode. */ - XOR; - }; - - /* - * java.awt.Graphics parameters - */ - private Point origin; - private Shape clip; - private Color color; - private Font font; - private PaintMode paintMode; - private Color XORColor; - - /* - * java.awt.Graphics2D parameters - */ - private Color background; - private Composite composite; - private Paint paint; - private Stroke stroke; - private AffineTransform transform; - - /** - * Constants and associated data for graphics operations - */ - public static enum OpType { - - /** Describes the clearRect method. */ - CLEAR_RECT("clearRect", int.class, int.class, int.class, int.class), - - /** Describes the copyArea method. */ - COPY_AREA("copyArea", int.class, int.class, int.class, int.class, int.class, int.class), - - /** Describes the drawArc method. */ - DRAW_ARC("drawArc", int.class, int.class, int.class, int.class, int.class, int.class), - - /** Describes the drawImage method. */ - DRAW_BUFFERED_IMAGE("drawImage", BufferedImage.class, BufferedImageOp.class, int.class, int.class), - - /** Describes the drawGlyphVector method. */ - DRAW_GLYPH_VECTOR("drawGlyphVector", GlyphVector.class, float.class, float.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_DEST_SRC("drawImage", Image.class, int.class, int.class, int.class, int.class, - int.class, int.class, int.class, int.class, ImageObserver.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_DEST_SRC_COL("drawImage", Image.class, int.class, int.class, int.class, int.class, - int.class, int.class, int.class, int.class, Color.class, ImageObserver.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_TRANSFORM("drawImage", Image.class, AffineTransform.class, ImageObserver.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_XY("drawImage", Image.class, int.class, int.class, ImageObserver.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_XY_COL("drawImage", Image.class, int.class, int.class, Color.class, ImageObserver.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_XYWH("drawImage", Image.class, int.class, int.class, int.class, int.class, ImageObserver.class), - - /** Describes the drawImage method. */ - DRAW_IMAGE_XYWH_COL("drawImage", Image.class, int.class, int.class, int.class, int.class, Color.class, ImageObserver.class), - - /** Describes the drawLine method. */ - DRAW_LINE("drawLine", int.class, int.class, int.class, int.class), - - /** Describes the drawOval method. */ - DRAW_OVAL("drawOval", int.class, int.class, int.class, int.class), - - /** Describes the drawPolygon method. */ - DRAW_POLYGON("drawPolygon", int[].class, int[].class, int.class), - - /** Describes the drawPolyline method. */ - DRAW_POLYLINE("drawPolyline", int[].class, int[].class, int.class), - - /** Describes the drawRenderableImage method. */ - DRAW_RENDERABLE_IMAGE("drawRenderableImage", RenderableImage.class, AffineTransform.class), - - /** Describes the drawRenderedImage method. */ - DRAW_RENDERED_IMAGE("drawRenderedImage", RenderedImage.class, AffineTransform.class), - - /** Describes the drawRoundRect method. */ - DRAW_ROUND_RECT("drawRoundRect", int.class, int.class, int.class, int.class, int.class, int.class), - - /** Describes the draw method. */ - DRAW_SHAPE("draw", Shape.class), - - /** Describes the drawString method. */ - DRAW_STRING_XY("drawString", String.class, float.class, float.class), - - /** Describes the drawString method. */ - DRAW_STRING_ITER_XY("drawString", AttributedCharacterIterator.class, float.class, float.class), - - /** Describes the fill method. */ - FILL("fill", Shape.class), - - /** Describes the fillArc method. */ - FILL_ARC("fillArc", int.class, int.class, int.class, int.class, int.class, int.class), - - /** Describes the fillOval method. */ - FILL_OVAL("fillOval", int.class, int.class, int.class, int.class), - - /** Describes the fillPolygon method. */ - FILL_POLYGON("fillPolygon", int[].class, int[].class, int.class), - - /** Describes the fillRect method. */ - FILL_RECT("fillRect", int.class, int.class, int.class, int.class), - - /** Describes the fillRoundRect method. */ - FILL_ROUND_RECT("fillRoundRect", int.class, int.class, int.class, int.class, int.class, int.class); - - - private String methodName; - private Class[] paramTypes; - - private OpType(String methodName, Class ...types) { - this.methodName = methodName; - this.paramTypes = new Class[types.length]; - System.arraycopy(types, 0, this.paramTypes, 0, types.length); - } - - /** - * Gets the method name. - * @return method name - */ - public String getMethodName() { - return methodName; - } - - /** - * Gets the full method name. - * @return full name - */ - public String getFullMethodName() { - StringBuilder sb = new StringBuilder(); - sb.append(methodName); - sb.append("("); - if (paramTypes.length > 0) { - sb.append(paramTypes[0].getSimpleName()); - for (int i = 1; i < paramTypes.length; i++) { - sb.append(", "); - sb.append(paramTypes[i].getSimpleName()); - } - } - sb.append(")"); - return sb.toString(); - } - - /** - * Gets the number of method arguments - * @return number of arguments - */ - public int getNumArgs() { - return paramTypes.length; - } - - /** - * Gets the types of the arguments. - * @return argument types - */ - public Class[] getArgTypes() { - Class[] copy = new Class[ paramTypes.length]; - System.arraycopy(paramTypes, 0, copy, 0, paramTypes.length); - return copy; - } - } - - /** - * Creates an instance for the given target image - * - * @param targetImage the image to be drawn into - */ - DiskMemImageGraphics(DiskMemImage targetImage) { - this.targetImage = targetImage; - this.renderingHints = new RenderingHints(null); - setColorModel(); - setProperties(); - setGraphicsParams(); - } - - @Override - public void draw(Shape s) { - doDraw(OpType.DRAW_SHAPE, correctForStroke(s.getBounds2D()), s); - } - - @Override - public boolean drawImage(Image img, AffineTransform xform, ImageObserver obs) { - Rectangle2D bounds = new Rectangle(0, 0, img.getWidth(obs), img.getHeight(obs)); - Rectangle2D trBounds = xform.createTransformedShape(bounds).getBounds2D(); - - return doDraw(OpType.DRAW_IMAGE_TRANSFORM, trBounds, img, xform, obs); - } - - @Override - public void drawImage(BufferedImage img, BufferedImageOp op, int x, int y) { - Rectangle2D bounds = op.getBounds2D(img); - doDraw(OpType.DRAW_BUFFERED_IMAGE, bounds, img, op, x, y); - } - - @Override - public void drawRenderedImage(RenderedImage img, AffineTransform xform) { - Rectangle2D bounds = new Rectangle(img.getMinX(), img.getMinY(), - img.getWidth(), img.getHeight()); - - Rectangle2D trBounds = xform.createTransformedShape(bounds).getBounds2D(); - - doDraw(OpType.DRAW_RENDERED_IMAGE, trBounds, img, xform); - } - - @Override - public void drawRenderableImage(RenderableImage img, AffineTransform xform) { - Rectangle2D bounds = new Rectangle2D.Float(img.getMinX(), img.getMinY(), - img.getWidth(), img.getHeight()); - - Rectangle2D trBounds = xform.createTransformedShape(bounds).getBounds2D(); - - doDraw(OpType.DRAW_RENDERABLE_IMAGE, trBounds, img, xform); - } - - @Override - public void drawString(String str, int x, int y) { - drawString( str, (float)x, (float)y ); - } - - @Override - public void drawString(String str, float x, float y) { - Rectangle2D bounds = getFontMetrics().getStringBounds(str, this); - bounds.setRect(x, - y - bounds.getHeight() + 1, - bounds.getWidth(), - bounds.getHeight() ); - - doDraw(OpType.DRAW_STRING_XY, bounds, str, x, y); - } - - @Override - public void drawString(AttributedCharacterIterator iter, int x, int y) { - drawString( iter, (float)x, (float)y ); - } - - @Override - public void drawString(AttributedCharacterIterator iter, float x, float y) { - Rectangle2D bounds = getFontMetrics().getStringBounds( - iter, iter.getBeginIndex(), iter.getEndIndex(), this); - - bounds.setRect(x, - y - bounds.getHeight() + 1, - bounds.getWidth(), - bounds.getHeight() ); - - doDraw(OpType.DRAW_STRING_ITER_XY, bounds, iter, x, y); - } - - @Override - public void drawGlyphVector(GlyphVector g, float x, float y) { - Rectangle2D bounds = g.getVisualBounds(); - - // expand the rectangle by a single pixel width to allow for - // rasterizing round-off errors - // (see apidocs for Graphics2D.drawGlyphVector) - bounds.setFrame(bounds.getMinX() - 1, bounds.getMinY() - 1, - bounds.getWidth() + 2, bounds.getHeight() + 2); - - doDraw(OpType.DRAW_GLYPH_VECTOR, bounds, g, x, y); - } - - @Override - public void fill(Shape s) { - doDraw(OpType.FILL, s.getBounds2D(), s); - } - - @Override - public boolean hit(Rectangle rect, Shape s, boolean onStroke) { - Graphics2D gr = getProxy(); - copyGraphicsParams(gr); - boolean rtnVal = gr.hit(rect, s, onStroke); - gr.dispose(); - return rtnVal; - } - - @Override - public GraphicsConfiguration getDeviceConfiguration() { - Graphics2D gr = getProxy(); - copyGraphicsParams(gr); - GraphicsConfiguration gc = gr.getDeviceConfiguration(); - gr.dispose(); - return gc; - } - - @Override - public void setComposite(Composite comp) { - composite = comp; - } - - @Override - public void setPaint(Paint p) { - paint = p; - } - - @Override - public void setStroke(Stroke s) { - stroke = s; - } - - @Override - public void setRenderingHint(Key hintKey, Object hintValue) { - renderingHints.put(hintKey, hintValue); - } - - @Override - public Object getRenderingHint(Key hintKey) { - return renderingHints.get(hintKey); - } - - @Override - public void setRenderingHints(Map hints) { - renderingHints = new RenderingHints(null); - renderingHints.putAll(hints); - } - - @Override - public void addRenderingHints(Map hints) { - renderingHints.putAll(hints); - } - - @Override - public RenderingHints getRenderingHints() { - return renderingHints; - } - - @Override - public void translate(int x, int y) { - origin.setLocation(x, y); - transform.translate( (double)x, (double)y ); - } - - @Override - public void translate(double tx, double ty) { - transform.translate(tx, ty); - } - - @Override - public void rotate(double theta) { - transform.rotate(theta); - } - - @Override - public void rotate(double theta, double x, double y) { - transform.rotate(theta, x, y); - } - - @Override - public void scale(double sx, double sy) { - transform.scale(sx, sy); - } - - @Override - public void shear(double shx, double shy) { - transform.shear(shx, shy); - } - - @Override - public void transform(AffineTransform Tx) { - transform.concatenate(Tx); - } - - @Override - public void setTransform(AffineTransform Tx) { - transform = Tx; - } - - @Override - public AffineTransform getTransform() { - return transform; - } - - @Override - public Paint getPaint() { - return paint; - } - - @Override - public Composite getComposite() { - return composite; - } - - @Override - public void setBackground(Color color) { - background = color; - } - - @Override - public Color getBackground() { - return background; - } - - @Override - public Stroke getStroke() { - return stroke; - } - - @Override - public void clip(Shape s) { - if(clip == null) { - clip = s; - } else { - Area clipArea = (clip instanceof Area ? (Area)clip : new Area(clip)); - clipArea.intersect(s instanceof Area ? (Area)s : new Area(s)); - clip = clipArea; - } - } - - @Override - public FontRenderContext getFontRenderContext() { - Graphics2D gr = getProxy(); - copyGraphicsParams(gr); - FontRenderContext frc = gr.getFontRenderContext(); - gr.dispose(); - return frc; - } - - /** - * Returns a copy of this object with its current settings - * @return a new instance of this class - */ - @Override - public Graphics create() { - DiskMemImageGraphics gr = new DiskMemImageGraphics(targetImage); - copyGraphicsParams(gr); - return gr; - } - - @Override - public Color getColor() { - return color; - } - - @Override - public void setColor(Color c) { - color = c; - } - - @Override - public void setPaintMode() { - paintMode = PaintMode.PAINT; - } - - @Override - public void setXORMode(Color color) { - paintMode = PaintMode.XOR; - XORColor = color; - } - - @Override - public Font getFont() { - return font; - } - - @Override - public void setFont(Font font) { - if (font != null) { - this.font = font; - } - } - - @Override - public FontMetrics getFontMetrics(Font f) { - Graphics2D gr = getProxy(); - copyGraphicsParams(gr); - FontMetrics fm = gr.getFontMetrics(f); - gr.dispose(); - return fm; - } - - @Override - public Rectangle getClipBounds() { - return clip.getBounds(); - } - - @Override - public void clipRect(int x, int y, int width, int height) { - clip(new Rectangle(x, y, width, height)); - } - - @Override - public void setClip(int x, int y, int width, int height) { - setClip(new Rectangle(x, y, width, height)); - } - - @Override - public Shape getClip() { - return clip; - } - - @Override - public void setClip(Shape clip) { - this.clip = clip; - } - - @Override - public void copyArea(int x, int y, int width, int height, int dx, int dy) { - doDraw(OpType.COPY_AREA, - new Rectangle(x + dx, y + dy, width, height), - x, y, width, height, dx, dy); - } - - @Override - public void drawLine(int x1, int y1, int x2, int y2) { - Rectangle2D bounds = new Rectangle(); - bounds.setFrameFromDiagonal(x1, y1, x2, y2); - doDraw(OpType.DRAW_LINE, correctForStroke(bounds), x1, y1, x2, y2); - } - - @Override - public void fillRect(int x, int y, int width, int height) { - doDraw(OpType.FILL_RECT, new Rectangle(x, y, width, height), x, y, width, height); - } - - @Override - public void clearRect(int x, int y, int width, int height) { - doDraw(OpType.CLEAR_RECT, new Rectangle(x, y, width, height), x, y, width, height); - } - - @Override - public void drawRoundRect(int x, int y, int width, int height, int arcWidth, int arcHeight) { - Rectangle2D bounds = new Rectangle( - x - arcWidth, y - arcHeight, - width + 2 * arcWidth, height + 2 * arcHeight); - - doDraw(OpType.DRAW_ROUND_RECT, correctForStroke(bounds), - x, y, width, height, arcWidth, arcHeight); - } - - @Override - public void fillRoundRect(int x, int y, int width, int height, int arcWidth, int arcHeight) { - Rectangle2D bounds = new Rectangle( - x - arcWidth, y - arcHeight, - width + 2 * arcWidth, height + 2 * arcHeight); - - doDraw(OpType.FILL_ROUND_RECT, bounds, x, y, width, height, arcWidth, arcHeight); - } - - @Override - public void drawOval(int x, int y, int width, int height) { - Rectangle2D bounds = new Rectangle(x, y, width, height); - doDraw(OpType.DRAW_OVAL, correctForStroke(bounds), x, y, width, height); - } - - @Override - public void fillOval(int x, int y, int width, int height) { - Rectangle2D bounds = new Rectangle(x, y, width, height); - doDraw(OpType.FILL_OVAL, bounds, x, y, width, height); - } - - @Override - public void drawArc(int x, int y, int width, int height, int startAngle, int arcAngle) { - Rectangle2D bounds = new Rectangle(x, y, width, height); - doDraw(OpType.DRAW_ARC, correctForStroke(bounds), x, y, width, height, startAngle, arcAngle); - } - - @Override - public void fillArc(int x, int y, int width, int height, int startAngle, int arcAngle) { - Rectangle2D bounds = new Rectangle(x, y, width, height); - doDraw(OpType.FILL_ARC, bounds, x, y, width, height, startAngle, arcAngle); - } - - @Override - public void drawPolyline(int[] xPoints, int[] yPoints, int nPoints) { - doDraw(OpType.DRAW_POLYLINE, - correctForStroke(getPolyBounds(xPoints, yPoints, nPoints)), - xPoints, yPoints, nPoints); - } - - @Override - public void drawPolygon(int[] xPoints, int[] yPoints, int nPoints) { - doDraw(OpType.DRAW_POLYGON, - correctForStroke(getPolyBounds(xPoints, yPoints, nPoints)), - xPoints, yPoints, nPoints); - } - - @Override - public void fillPolygon(int[] xPoints, int[] yPoints, int nPoints) { - doDraw(OpType.FILL_POLYGON, - getPolyBounds(xPoints, yPoints, nPoints), - xPoints, yPoints, nPoints); - } - - @Override - public boolean drawImage(Image img, int x, int y, ImageObserver obs) { - Rectangle2D bounds = new Rectangle(x, y, img.getWidth(obs), img.getHeight(obs)); - return doDraw(OpType.DRAW_IMAGE_XY, bounds, img, x, y, obs); - } - - @Override - public boolean drawImage(Image img, int x, int y, int width, int height, ImageObserver obs) { - Rectangle2D bounds = new Rectangle(x, y, width, height); - return doDraw(OpType.DRAW_IMAGE_XYWH, bounds, img, x, y, width, height, obs); - } - - @Override - public boolean drawImage(Image img, int x, int y, Color bgcolor, ImageObserver obs) { - Rectangle2D bounds = new Rectangle(x, y, img.getWidth(obs), img.getHeight(obs)); - return doDraw(OpType.DRAW_IMAGE_XY_COL, bounds, img, x, y, bgcolor, obs); - } - - @Override - public boolean drawImage(Image img, - int x, int y, int width, int height, - Color bgcolor, ImageObserver obs) { - - Rectangle2D bounds = new Rectangle(x, y, width, height); - return doDraw(OpType.DRAW_IMAGE_XYWH_COL, bounds, img, x, y, width, height, bgcolor, obs); - } - - @Override - public boolean drawImage(Image img, - int dx1, int dy1, int dx2, int dy2, - int sx1, int sy1, int sx2, int sy2, - ImageObserver obs) { - - Rectangle2D bounds = new Rectangle(dx1, dy1, dx2-dx1+1, dy2-dy1+1); - return doDraw(OpType.DRAW_IMAGE_DEST_SRC, bounds, img, dx1, dy1, dx2, dy2, sx1, sy1, sx2, sy2, obs); - } - - @Override - public boolean drawImage(Image img, - int dx1, int dy1, int dx2, int dy2, - int sx1, int sy1, int sx2, int sy2, - Color bgcolor, ImageObserver obs) { - - Rectangle2D bounds = new Rectangle(dx1, dy1, dx2-dx1+1, dy2-dy1+1); - return doDraw(OpType.DRAW_IMAGE_DEST_SRC_COL, bounds, img, - dx1, dy1, dx2, dy2, sx1, sy1, sx2, sy2, - bgcolor, obs); - } - - @Override - public void dispose() { - /* - * No need to do anything here - */ - } - - /** - * Performs the graphics operation by partitioning the work across the image's - * tiles and using Graphics2D routines to draw into each tile. - * - * @param opType the type of operation - * @param bounds bounds of the element to be drawn - * @param args a variable length list of arguments for the operation - */ - private boolean doDraw(OpType opType, Rectangle2D bounds, Object ...args) { - Method method = null; - boolean rtnVal = false; - - try { - method = Graphics2D.class.getMethod(opType.getMethodName(), opType.getArgTypes()); - - } catch (NoSuchMethodException nsmEx) { - // programmer error :-( - throw new RuntimeException("No such method: " + opType.getFullMethodName()); - } - - int minTileX = Math.max(targetImage.XToTileX((int)bounds.getMinX()), - targetImage.getMinTileX()); - - int maxTileX = Math.min(targetImage.XToTileX((int)(bounds.getMaxX() + 0.5)), - targetImage.getMaxTileX()); - - int minTileY = Math.max(targetImage.YToTileY((int)bounds.getMinY()), - targetImage.getMinTileY()); - - int maxTileY = Math.min(targetImage.YToTileY((int)(bounds.getMaxY() + 0.5)), - targetImage.getMaxTileY()); - - for (int tileY = minTileY; tileY <= maxTileY; tileY++) { - int minY = targetImage.tileYToY(tileY); - - for (int tileX = minTileX; tileX <= maxTileX; tileX++) { - int minX = targetImage.tileXToX(tileX); - - WritableRaster tile = targetImage.getWritableTile(tileX, tileY); - - // create a live-copy of the tile with the upper-left corner - // translated to 0,0 - WritableRaster copy = tile.createWritableTranslatedChild(0, 0); - - BufferedImage bufImg = new BufferedImage( - colorModel, - copy, - colorModel.isAlphaPremultiplied(), - properties); - - Graphics2D gr = bufImg.createGraphics(); - - // Note: we use the version of copyGraphicsParams taking a - // Point arg used to adjust the clip area before copying it - // into the graphics object - copyGraphicsParams(gr, new Point(minX, minY)); - - try { - Point2D p2d = gr.getTransform().transform(new Point2D.Double(0, 0), null); - Point p = new Point((int)p2d.getX() - minX, (int)p2d.getY() - minY); - p2d = gr.getTransform().inverseTransform(p, null); - gr.translate(p2d.getX(), p2d.getY()); - - } catch(NoninvertibleTransformException nte) { - // TODO replace this with decent error handling - throw new RuntimeException(nte); - } - - try { - Object oRtnVal = method.invoke(gr, args); - if(oRtnVal != null && oRtnVal.getClass() == boolean.class) { - rtnVal = ((Boolean)oRtnVal).booleanValue(); - } - - } catch(Exception ex) { - // TODO replace this with decent error handling - throw new RuntimeException(ex); - } - - gr.dispose(); - targetImage.releaseWritableTile(tileX, tileY); - } - } - - return rtnVal; - } - - /** - * Takes a bounding rectangle calculated by - * one of the drawing methods and expands it, if necessary, to - * account for the current Stroke width. - *

- * This correction appears to be missing from JAI's TiledImageGraphics - * class. - * - * @param rect input bounds - * @return expanded bounds as a new Rectangle2D object; or the - * input Rectangle2D if no stroke is set - */ - private Rectangle2D correctForStroke(Rectangle2D rect) { - if (stroke != null) { - Shape shp = stroke.createStrokedShape(rect); - return shp.getBounds2D(); - } else { - return rect; - } - } - - /** - * Gets the bounding rectangle of the given vertices. - * - * @param xPoints X ordinates - * @param yPoints Y ordinates - * @param nPoints number of vertices - * @return the bounding rectangle - */ - private Rectangle2D getPolyBounds(int[] xPoints, int[] yPoints, int nPoints) { - Rectangle bounds = new Rectangle(); - - if (nPoints > 0) { - int minX = xPoints[0]; - int maxX = minX; - int minY = yPoints[0]; - int maxY = minY; - - for (int i = 1; i < nPoints; i++) { - if (xPoints[i] < minX) { - minX = xPoints[i]; - } else if (xPoints[i] > maxX) { - maxX = xPoints[i]; - } - - if (yPoints[i] < minY) { - minY = yPoints[i]; - } else if (yPoints[i] > maxY) { - maxY = yPoints[i]; - } - } - - bounds.setBounds(minX, minY, maxX - minX + 1, maxY - minY + 1); - } - - return bounds; - } - - /** - * Attempts to retrieve or create a ColorModel for the target - * image. - * - * @throws UnsupportedOperationException if a compatible - * ColorModel is not found - */ - private void setColorModel() { - assert(targetImage != null); - - colorModel = targetImage.getColorModel(); - - if (colorModel == null) { - SampleModel sm = targetImage.getSampleModel(); - colorModel = PlanarImage.createColorModel(sm); - - if (colorModel == null) { - // try simple default - if (ColorModel.getRGBdefault().isCompatibleSampleModel(sm)) { - colorModel = ColorModel.getRGBdefault(); - - } else { - // admit defeat - throw new UnsupportedOperationException( - "Failed to get or construct a ColorModel for the image"); - } - } - } - } - - /** - * Retrieves any properties set for the target image. - */ - private void setProperties() { - assert(targetImage != null); - - properties = new Hashtable(); - String[] propertyNames = targetImage.getPropertyNames(); - if (propertyNames != null) { - for (String name : propertyNames) { - properties.put(name, targetImage.getProperty(name)); - } - } - - // TODO: set rendering hints - } - - /** - * Creates a Graphics2D instance based on the first tile of the - * target image and uses its state to set the graphics parameters - * of this instance. - */ - private void setGraphicsParams() { - assert(targetImage != null); - assert(colorModel != null); - assert(properties != null); - - Graphics2D gr = getProxy(); - - origin = new Point(0, 0); - clip = targetImage.getBounds(); - color = gr.getColor(); - font = gr.getFont(); - - paintMode = PaintMode.PAINT; - XORColor = null; - - background = gr.getBackground(); - composite = gr.getComposite(); - paint = null; - stroke = gr.getStroke(); - transform = gr.getTransform(); - - gr.dispose(); - } - - /** - * Copies the current graphics parameters into the given Graphics2D - * object. - * - * @param gr a Graphics2D object - */ - private void copyGraphicsParams(Graphics2D gr) { - copyGraphicsParams(gr, null); - } - - /** - * Copies the current graphics parameters into the given Graphics2D - * object. If {@code workingOrigin} is non-null it is used to translate the - * clip area before copying it across. - * - * @param gr a Graphics2D object - */ - private void copyGraphicsParams(Graphics2D gr, Point workingOrigin) { - gr.translate(origin.x, origin.y); - gr.setColor(getColor()); - - if (workingOrigin == null) { - gr.setClip(clip); - } else { - AffineTransform tr = AffineTransform.getTranslateInstance( - -workingOrigin.x, -workingOrigin.y); - Shape trclip = tr.createTransformedShape(clip); - gr.setClip(trclip); - } - - if(paintMode == PaintMode.PAINT) { - gr.setPaintMode(); - } else if (XORColor != null) { - gr.setXORMode(XORColor); - } - - gr.setFont(font); - - // java.awt.Graphics2D state - gr.setBackground(background); - gr.setComposite(composite); - if(paint != null) { - gr.setPaint(paint); - } - if (renderingHints != null) { - gr.setRenderingHints(renderingHints); - } - gr.setStroke(stroke); - gr.setTransform(transform); - } - - /** - * Helper method for other methods that need an instantiated - * Graphics2D object that is 'representative' of the target image. - * - * @return a new Graphics2D instance - */ - private Graphics2D getProxy() { - Raster tile = targetImage.getTile(targetImage.getMinTileX(), targetImage.getMinTileY()); - WritableRaster tiny = tile.createCompatibleWritableRaster(1, 1); - - BufferedImage img = new BufferedImage( - colorModel, tiny, colorModel.isAlphaPremultiplied(), properties); - - return img.createGraphics(); - } -} diff --git a/klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java b/klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java deleted file mode 100644 index 5d12d14be..000000000 --- a/klab.engine/src/main/java/org/jaitools/tiledimage/PixelOutsideImageException.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2009, Michael Bedward. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.jaitools.tiledimage; - -/** - * Exception thrown when a given pixel / band location is outside an - * image's bounds - * - * @author Michael Bedward - * @since 1.0 - * @version $Id$ - */ -public class PixelOutsideImageException extends IndexOutOfBoundsException { - - /** - * Creates a new instance. - * - * @param x pixel x coordinate - * @param y pixel y coordinate - * @param b band index - */ - PixelOutsideImageException(int x, int y, int b) { - super(String.format("Pixel location x=%d y=%d band=%d is outside image bounds", x, y, b)); - } - -} From c9bd14bc5de99a333d8bfa068a307c92b5b57dd6 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 17 Mar 2026 20:10:31 +0100 Subject: [PATCH 13/74] Add exclusion to HM to avoid poi NoSuchMethodException --- adapters/klab.ogc/pom.xml | 4 ++++ components/klab.component.hydrology/pom.xml | 4 ++++ klab.engine/pom.xml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/adapters/klab.ogc/pom.xml b/adapters/klab.ogc/pom.xml index 08cd69ca5..2ffa58486 100644 --- a/adapters/klab.ogc/pom.xml +++ b/adapters/klab.ogc/pom.xml @@ -60,6 +60,10 @@ org.mongodb mongo-java-driver + + edu.colostate.omslab + oms + diff --git a/components/klab.component.hydrology/pom.xml b/components/klab.component.hydrology/pom.xml index 8a42ff8f3..d66ec1140 100644 --- a/components/klab.component.hydrology/pom.xml +++ b/components/klab.component.hydrology/pom.xml @@ -28,6 +28,10 @@ org.mongodb mongo-java-driver + + edu.colostate.omslab + oms + diff --git a/klab.engine/pom.xml b/klab.engine/pom.xml index fc3b6f3bd..d3d8911e2 100644 --- a/klab.engine/pom.xml +++ b/klab.engine/pom.xml @@ -336,6 +336,10 @@ org.locationtech.jts jts-core + + edu.colostate.omslab + oms + From 6bd647f0fa85fb80619c90bf6acac91a61d3a47d Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 20 Mar 2026 12:16:55 +0100 Subject: [PATCH 14/74] Clean poms --- components/klab.component.hydrology/pom.xml | 5 +---- components/klab.component.tables/pom.xml | 18 +++++++++++++++--- klab.authentication/pom.xml | 1 - klab.engine/pom.xml | 2 -- pom.xml | 21 +++++++++++++++++++++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/components/klab.component.hydrology/pom.xml b/components/klab.component.hydrology/pom.xml index d66ec1140..b70b52f0d 100644 --- a/components/klab.component.hydrology/pom.xml +++ b/components/klab.component.hydrology/pom.xml @@ -28,10 +28,6 @@ org.mongodb mongo-java-driver - - edu.colostate.omslab - oms - @@ -57,5 +53,6 @@ org.integratedmodelling.klab.api ${klab.version} + \ No newline at end of file diff --git a/components/klab.component.tables/pom.xml b/components/klab.component.tables/pom.xml index b2066517c..93196a16c 100644 --- a/components/klab.component.tables/pom.xml +++ b/components/klab.component.tables/pom.xml @@ -36,9 +36,20 @@ - tech.tablesaw - tablesaw-excel - 0.38.5 + tech.tablesaw + tablesaw-excel + 0.38.5 + + + org.apache.poi + poi-ooxml + + + + + + org.apache.poi + poi-ooxml @@ -76,3 +87,4 @@ + diff --git a/klab.authentication/pom.xml b/klab.authentication/pom.xml index 25a62c91b..c4189aab0 100644 --- a/klab.authentication/pom.xml +++ b/klab.authentication/pom.xml @@ -36,7 +36,6 @@ commons-io commons-io - 2.14.0 diff --git a/klab.engine/pom.xml b/klab.engine/pom.xml index d3d8911e2..02863bd10 100644 --- a/klab.engine/pom.xml +++ b/klab.engine/pom.xml @@ -148,13 +148,11 @@ org.apache.poi poi - 5.4.1 org.apache.poi poi-ooxml - 5.4.1 diff --git a/pom.xml b/pom.xml index 839f76d2b..4799bd054 100644 --- a/pom.xml +++ b/pom.xml @@ -146,6 +146,8 @@ 2.0.9 0.1.23 24.0.4 + 5.4.1 + 2.14.0 @@ -293,4 +295,23 @@ + + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + commons-io + commons-io + ${commons-io.version} + + + From f478bcdf65f1ac7ed7701a785fc924155ed8f3b5 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 20 Mar 2026 16:04:06 +0100 Subject: [PATCH 15/74] [docker build] Add maven plugin to remove offensive library to jar Removed old commons-io from HM dependecy jar edu.colostate.omslab This is a temporary solution to enable deployment to proceed. Not working obviously in eclipse --- klab.engine/pom.xml | 5 ++--- products/cli/pom.xml | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/klab.engine/pom.xml b/klab.engine/pom.xml index 02863bd10..06339a4f6 100644 --- a/klab.engine/pom.xml +++ b/klab.engine/pom.xml @@ -334,10 +334,10 @@ org.locationtech.jts jts-core - + @@ -891,7 +891,6 @@ commons-io commons-io - 2.14.0 org.apache.commons diff --git a/products/cli/pom.xml b/products/cli/pom.xml index 37531dd15..46ea4497b 100644 --- a/products/cli/pom.xml +++ b/products/cli/pom.xml @@ -28,6 +28,29 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + strip-commons-io-from-oms + prepare-package + + + + + + + + + + + run + + + + com.google.cloud.tools jib-maven-plugin From 5793d036f1793bcd0e0e07ee8d252cc094169305 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 20 Mar 2026 16:27:07 +0100 Subject: [PATCH 16/74] =?UTF-8?q?[docker=20build]=20Use=20Ant=E2=80=99s=20?= =?UTF-8?q?built-in=20zip=20on=20archive=20tasks=20in=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- products/cli/pom.xml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/products/cli/pom.xml b/products/cli/pom.xml index 46ea4497b..1928942ed 100644 --- a/products/cli/pom.xml +++ b/products/cli/pom.xml @@ -38,11 +38,29 @@ prepare-package - - - - - + + + + + + + + + + + + + + From 47b35251c55fdc9266b7caa9220f60f9369660c4 Mon Sep 17 00:00:00 2001 From: Andrea Antonello Date: Thu, 26 Mar 2026 09:53:05 +0100 Subject: [PATCH 17/74] fix for shifter rasterized points --- .../geospace/processing/Rasterizer.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java index 8f294978e..8366a38ba 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java @@ -14,6 +14,7 @@ import org.geotools.geometry.jts.Geometries; import org.geotools.process.vector.VectorToRasterProcess; +import org.integratedmodelling.klab.api.observations.scale.space.Direction; import org.integratedmodelling.klab.api.observations.scale.space.IGrid; import org.integratedmodelling.klab.api.observations.scale.space.IShape; import org.integratedmodelling.klab.components.geospace.extents.Grid; @@ -88,8 +89,6 @@ public class Rasterizer { // this is used only for the getCoordinates() operation private BufferedImage coordinateRaster = null; private Graphics2D coordinateGraphics; - private VectorToRasterProcess gtp; - private GeometryFactory geoFactory = new GeometryFactory(); /** * @@ -309,10 +308,13 @@ private void drawGeometry(Geometry geometry, Color color) { coordGridX = new int[coords.length]; coordGridY = new int[coords.length]; } + double yRes = extent.getCellHeight(); + double xRes = extent.getCellWidth(); + double west = extent.getWest(); + double south = extent.getSouth(); for (int n = 0; n < coords.length; n++) { - coordGridX[n] = (int) (((coords[n].x - extent.getWest()) / extent.getCellWidth())); - coordGridY[n] = this.raster.getHeight() - - (int) (((coords[n].y - extent.getSouth()) / extent.getCellHeight())); + coordGridX[n] = (int) (((coords[n].x - west) / xRes)); + coordGridY[n] = this.raster.getHeight() - (int) (((coords[n].y - south) / yRes)); } if (geometry.getClass().equals(Polygon.class)) { @@ -322,7 +324,12 @@ private void drawGeometry(Geometry geometry, Color color) { } else if (geometry.getClass().equals(LineString.class)) { graphics.drawPolyline(coordGridX, coordGridY, coords.length); } else if (geometry.getClass().equals(Point.class)) { - graphics.drawLine(coordGridX[0], coordGridY[0], coordGridX[0], coordGridY[0]); + double cellWest = extent.snapX(coords[0].x, Direction.LEFT); + double centerX = cellWest + xRes/2.0; + double cellSouth = extent.snapY(coords[0].y, Direction.BOTTOM); + double centerY = cellSouth + yRes/2.0; + long[] gc = extent.getGridCoordinatesAt(centerX, centerY); + graphics.drawLine((int)gc[0], (int)gc[1], (int)gc[0], (int)gc[1]); } } From 64fe664f2544bb5c762c118a67dfd4c8710027df Mon Sep 17 00:00:00 2001 From: Andrea Antonello Date: Thu, 26 Mar 2026 12:23:55 +0100 Subject: [PATCH 18/74] fix for Unsolicited bottom left pixel --- .../org/integratedmodelling/klab/scale/Scale.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java b/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java index 9cfc519de..543ed36bc 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java @@ -455,19 +455,27 @@ public List getKimSpecification() { private class ScaleIterator implements Iterator { long offset = 0; + + private void moveToCovered() { + while (this.offset < size() && !isCovered(offset)) { + this.offset++; + } + } @Override public boolean hasNext() { + moveToCovered(); return offset < size(); } @Override public IScale next() { + moveToCovered(); IScale ret = new Scale(Scale.this, offset); this.offset++; - while (this.offset < size() && !isCovered(offset)) { - this.offset++; - } +// while (this.offset < size() && !isCovered(offset)) { +// this.offset++; +// } return ret; } } From 2b771c8b9427c00801bd70cc37aa1335cab7d0c4 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 21 Apr 2026 14:58:08 +0200 Subject: [PATCH 19/74] Add core concept to owl --- .../src/main/resources/knowledge/observation.owl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/klab.engine/src/main/resources/knowledge/observation.owl b/klab.engine/src/main/resources/knowledge/observation.owl index 2b177c567..2779e27bf 100644 --- a/klab.engine/src/main/resources/knowledge/observation.owl +++ b/klab.engine/src/main/resources/knowledge/observation.owl @@ -1549,6 +1549,14 @@ Importantly, a deliberative agent can access history, although the full meaning + + + + + + + + @@ -1813,12 +1821,6 @@ Importantly, a deliberative agent can access history, although the full meaning - - - - - - + From 5df11a65b5b013ec2ffb7df055d7b1ca013ad913 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Wed, 29 Apr 2026 15:22:34 +0200 Subject: [PATCH 20/74] Pinned Jackson Dependency, Updates STACEncoder to work with HM 0.11.1 and Geotools 34 --- adapters/klab.ogc/pom.xml | 17 +++ .../klab/stac/STACEncoder.java | 125 ++++++++++++++++-- .../remotesensing/SentinelResolver.java | 2 +- klab.commons/pom.xml | 4 +- klab.engine/pom.xml | 4 - klab.hub/pom.xml | 5 +- klab.node/pom.xml | 1 - pom.xml | 13 +- 8 files changed, 143 insertions(+), 28 deletions(-) diff --git a/adapters/klab.ogc/pom.xml b/adapters/klab.ogc/pom.xml index 2ffa58486..bf4eef970 100644 --- a/adapters/klab.ogc/pom.xml +++ b/adapters/klab.ogc/pom.xml @@ -47,6 +47,22 @@ 1.68 + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + + + + com.fasterxml.jackson.core + jackson-annotations + + org.hortonmachine hm-gears @@ -56,6 +72,7 @@ javax.measure jsr-275 + org.mongodb mongo-java-driver diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 8378e6d0a..80f99829f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Date; +import java.util.function.Predicate; import java.util.List; import java.util.Map; import java.util.Set; @@ -12,13 +13,14 @@ import org.geotools.api.data.FeatureSource; import org.geotools.geometry.jts.ReferencedEnvelope; import org.hortonmachine.gears.io.stac.HMStacCollection; +import org.hortonmachine.gears.io.stac.HMStacAsset; import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; import org.hortonmachine.gears.libs.modules.HMRaster.HMRasterWritableBuilder; import org.hortonmachine.gears.libs.modules.HMRaster.MergeMode; import org.hortonmachine.gears.libs.monitor.LogProgressMonitor; -import org.hortonmachine.gears.utils.CrsUtilities; +import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.RegionMap; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; import org.integratedmodelling.klab.Authentication; @@ -58,6 +60,7 @@ import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; +import org.geotools.coverage.processing.Operations; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; import org.geotools.api.feature.simple.SimpleFeature; @@ -67,8 +70,11 @@ import org.geotools.api.referencing.FactoryException; import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; +import java.time.*; +import java.time.format.DateTimeFormatter; import kong.unirest.json.JSONObject; @@ -239,6 +245,7 @@ public void getEncodedData(IResource resource, Map urnParameters String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collectionData); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); String assetId = resource.getParameters().get("asset", String.class); + Integer bandIndex = resource.getParameters().get("band", Integer.class); boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); // This is part of a WIP that will be removed in the future @@ -316,9 +323,34 @@ public void getEncodedData(IResource resource, Map urnParameters if (collection == null) { scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); } - - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetId, items, true, MergeMode.SUBSTITUTE, new LogProgressMonitor()); + + var p = new Predicate() { + + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there + + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + ArrayNode bandsArray = (ArrayNode) bands; + System.out.println("Bueno!"); + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } + return false; + } + }; + + + System.out.println("Making HM Call..."); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); + if (bandIndex != null) { // Which means theat it's a Multi Band COG + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + } } catch (Exception e) { throw new KlabResourceAccessException("Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); } @@ -327,6 +359,7 @@ public void getEncodedData(IResource resource, Map urnParameters return; } + System.out.println("Found Search Option!"); LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -349,23 +382,25 @@ public void getEncodedData(IResource resource, Map urnParameters Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); collection.setGeometryFilter(poly); + if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { time = validateTemporalDimension(time, resourceTime); } ITimeInstant start = time.getStart(); ITimeInstant end = time.getEnd(); - collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); + //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); + + System.out.println(start.getMilliseconds()); + System.out.println(end.getMilliseconds()); GridCoverage2D coverage = null; try { List items = collection.searchItems(); - if (items.isEmpty()) { manager.close(); throw new KlabIllegalStateException("No STAC items found for this context."); } - scope.getMonitor().debug("Found " + items.size() + " STAC items."); if (mergeMode == HMRaster.MergeMode.SUBSTITUTE) { sortByDate(items, scope.getMonitor()); @@ -381,10 +416,6 @@ public void getEncodedData(IResource resource, Map urnParameters space.getProjection().getCoordinateReferenceSystem()); RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); - Set EPSGsAtItems = items.stream().map(i -> i.getEpsg()).collect(Collectors.toUnmodifiableSet()); - if (EPSGsAtItems.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the items " + EPSGsAtItems.toString() + ". The transformation process could affect the data."); - } if (resource.getParameters().contains("awsRegion")) { String bucketRegion = resource.getParameters().get("awsRegion", String.class); @@ -395,10 +426,78 @@ public void getEncodedData(IResource resource, Map urnParameters // Allow transform ensures the process to finish, but I would not bet on the resulting // data. final boolean allowTransform = true; - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetId, items, allowTransform, mergeMode, lpm); - coverage = outRaster.buildCoverage(); - manager.close(); + var p = new Predicate() { + + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + var bandsArray = (ArrayNode) bands; + System.out.println("Bueno!"); + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } + return false; + } + }; + + // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC + items = items.stream().filter(new Predicate() { + + @Override + public boolean test(HMStacItem item) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } + return false; + } + }).collect(Collectors.toList()); + + if (items.size() == 0) { + throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); + } else { + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + } + + + Set EPSGAtAssets = + items.stream() + .flatMap(item -> item.getAssets().stream()) + .filter(p) + .map(HMStacAsset::getEpsg) + .collect(Collectors.toUnmodifiableSet()); + + + if (EPSGAtAssets.size() > 1) { + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); + } + + + System.out.println("Making HM Call!..."); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + if (bandIndex != null) { // Which means theat it's a Multi Band COG + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + } + + manager.close(); } catch (Exception e) { + e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } encoder = new RasterEncoder(); diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java index 818646a37..1335ac718 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java @@ -12,7 +12,7 @@ import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; -import org.hortonmachine.gears.utils.CrsUtilities; +import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.RegionMap; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; import org.integratedmodelling.geoprocessing.TaskMonitor; diff --git a/klab.commons/pom.xml b/klab.commons/pom.xml index 38e8bb8d1..092ea4309 100644 --- a/klab.commons/pom.xml +++ b/klab.commons/pom.xml @@ -1,6 +1,6 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -33,13 +33,11 @@ com.fasterxml.jackson.core jackson-annotations - 2.15.2 com.fasterxml.jackson.core jackson-databind - 2.15.2 diff --git a/klab.engine/pom.xml b/klab.engine/pom.xml index 06339a4f6..24b36110d 100644 --- a/klab.engine/pom.xml +++ b/klab.engine/pom.xml @@ -135,12 +135,10 @@ com.fasterxml.jackson.core jackson-databind - 2.14.3 com.fasterxml.jackson.core jackson-core - 2.14.3 @@ -529,7 +527,6 @@ com.fasterxml.jackson.module jackson-module-jsonSchema - ${jackson-version} @@ -574,7 +571,6 @@ com.fasterxml.jackson.datatype jackson-datatype-joda - ${jackson-version} org.integratedmodelling diff --git a/klab.hub/pom.xml b/klab.hub/pom.xml index ea463c346..a7e249bca 100644 --- a/klab.hub/pom.xml +++ b/klab.hub/pom.xml @@ -235,22 +235,19 @@ com.fasterxml.jackson.core jackson-databind - 2.12.4 com.fasterxml.jackson.core jackson-core - 2.12.4 + com.fasterxml.jackson.core jackson-annotations - 2.12.4 com.fasterxml.jackson.datatype jackson-datatype-jsr310 - 2.12.4 com.kjetland diff --git a/klab.node/pom.xml b/klab.node/pom.xml index 36321591a..2d967d4b8 100644 --- a/klab.node/pom.xml +++ b/klab.node/pom.xml @@ -85,7 +85,6 @@ com.fasterxml.jackson.datatype jackson-datatype-joda - ${jackson-version} 34.0 - 0.11.0-SNAPSHOT + 0.11.1-SNAPSHOT 1.20.0 1.5.0 17 @@ -131,7 +131,7 @@ 5.5.1 2020.0.3 5.6.3 - 2.12.4 + 2.17.2 0.13.4 @@ -312,6 +312,15 @@ commons-io ${commons-io.version} + + + com.fasterxml.jackson + jackson-bom + ${jackson-version} + pom + import + + From 2ec3cfa2a8dd8d1f268793e6805019f8d79d0673 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Wed, 29 Apr 2026 15:37:21 +0200 Subject: [PATCH 21/74] removes prints --- .../integratedmodelling/klab/stac/STACEncoder.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 80f99829f..3fc44480a 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -343,9 +343,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou return false; } }; - - - System.out.println("Making HM Call..."); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG @@ -359,7 +357,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou return; } - System.out.println("Found Search Option!"); LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -389,10 +386,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } ITimeInstant start = time.getStart(); ITimeInstant end = time.getEnd(); - //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); - - System.out.println(start.getMilliseconds()); - System.out.println(end.getMilliseconds()); + //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) GridCoverage2D coverage = null; try { @@ -433,7 +427,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou var bands = asset.getAssetNode().get("eo:bands"); if (bands != null && bands.isArray()) { var bandsArray = (ArrayNode) bands; - System.out.println("Bueno!"); for (var bandNode : bandsArray) { String bandName = bandNode.get("name").asText(); if (bandName.equals(assetId)) { // under eo:band it's one of the band From c548a97865ccdd225fc24416f64a7ddf870da0ea Mon Sep 17 00:00:00 2001 From: AM1729 Date: Thu, 30 Apr 2026 10:46:15 +0200 Subject: [PATCH 22/74] fixes --- adapters/klab.adapter.datacube/pom.xml | 2 +- .../java/org/integratedmodelling/klab/stac/STACEncoder.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/adapters/klab.adapter.datacube/pom.xml b/adapters/klab.adapter.datacube/pom.xml index e02c10f85..1f460d015 100644 --- a/adapters/klab.adapter.datacube/pom.xml +++ b/adapters/klab.adapter.datacube/pom.xml @@ -1,6 +1,6 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 org.integratedmodelling diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 3fc44480a..9c331a850 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -332,7 +332,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou var bands = asset.getAssetNode().get("eo:bands"); if (bands != null && bands.isArray()) { ArrayNode bandsArray = (ArrayNode) bands; - System.out.println("Bueno!"); for (var bandNode : bandsArray) { String bandName = bandNode.get("name").asText(); if (bandName.equals(assetId)) { // under eo:band it's one of the band @@ -480,8 +479,6 @@ public boolean test(HMStacItem item) { scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); } - - System.out.println("Making HM Call!..."); HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG From 1c7ec8f60c7f22175b83e17cf27bb52fb54ce9ac Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 4 May 2026 14:29:25 +0200 Subject: [PATCH 23/74] Ensure close manager --- .../klab/stac/STACEncoder.java | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9c331a850..d2bb60d42 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -311,40 +311,41 @@ public void getEncodedData(IResource resource, Map urnParameters try { // TODO see if we can access to the same readRasterBandOnRegion without using a collection LogProgressMonitor lpm = new LogProgressMonitor(); - HMStacManager manager = new HMStacManager(catalogUrl, lpm); - HMStacCollection collection = null; - try { - manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e1) { - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); - } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - } - - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - ArrayNode bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; - - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); + try (HMStacManager manager = new HMStacManager(catalogUrl, lpm)) { + HMStacCollection collection = null; + try { + manager.open(); + collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + } catch (Exception e1) { + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); + } + + if (collection == null) { + scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + } + + var p = new Predicate() { + + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there + + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + ArrayNode bandsArray = (ArrayNode) bands; + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } + return false; + } + }; + + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + } if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } @@ -363,6 +364,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); } catch (Exception e) { + try { + manager.close(); + } catch (Exception e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl + ". Reason :" + e.getMessage()); } @@ -461,6 +468,7 @@ public boolean test(HMStacItem item) { }).collect(Collectors.toList()); if (items.size() == 0) { + manager.close(); throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); From 331e52f7ffddb2c2a9afb31f495879dfedd4a41b Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 4 May 2026 14:35:04 +0200 Subject: [PATCH 24/74] Better generated xtext code --- .../kactors/ui/KactorsUiModule.java | 4 +- .../kdl/ui/KdlUiModule.java | 4 +- .../kim/ui/KimUiModule.java | 4 +- .../kim/formatting2/KimFormatter.java | 2 + .../kim/validation/KimValidator.java | 52 +++++++++---------- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java b/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java index 4a329480b..7537652c7 100644 --- a/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java +++ b/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java @@ -30,7 +30,7 @@ public Class bindIHighlightingConfiguratio return KactorsHighlightingConfiguration.class; } - public KactorsUiModule(final AbstractUIPlugin arg0) { - super(arg0); + public KactorsUiModule(final AbstractUIPlugin plugin) { + super(plugin); } } diff --git a/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java b/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java index 9d12f273a..26b2c6bf3 100644 --- a/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java +++ b/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java @@ -30,7 +30,7 @@ public Class bindAbstractAntlrTo return KdlSyntaxHighlighter.class; } - public KdlUiModule(final AbstractUIPlugin arg0) { - super(arg0); + public KdlUiModule(final AbstractUIPlugin plugin) { + super(plugin); } } diff --git a/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java b/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java index 059a6f44a..0ede2c465 100644 --- a/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java +++ b/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java @@ -100,7 +100,7 @@ public Class bindIEObjectDocumentationP return KimDocumentationProvider.class; } - public KimUiModule(final AbstractUIPlugin arg0) { - super(arg0); + public KimUiModule(final AbstractUIPlugin plugin) { + super(plugin); } } diff --git a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java index 88dea3ee1..2958fb5f5 100644 --- a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java +++ b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java @@ -12,6 +12,7 @@ import org.eclipse.xtext.formatting2.IFormattableDocument; import org.eclipse.xtext.resource.XtextResource; import org.eclipse.xtext.xbase.lib.Extension; +import org.eclipse.xtext.xbase.lib.XbaseGenerated; import org.integratedmodelling.kim.kim.Concept; import org.integratedmodelling.kim.kim.ConceptDeclaration; import org.integratedmodelling.kim.kim.ConceptStatement; @@ -60,6 +61,7 @@ protected void _format(final ConceptDeclaration conceptDeclaration, @Extension f document.format(conceptDeclaration.getContext()); } + @XbaseGenerated public void format(final Object conceptDeclaration, final IFormattableDocument document) { if (conceptDeclaration instanceof XtextResource) { _format((XtextResource)conceptDeclaration, document); diff --git a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java index 5e927d1d4..8ad16aee2 100644 --- a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java +++ b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java @@ -4,12 +4,12 @@ */ package org.integratedmodelling.kim.validation; -import com.google.common.base.Objects; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.Objects; import java.util.Set; import java.util.logging.Level; import org.eclipse.emf.common.util.EList; @@ -443,20 +443,20 @@ public void checkModelStatement(final ModelStatement model) { int _level = ref.getLevel(); boolean _matched = false; int _intValue = Level.SEVERE.intValue(); - if (Objects.equal(_level, _intValue)) { + if (Objects.equals(_level, _intValue)) { _matched=true; this.error(ref.getMessage(), KimPackage.Literals.MODEL_STATEMENT__MODEL, KimValidator.REASONING_PROBLEM); } if (!_matched) { int _intValue_1 = Level.WARNING.intValue(); - if (Objects.equal(_level, _intValue_1)) { + if (Objects.equals(_level, _intValue_1)) { _matched=true; this.warning(ref.getMessage(), KimPackage.Literals.MODEL_STATEMENT__MODEL, KimValidator.REASONING_PROBLEM); } } if (!_matched) { int _intValue_2 = Level.INFO.intValue(); - if (Objects.equal(_level, _intValue_2)) { + if (Objects.equals(_level, _intValue_2)) { _matched=true; this.info(ref.getMessage(), KimPackage.Literals.MODEL_STATEMENT__MODEL, KimValidator.REASONING_PROBLEM); } @@ -533,14 +533,14 @@ public void checkModelDefinition(final ModelBodyStatement model) { int _level = ref.getLevel(); boolean _matched = false; int _intValue = Level.SEVERE.intValue(); - if (Objects.equal(_level, _intValue)) { + if (Objects.equals(_level, _intValue)) { _matched=true; this.error(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__OBSERVABLES, obsIdx, KimValidator.REASONING_PROBLEM); } if (!_matched) { int _intValue_1 = Level.WARNING.intValue(); - if (Objects.equal(_level, _intValue_1)) { + if (Objects.equals(_level, _intValue_1)) { _matched=true; this.warning(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__OBSERVABLES, obsIdx, KimValidator.REASONING_PROBLEM); @@ -548,7 +548,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { } if (!_matched) { int _intValue_2 = Level.INFO.intValue(); - if (Objects.equal(_level, _intValue_2)) { + if (Objects.equals(_level, _intValue_2)) { _matched=true; this.info(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__OBSERVABLES, obsIdx, KimValidator.REASONING_PROBLEM); @@ -668,14 +668,14 @@ public void checkModelDefinition(final ModelBodyStatement model) { int _level = ref.getLevel(); boolean _matched = false; int _intValue = Level.SEVERE.intValue(); - if (Objects.equal(_level, _intValue)) { + if (Objects.equals(_level, _intValue)) { _matched=true; this.error(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__DEPENDENCIES, i, KimValidator.REASONING_PROBLEM); } if (!_matched) { int _intValue_1 = Level.WARNING.intValue(); - if (Objects.equal(_level, _intValue_1)) { + if (Objects.equals(_level, _intValue_1)) { _matched=true; this.warning(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__DEPENDENCIES, i, KimValidator.REASONING_PROBLEM); @@ -683,7 +683,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { } if (!_matched) { int _intValue_2 = Level.INFO.intValue(); - if (Objects.equal(_level, _intValue_2)) { + if (Objects.equals(_level, _intValue_2)) { _matched=true; this.info(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__DEPENDENCIES, i, KimValidator.REASONING_PROBLEM); @@ -775,7 +775,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { for (final Classifier classifier : _classifiers) { { KimConcept decl = Kim.INSTANCE.declareConcept(classifier.getDeclaration()); - boolean _equals = Objects.equal(type, Integer.valueOf(0)); + boolean _equals = Objects.equals(type, Integer.valueOf(0)); if (_equals) { type = decl.getType(); } else { @@ -853,10 +853,10 @@ public void checkModelDefinition(final ModelBodyStatement model) { String _id = arg.getId(); boolean _tripleNotEquals_4 = (_id != null); if (_tripleNotEquals_4) { - if (((!Objects.equal(arg.getId(), "?")) && (!Objects.equal(arg.getId(), "*")))) { + if (((!Objects.equals(arg.getId(), "?")) && (!Objects.equals(arg.getId(), "*")))) { boolean found = false; for (final KimObservable dependency : dependencies) { - if (((dependency.getName() != null) && Objects.equal(dependency.getName(), arg))) { + if (((dependency.getName() != null) && Objects.equals(dependency.getName(), arg))) { found = true; } } @@ -864,7 +864,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { } } else { String _id_1 = arg.getId(); - boolean _equals = Objects.equal(_id_1, "?"); + boolean _equals = Objects.equals(_id_1, "?"); if (_equals) { if (checkFound) { this.error("Only one \'?\' is allowed in the argument list, to mark the result column", @@ -1118,18 +1118,18 @@ public void checkModelDefinition(final ModelBodyStatement model) { public void notify(final KimNotification notification, final EObject object, final EStructuralFeature cls) { final Level _switchValue = notification.level; boolean _matched = false; - if (Objects.equal(_switchValue, Level.SEVERE)) { + if (Objects.equals(_switchValue, Level.SEVERE)) { _matched=true; this.error(notification.message, object, cls); } if (!_matched) { - if (Objects.equal(_switchValue, Level.WARNING)) { + if (Objects.equals(_switchValue, Level.WARNING)) { _matched=true; this.warning(notification.message, object, cls); } } if (!_matched) { - if (Objects.equal(_switchValue, Level.INFO)) { + if (Objects.equals(_switchValue, Level.INFO)) { _matched=true; this.info(notification.message, object, cls); } @@ -1208,7 +1208,7 @@ public KimAcknowledgement checkObservation(final ObserveStatementBody observatio for (final KimNotification notification : _addAction) { { this.notify(notification, observation, KimPackage.Literals.OBSERVE_STATEMENT_BODY__ACTIONS, i); - boolean _equals = Objects.equal(notification.level, Level.SEVERE); + boolean _equals = Objects.equals(notification.level, Level.SEVERE); if (_equals) { ok = false; } @@ -2366,7 +2366,7 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep EnumSet corectype = Kim.INSTANCE.getType(((ConceptStatement) _eContainer).getConcept(), null); EnumSet a = Kim.intersection(type, IKimConcept.DECLARABLE_TYPES); EnumSet b = Kim.intersection(corectype, IKimConcept.DECLARABLE_TYPES); - if ((((a.size() != 1) || (b.size() != 1)) || (!Objects.equal(((IKimConcept.Type[])Conversions.unwrapArray(a, IKimConcept.Type.class))[0], ((Object[])Conversions.unwrapArray(b, Object.class))[0])))) { + if ((((a.size() != 1) || (b.size() != 1)) || (!Objects.equals(((IKimConcept.Type[])Conversions.unwrapArray(a, IKimConcept.Type.class))[0], ((Object[])Conversions.unwrapArray(b, Object.class))[0])))) { this.error("Core concept is not compatible with the stated type", concept, KimPackage.Literals.CONCEPT_STATEMENT_BODY__PARENTS); error = true; @@ -2428,7 +2428,7 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep if (_not_1) { KimConcept declaration = Kim.INSTANCE.declareConcept(p); if (((declaration != null) && (!declaration.getType().isEmpty()))) { - if (((i == 0) || Objects.equal(concept.getConnectors().get((i - 1)), ","))) { + if (((i == 0) || Objects.equals(concept.getConnectors().get((i - 1)), ","))) { KimConceptStatement.ParentConceptImpl group = new KimConceptStatement.ParentConceptImpl(); group.getConcepts().add(declaration); declaredParents.add(group); @@ -2458,12 +2458,12 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep _xifexpression_1 = _xifexpression_2; } BinarySemanticOperator connector = _xifexpression_1; - if ((Objects.equal(connector, BinarySemanticOperator.FOLLOWS) && + if ((Objects.equals(connector, BinarySemanticOperator.FOLLOWS) && (!declaration.getType().contains(IKimConcept.Type.EVENT)))) { this.error("The consequentiality (\'follows\') operator is only allowed between events", concept, KimPackage.Literals.CONCEPT_STATEMENT_BODY__PARENTS, i); error = true; } - if (((group_1.getConnector() != BinarySemanticOperator.NONE) && (!Objects.equal(group_1.getConnector(), connector)))) { + if (((group_1.getConnector() != BinarySemanticOperator.NONE) && (!Objects.equals(group_1.getConnector(), connector)))) { this.error( "Cannot mix union (\'or\'), intersection (\'and\') and consequentiality (\'follows\') operators", concept, KimPackage.Literals.CONCEPT_STATEMENT_BODY__PARENTS, i); error = true; @@ -2592,7 +2592,7 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep { KimConcept iden = Kim.INSTANCE.declareConcept(identity); String _type = requirement.getType(); - boolean _equals = Objects.equal(_type, "identity"); + boolean _equals = Objects.equals(_type, "identity"); if (_equals) { boolean _contains = iden.getType().contains(IKimConcept.Type.IDENTITY); boolean _not_1 = (!_contains); @@ -3026,18 +3026,18 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep public void notify(final KimNotification notification, final EObject object, final EStructuralFeature cls, final int index) { final Level _switchValue = notification.level; boolean _matched = false; - if (Objects.equal(_switchValue, Level.SEVERE)) { + if (Objects.equals(_switchValue, Level.SEVERE)) { _matched=true; this.error(notification.message, object, cls, index); } if (!_matched) { - if (Objects.equal(_switchValue, Level.WARNING)) { + if (Objects.equals(_switchValue, Level.WARNING)) { _matched=true; this.warning(notification.message, object, cls, index); } } if (!_matched) { - if (Objects.equal(_switchValue, Level.INFO)) { + if (Objects.equals(_switchValue, Level.INFO)) { _matched=true; this.info(notification.message, object, cls, index); } From 094383269021e8dda9b56c07cf0ade1d4731b51d Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 5 May 2026 12:50:51 +0200 Subject: [PATCH 25/74] Add first version of predicate for assets using path expressions --- .../klab/stac/STACEncoder.java | 140 ++++------ .../klab/stac/STACPathExpression.java | 227 ++++++++++++++++ .../ogc/stac/test/STACPathExpressionTest.java | 254 ++++++++++++++++++ 3 files changed, 535 insertions(+), 86 deletions(-) create mode 100644 adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java create mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index d2bb60d42..7eece7172 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -56,6 +56,7 @@ import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; +import org.integratedmodelling.klab.stac.STACPathExpression.STACAssetPredicate; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; @@ -70,6 +71,7 @@ import org.geotools.api.referencing.FactoryException; import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; @@ -79,8 +81,8 @@ import kong.unirest.json.JSONObject; public class STACEncoder implements IResourceEncoder { - - /** + + /** * The raster or vector encoder that does the actual work after we get our coverage from the service. */ IResourceEncoder encoder; @@ -324,25 +326,7 @@ public void getEncodedData(IResource resource, Map urnParameters scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); } - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - ArrayNode bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; - + Predicate p = STACAssetPredicate.from("eo:bands[*]>name", assetId); HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); } @@ -426,74 +410,58 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // Allow transform ensures the process to finish, but I would not bet on the resulting // data. final boolean allowTransform = true; - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; + Predicate p = STACAssetPredicate.from(STACPathExpression.PREDICATE_EO_BANDS_NAME, assetId); - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } - return false; - } - }).collect(Collectors.toList()); + // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC + items = items.stream().filter(new Predicate() { + + @Override + public boolean test(HMStacItem item) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } + return false; + } + }).collect(Collectors.toList()); - if (items.size() == 0) { - manager.close(); - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); - } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); - } - - - Set EPSGAtAssets = - items.stream() - .flatMap(item -> item.getAssets().stream()) - .filter(p) - .map(HMStacAsset::getEpsg) - .collect(Collectors.toUnmodifiableSet()); - - - if (EPSGAtAssets.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); - } + if (items.size() == 0) { + manager.close(); + throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); + } else { + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + } + + + Set EPSGAtAssets = + items.stream() + .flatMap(item -> item.getAssets().stream()) + .filter(p) + .map(HMStacAsset::getEpsg) + .collect(Collectors.toUnmodifiableSet()); + + + if (EPSGAtAssets.size() > 1) { + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); + } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); - if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); - } - - manager.close(); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + if (bandIndex != null) { // Which means theat it's a Multi Band COG + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + } + + manager.close(); } catch (Exception e) { e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java new file mode 100644 index 000000000..3daa068ab --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -0,0 +1,227 @@ +package org.integratedmodelling.klab.stac; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; + +public final class STACPathExpression { + + private final List path; + + private STACPathExpression(List path) { + this.path = path; + } + + public static STACPathExpression parse(String jsonPath) { + if (jsonPath == null || jsonPath.isBlank()) { + throw new IllegalArgumentException("JSON path cannot be empty"); + } + + return new STACPathExpression(parsePath(jsonPath)); + } + + public boolean matches(JsonNode root, String expectedValue) { + List resolvedNodes = resolve(root); + + for (JsonNode node : resolvedNodes) { + if (jsonValueEquals(node, expectedValue)) { + return true; + } + } + + return false; + } + + public List resolve(JsonNode root) { + List currentNodes = new ArrayList<>(); + currentNodes.add(root); + + for (PathPart part : path) { + List nextNodes = new ArrayList<>(); + + for (JsonNode current : currentNodes) { + if (current == null || current.isNull() || current.isMissingNode()) { + continue; + } + + JsonNode fieldNode = current.get(part.fieldName()); + + if (fieldNode == null || fieldNode.isNull() || fieldNode.isMissingNode()) { + continue; + } + + if (part.arrayMode() == ArrayMode.NONE) { + nextNodes.add(fieldNode); + } else if (part.arrayMode() == ArrayMode.INDEX) { + if (fieldNode.isArray()) { + JsonNode indexedNode = fieldNode.get(part.arrayIndex()); + + if (indexedNode != null && !indexedNode.isNull() && !indexedNode.isMissingNode()) { + nextNodes.add(indexedNode); + } + } + } else if (part.arrayMode() == ArrayMode.WILDCARD) { + if (fieldNode.isArray()) { + for (JsonNode arrayElement : fieldNode) { + if (arrayElement != null && !arrayElement.isNull() && !arrayElement.isMissingNode()) { + nextNodes.add(arrayElement); + } + } + } + } + } + + currentNodes = nextNodes; + + if (currentNodes.isEmpty()) { + break; + } + } + + return currentNodes; + } + + private static List parsePath(String jsonPath) { + String[] tokens = jsonPath.split(">"); + + List parts = new ArrayList<>(); + + for (String token : tokens) { + String trimmed = token.trim(); + + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("Invalid empty path element"); + } + + parts.add(PathPart.parse(trimmed)); + } + + return parts; + } + + private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { + if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { + return expectedValue == null; + } + + if (expectedValue == null) { + return false; + } + + if (actualValue.isNumber()) { + return numberEquals(actualValue, expectedValue); + } + + if (actualValue.isBoolean()) { + return Boolean.toString(actualValue.booleanValue()) + .equalsIgnoreCase(expectedValue); + } + + if (actualValue.isTextual()) { + return Objects.equals(actualValue.asText(), expectedValue); + } + + return Objects.equals(actualValue.asText(), expectedValue); + } + + private static boolean numberEquals(JsonNode actualValue, String expectedValue) { + try { + BigDecimal actual = actualValue.decimalValue(); + BigDecimal expected = new BigDecimal(expectedValue); + + return actual.compareTo(expected) == 0; + } catch (NumberFormatException e) { + return false; + } + } + + public record PathPart( + String fieldName, + ArrayMode arrayMode, + Integer arrayIndex + ) { + + public static PathPart parse(String token) { + int bracketStart = token.indexOf('['); + + if (bracketStart < 0) { + return new PathPart(token, ArrayMode.NONE, null); + } + + int bracketEnd = token.indexOf(']', bracketStart); + + if (bracketEnd < 0) { + throw new IllegalArgumentException( + "Invalid array syntax in path element: " + token + ); + } + + if (bracketEnd != token.length() - 1) { + throw new IllegalArgumentException( + "Unexpected characters after array syntax in path element: " + token + ); + } + + String fieldName = token.substring(0, bracketStart).trim(); + String indexText = token.substring(bracketStart + 1, bracketEnd).trim(); + + if (fieldName.isEmpty()) { + throw new IllegalArgumentException( + "Field name cannot be empty in path element: " + token + ); + } + + if ("*".equals(indexText)) { + return new PathPart(fieldName, ArrayMode.WILDCARD, null); + } + + int index; + + try { + index = Integer.parseInt(indexText); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid array index in path element: " + token, + e + ); + } + + if (index < 0) { + throw new IllegalArgumentException( + "Array index cannot be negative in path element: " + token + ); + } + + return new PathPart(fieldName, ArrayMode.INDEX, index); + } + } + + public enum ArrayMode { + NONE, + INDEX, + WILDCARD + } + + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; + + public final class STACAssetPredicate { + + private STACAssetPredicate() { + } + + public static Predicate from(String jsonPath, String expectedValue) { + STACPathExpression expression = STACPathExpression.parse(jsonPath); + + return asset -> expression.matches(asset.getAssetNode(), expectedValue); + } + } + + + +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java new file mode 100644 index 000000000..f66774e9a --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java @@ -0,0 +1,254 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class STACPathExpressionTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static JsonNode testJson() throws Exception { + String json = """ + { + "node1": { + "node2": [ + { + "name": "something", + "count": 10, + "active": true + }, + { + "name": "other", + "count": 20, + "active": false + } + ] + }, + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + }, + { + "name": "B08", + "common_name": "nir" + } + ], + "properties": { + "cloud_cover": 12.5, + "enabled": true + } + } + """; + + return MAPPER.readTree(json); + } + + @Test + void shouldResolveIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertTrue(expression.matches(root, "something")); + assertFalse(expression.matches(root, "other")); + } + + @Test + void shouldResolveSecondIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[1]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("other", resolved.get(0).asText()); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "something")); + } + + @Test + void shouldResolveWildcardArrayElements() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(2, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertEquals("other", resolved.get(1).asText()); + + assertTrue(expression.matches(root, "something")); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "missing")); + } + + @Test + void shouldMatchEoBandsUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(3, resolved.size()); + assertEquals("B01", resolved.get(0).asText()); + assertEquals("B04", resolved.get(1).asText()); + assertEquals("B08", resolved.get(2).asText()); + + assertTrue(expression.matches(root, "B01")); + assertTrue(expression.matches(root, "B04")); + assertTrue(expression.matches(root, "B08")); + assertFalse(expression.matches(root, "B99")); + } + + @Test + void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>common_name"); + + assertTrue(expression.matches(root, "coastal")); + assertTrue(expression.matches(root, "red")); + assertTrue(expression.matches(root, "nir")); + assertFalse(expression.matches(root, "green")); + } + + @Test + void shouldMatchNumbers() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>count"); + + assertTrue(expression.matches(root, "10")); + assertTrue(expression.matches(root, "10.0")); + assertFalse(expression.matches(root, "20")); + assertFalse(expression.matches(root, "not-a-number")); + } + + @Test + void shouldMatchBooleans() throws Exception { + JsonNode root = testJson(); + + STACPathExpression firstExpression = + STACPathExpression.parse("node1>node2[0]>active"); + + STACPathExpression secondExpression = + STACPathExpression.parse("node1>node2[1]>active"); + + assertTrue(firstExpression.matches(root, "true")); + assertTrue(firstExpression.matches(root, "TRUE")); + assertFalse(firstExpression.matches(root, "false")); + + assertTrue(secondExpression.matches(root, "false")); + assertTrue(secondExpression.matches(root, "FALSE")); + assertFalse(secondExpression.matches(root, "true")); + } + + @Test + void shouldMatchTopLevelProperties() throws Exception { + JsonNode root = testJson(); + + STACPathExpression cloudCoverExpression = + STACPathExpression.parse("properties>cloud_cover"); + + STACPathExpression enabledExpression = + STACPathExpression.parse("properties>enabled"); + + assertTrue(cloudCoverExpression.matches(root, "12.5")); + assertTrue(cloudCoverExpression.matches(root, "12.50")); + assertFalse(cloudCoverExpression.matches(root, "13")); + + assertTrue(enabledExpression.matches(root, "true")); + assertFalse(enabledExpression.matches(root, "false")); + } + + @Test + void shouldReturnEmptyListForMissingPath() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>missing"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "anything")); + } + + @Test + void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("properties[*]>cloud_cover"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "12.5")); + } + + @Test + void shouldRejectInvalidWildcardSyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[abc]>name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void shouldRejectNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[-1]>name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void shouldRejectUnexpectedCharactersAfterArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[0]extra>name") + ); + + assertTrue(exception.getMessage().contains("Unexpected characters")); + } + + @Test + void shouldRejectEmptyPathElement() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>>name") + ); + + assertTrue(exception.getMessage().contains("Invalid empty path element")); + } +} \ No newline at end of file From f48b09d0d269aa78a895a2fc124b55dcc1103e29 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 5 May 2026 14:23:36 +0200 Subject: [PATCH 26/74] Add predicate using HMAsset attributes --- .../klab/stac/STACPathExpression.java | 137 ++++- .../STACPathExpressionAndAttributeTest.java | 545 ++++++++++++++++++ .../ogc/stac/test/STACPathExpressionTest.java | 254 -------- 3 files changed, 679 insertions(+), 257 deletions(-) create mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java delete mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 3daa068ab..6bdd725ef 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -13,6 +13,8 @@ public final class STACPathExpression { private final List path; + + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; private STACPathExpression(List path) { this.path = path; @@ -104,6 +106,37 @@ private static List parsePath(String jsonPath) { return parts; } + + private static boolean valueEquals(Object actualValue, String expectedValue) { + if (actualValue == null) { + return expectedValue == null; + } + + if (expectedValue == null) { + return false; + } + + if (actualValue instanceof Number number) { + return numberEquals(number, expectedValue); + } + + if (actualValue instanceof Boolean bool) { + return Boolean.toString(bool).equalsIgnoreCase(expectedValue); + } + + return Objects.equals(String.valueOf(actualValue), expectedValue); + } + + private static boolean numberEquals(Number actualValue, String expectedValue) { + try { + BigDecimal actual = new BigDecimal(actualValue.toString()); + BigDecimal expected = new BigDecimal(expectedValue); + + return actual.compareTo(expected) == 0; + } catch (NumberFormatException e) { + return false; + } + } private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { @@ -208,12 +241,42 @@ public enum ArrayMode { WILDCARD } - public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; - public final class STACAssetPredicate { private STACAssetPredicate() { } + + public static Predicate fromJsonPath( + String jsonPath, + String expectedValue + ) { + STACPathExpression expression = STACPathExpression.parse(jsonPath); + + return asset -> { + if (asset == null || asset.getAssetNode() == null) { + return false; + } + + return expression.matches(asset.getAssetNode(), expectedValue); + }; + } + + public static Predicate fromAssetAttribute( + String attributeName, + String expectedValue + ) { + AssetAttribute attribute = AssetAttribute.fromName(attributeName); + + return asset -> { + if (asset == null) { + return false; + } + + Object actualValue = attribute.read(asset); + + return valueEquals(actualValue, expectedValue); + }; + } public static Predicate from(String jsonPath, String expectedValue) { STACPathExpression expression = STACPathExpression.parse(jsonPath); @@ -222,6 +285,74 @@ public static Predicate from(String jsonPath, String expectedValue) } } - + public enum AssetAttribute { + + ID("id") { + @Override + Object read(HMStacAsset asset) { + return asset.getId(); + } + }, + + TITLE("title") { + @Override + Object read(HMStacAsset asset) { + return asset.getTitle(); + } + }, + + TYPE("type") { + @Override + Object read(HMStacAsset asset) { + return asset.getType(); + } + }, + + VALID("valid") { + @Override + Object read(HMStacAsset asset) { + return asset.isValid(); + } + }, + + EPSG("epsg") { + @Override + Object read(HMStacAsset asset) { + return asset.getEpsg(); + } + }, + + NON_VALID_REASON("nonValidReason") { + @Override + Object read(HMStacAsset asset) { + return asset.getNonValidReason(); + } + }; + + private final String name; + + AssetAttribute(String name) { + this.name = name; + } + + abstract Object read(HMStacAsset asset); + + public static AssetAttribute fromName(String name) { + if (name == null || name.isBlank()) { + throw new IllegalArgumentException("Asset attribute name cannot be empty"); + } + + for (AssetAttribute attribute : values()) { + if (attribute.name.equalsIgnoreCase(name.trim())) { + return attribute; + } + } + + throw new IllegalArgumentException( + "Unsupported HMStacAsset attribute: " + name + + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason" + ); + } + } } \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java new file mode 100644 index 000000000..d24b147f3 --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java @@ -0,0 +1,545 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.*; + +class STACPathExpressionAndAttributeTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static JsonNode testJson() throws Exception { + String json = """ + { + "node1": { + "node2": [ + { + "name": "something", + "count": 10, + "active": true + }, + { + "name": "other", + "count": 20, + "active": false + } + ] + }, + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + }, + { + "name": "B08", + "common_name": "nir" + } + ], + "properties": { + "cloud_cover": 12.5, + "enabled": true + } + } + """; + + return MAPPER.readTree(json); + } + + @Test + void shouldResolveIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertTrue(expression.matches(root, "something")); + assertFalse(expression.matches(root, "other")); + } + + @Test + void shouldResolveSecondIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[1]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("other", resolved.get(0).asText()); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "something")); + } + + @Test + void shouldResolveWildcardArrayElements() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(2, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertEquals("other", resolved.get(1).asText()); + + assertTrue(expression.matches(root, "something")); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "missing")); + } + + @Test + void shouldMatchEoBandsUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(3, resolved.size()); + assertEquals("B01", resolved.get(0).asText()); + assertEquals("B04", resolved.get(1).asText()); + assertEquals("B08", resolved.get(2).asText()); + + assertTrue(expression.matches(root, "B01")); + assertTrue(expression.matches(root, "B04")); + assertTrue(expression.matches(root, "B08")); + assertFalse(expression.matches(root, "B99")); + } + + @Test + void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>common_name"); + + assertTrue(expression.matches(root, "coastal")); + assertTrue(expression.matches(root, "red")); + assertTrue(expression.matches(root, "nir")); + assertFalse(expression.matches(root, "green")); + } + + @Test + void shouldMatchNumbers() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>count"); + + assertTrue(expression.matches(root, "10")); + assertTrue(expression.matches(root, "10.0")); + assertFalse(expression.matches(root, "20")); + assertFalse(expression.matches(root, "not-a-number")); + } + + @Test + void shouldMatchBooleans() throws Exception { + JsonNode root = testJson(); + + STACPathExpression firstExpression = + STACPathExpression.parse("node1>node2[0]>active"); + + STACPathExpression secondExpression = + STACPathExpression.parse("node1>node2[1]>active"); + + assertTrue(firstExpression.matches(root, "true")); + assertTrue(firstExpression.matches(root, "TRUE")); + assertFalse(firstExpression.matches(root, "false")); + + assertTrue(secondExpression.matches(root, "false")); + assertTrue(secondExpression.matches(root, "FALSE")); + assertFalse(secondExpression.matches(root, "true")); + } + + @Test + void shouldMatchTopLevelProperties() throws Exception { + JsonNode root = testJson(); + + STACPathExpression cloudCoverExpression = + STACPathExpression.parse("properties>cloud_cover"); + + STACPathExpression enabledExpression = + STACPathExpression.parse("properties>enabled"); + + assertTrue(cloudCoverExpression.matches(root, "12.5")); + assertTrue(cloudCoverExpression.matches(root, "12.50")); + assertFalse(cloudCoverExpression.matches(root, "13")); + + assertTrue(enabledExpression.matches(root, "true")); + assertFalse(enabledExpression.matches(root, "false")); + } + + @Test + void shouldReturnEmptyListForMissingPath() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>missing"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "anything")); + } + + @Test + void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("properties[*]>cloud_cover"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "12.5")); + } + + @Test + void shouldRejectInvalidWildcardSyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[abc]>name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void shouldRejectNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[-1]>name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void shouldRejectUnexpectedCharactersAfterArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[0]extra>name") + ); + + assertTrue(exception.getMessage().contains("Unexpected characters")); + } + + @Test + void shouldRejectEmptyPathElement() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>>name") + ); + + assertTrue(exception.getMessage().contains("Invalid empty path element")); + } + + // Test by attributes + + @Test + void shouldMatchAssetByIdAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldNotMatchAssetByDifferentIdAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B08"); + + assertFalse(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByTitleAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Surface reflectance band 4", + "type": "image/tiff; application=geotiff" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute( + "title", + "Surface reflectance band 4" + ); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByTypeAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute( + "type", + "image/tiff; application=geotiff" + ); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByEpsgAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632"); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByEpsgAttributeUsingDecimalEquivalent() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632.0"); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldNotMatchAssetByDifferentEpsgAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "4326"); + + assertFalse(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByValidAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("valid", "true"); + + /* + * One caveat: HMStacAsset computes isValid through HMStacAssetHandlers.getHandler(this). + * Depending on your test classpath and supported MIME types, asset.isValid() + * may be either true or false. That is why the validity test uses: + * assertEquals(asset.isValid(), predicate.test(asset)); + */ + assertEquals(asset.isValid(), predicate.test(asset)); + } + + // JSON-path-through-asset tests + @Test + void shouldMatchAssetUsingJsonPathPredicate() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + } + ] + } + """); + + HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromJsonPath( + "eo:bands[*]>name", + "B04" + ); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldNotMatchAssetUsingJsonPathPredicateWhenValueIsMissing() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + } + ] + } + """); + + HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromJsonPath( + "eo:bands[*]>name", + "B08" + ); + + assertFalse(predicate.test(asset)); + } + + @Test + void shouldMatchAssetUsingDefaultJsonPathPredicateFactory() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B04" + } + ] + } + """); + + HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.from( + "eo:bands[*]>name", + "B04" + ); + + assertTrue(predicate.test(asset)); + } + + // negative/null-safety tests: + @Test + void shouldReturnFalseWhenAssetIsNullForAttributePredicate() { + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); + + assertFalse(predicate.test(null)); + } + + @Test + void shouldReturnFalseWhenAssetIsNullForJsonPathPredicate() { + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromJsonPath( + "eo:bands[*]>name", + "B04" + ); + + assertFalse(predicate.test(null)); + } + + @Test + void shouldRejectUnsupportedAssetAttribute() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( + "unsupportedAttribute", + "value" + ) + ); + + assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); + } + + @Test + void shouldRejectEmptyAssetAttributeName() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( + " ", + "value" + ) + ); + + assertTrue(exception.getMessage().contains("Asset attribute name cannot be empty")); + } + + +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java deleted file mode 100644 index f66774e9a..000000000 --- a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java +++ /dev/null @@ -1,254 +0,0 @@ -package org.integratedmodelling.klab.ogc.stac.test; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.integratedmodelling.klab.stac.STACPathExpression; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -class STACPathExpressionTest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private static JsonNode testJson() throws Exception { - String json = """ - { - "node1": { - "node2": [ - { - "name": "something", - "count": 10, - "active": true - }, - { - "name": "other", - "count": 20, - "active": false - } - ] - }, - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - }, - { - "name": "B08", - "common_name": "nir" - } - ], - "properties": { - "cloud_cover": 12.5, - "enabled": true - } - } - """; - - return MAPPER.readTree(json); - } - - @Test - void shouldResolveIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertTrue(expression.matches(root, "something")); - assertFalse(expression.matches(root, "other")); - } - - @Test - void shouldResolveSecondIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[1]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("other", resolved.get(0).asText()); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "something")); - } - - @Test - void shouldResolveWildcardArrayElements() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(2, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertEquals("other", resolved.get(1).asText()); - - assertTrue(expression.matches(root, "something")); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "missing")); - } - - @Test - void shouldMatchEoBandsUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(3, resolved.size()); - assertEquals("B01", resolved.get(0).asText()); - assertEquals("B04", resolved.get(1).asText()); - assertEquals("B08", resolved.get(2).asText()); - - assertTrue(expression.matches(root, "B01")); - assertTrue(expression.matches(root, "B04")); - assertTrue(expression.matches(root, "B08")); - assertFalse(expression.matches(root, "B99")); - } - - @Test - void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>common_name"); - - assertTrue(expression.matches(root, "coastal")); - assertTrue(expression.matches(root, "red")); - assertTrue(expression.matches(root, "nir")); - assertFalse(expression.matches(root, "green")); - } - - @Test - void shouldMatchNumbers() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>count"); - - assertTrue(expression.matches(root, "10")); - assertTrue(expression.matches(root, "10.0")); - assertFalse(expression.matches(root, "20")); - assertFalse(expression.matches(root, "not-a-number")); - } - - @Test - void shouldMatchBooleans() throws Exception { - JsonNode root = testJson(); - - STACPathExpression firstExpression = - STACPathExpression.parse("node1>node2[0]>active"); - - STACPathExpression secondExpression = - STACPathExpression.parse("node1>node2[1]>active"); - - assertTrue(firstExpression.matches(root, "true")); - assertTrue(firstExpression.matches(root, "TRUE")); - assertFalse(firstExpression.matches(root, "false")); - - assertTrue(secondExpression.matches(root, "false")); - assertTrue(secondExpression.matches(root, "FALSE")); - assertFalse(secondExpression.matches(root, "true")); - } - - @Test - void shouldMatchTopLevelProperties() throws Exception { - JsonNode root = testJson(); - - STACPathExpression cloudCoverExpression = - STACPathExpression.parse("properties>cloud_cover"); - - STACPathExpression enabledExpression = - STACPathExpression.parse("properties>enabled"); - - assertTrue(cloudCoverExpression.matches(root, "12.5")); - assertTrue(cloudCoverExpression.matches(root, "12.50")); - assertFalse(cloudCoverExpression.matches(root, "13")); - - assertTrue(enabledExpression.matches(root, "true")); - assertFalse(enabledExpression.matches(root, "false")); - } - - @Test - void shouldReturnEmptyListForMissingPath() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>missing"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "anything")); - } - - @Test - void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("properties[*]>cloud_cover"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "12.5")); - } - - @Test - void shouldRejectInvalidWildcardSyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[abc]>name") - ); - - assertTrue(exception.getMessage().contains("Invalid array index")); - } - - @Test - void shouldRejectNegativeArrayIndex() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[-1]>name") - ); - - assertTrue(exception.getMessage().contains("Array index cannot be negative")); - } - - @Test - void shouldRejectUnexpectedCharactersAfterArraySyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[0]extra>name") - ); - - assertTrue(exception.getMessage().contains("Unexpected characters")); - } - - @Test - void shouldRejectEmptyPathElement() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>>name") - ); - - assertTrue(exception.getMessage().contains("Invalid empty path element")); - } -} \ No newline at end of file From 8d9ce330c3808452e46a5b6bdf0a1ce21f8e3c2f Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 8 May 2026 10:25:11 +0200 Subject: [PATCH 27/74] Avoid NPE in activity --- .../integratedmodelling/klab/engine/runtime/SessionState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java index f9288190a..f3ca25837 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java @@ -350,7 +350,7 @@ public Future submit(String urn, BiConsumer, IArtifact> obse */ IGeometry geom = getGeometry(); activity.setStart(System.currentTimeMillis()); - if (this.currentActivity.getGeometrySet() == null && geom != null) { + if (this.currentActivity != null && this.currentActivity.getGeometrySet() == null && geom != null) { this.currentActivity.setGeometrySet(geom.encode()); } activity.setActivityId(task.getId()); From 689eb676e676cf378c134fe686b7a6016fc49c32 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 8 May 2026 13:23:34 +0200 Subject: [PATCH 28/74] Working with import and encode, need test it --- .../klab/stac/STACEncoder.java | 108 +++++++----- .../klab/stac/STACImporter.java | 6 +- .../klab/stac/STACPathExpression.java | 162 +++++++++--------- .../klab/stac/STACValidator.java | 47 +++-- .../main/resources/ogc/prototypes/stac.kdl | 10 +- .../STACPathExpressionAndAttributeTest.java | 118 +++++++++---- .../KlabIllegalArgumentException.java | 19 +- 7 files changed, 289 insertions(+), 181 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 7eece7172..1e38894e4 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -2,32 +2,39 @@ import java.io.IOException; import java.io.OutputStream; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Date; -import java.util.function.Predicate; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; -import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.api.data.FeatureSource; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.processing.Operations; import org.geotools.geometry.jts.ReferencedEnvelope; -import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; import org.hortonmachine.gears.libs.modules.HMRaster.HMRasterWritableBuilder; import org.hortonmachine.gears.libs.modules.HMRaster.MergeMode; import org.hortonmachine.gears.libs.monitor.LogProgressMonitor; -import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.RegionMap; +import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; import org.integratedmodelling.klab.Authentication; import org.integratedmodelling.klab.Observables; import org.integratedmodelling.klab.api.data.IGeometry; -import org.integratedmodelling.klab.api.data.IResource; import org.integratedmodelling.klab.api.data.IGeometry.Dimension.Type; +import org.integratedmodelling.klab.api.data.IResource; import org.integratedmodelling.klab.api.data.adapters.IKlabData.Builder; import org.integratedmodelling.klab.api.data.adapters.IResourceEncoder; import org.integratedmodelling.klab.api.knowledge.ICodelist; @@ -46,6 +53,7 @@ import org.integratedmodelling.klab.components.time.extents.TimeInstant; import org.integratedmodelling.klab.exceptions.KlabContextualizationException; import org.integratedmodelling.klab.exceptions.KlabIOException; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabIllegalStateException; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; @@ -56,27 +64,16 @@ import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; -import org.integratedmodelling.klab.stac.STACPathExpression.STACAssetPredicate; +import org.integratedmodelling.klab.stac.STACPathExpression.AssetAttribute; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; -import org.geotools.coverage.processing.Operations; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.geotools.api.feature.simple.SimpleFeature; -import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.api.geometry.Position; -import org.geotools.referencing.CRS; -import org.geotools.api.referencing.FactoryException; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; -import java.time.*; -import java.time.format.DateTimeFormatter; import kong.unirest.json.JSONObject; @@ -246,7 +243,6 @@ public void getEncodedData(IResource resource, Map urnParameters String collectionId = collectionData.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collectionData); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - String assetId = resource.getParameters().get("asset", String.class); Integer bandIndex = resource.getParameters().get("band", Integer.class); boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); @@ -325,9 +321,14 @@ public void getEncodedData(IResource resource, Map urnParameters if (collection == null) { scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); } - - Predicate p = STACAssetPredicate.from("eo:bands[*]>name", assetId); - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); } if (bandIndex != null) { // Which means theat it's a Multi Band COG @@ -410,27 +411,20 @@ public void getEncodedData(IResource resource, Map urnParameters // Allow transform ensures the process to finish, but I would not bet on the resulting // data. final boolean allowTransform = true; - Predicate p = STACAssetPredicate.from(STACPathExpression.PREDICATE_EO_BANDS_NAME, assetId); - - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } + // Filter here based on time, since in some STAC collections they don't yet support + // temporal filtering :( like ECDC + items = items.stream().filter(new Predicate(){ + + @Override + public boolean test(HMStacItem item) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime.parse(item.getStartTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } return false; } }).collect(Collectors.toList()); @@ -442,11 +436,17 @@ public boolean test(HMStacItem item) { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } - + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } Set EPSGAtAssets = items.stream() .flatMap(item -> item.getAssets().stream()) - .filter(p) + .filter(predicate) .map(HMStacAsset::getEpsg) .collect(Collectors.toUnmodifiableSet()); @@ -455,7 +455,7 @@ public boolean test(HMStacItem item) { scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); @@ -469,6 +469,24 @@ public boolean test(HMStacItem item) { encoder = new RasterEncoder(); ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } + + private Predicate getAssetPredicate(IResource resource){ + String assetId = resource.getParameters().get("asset", String.class); + if (assetId != null) { + return STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute(AssetAttribute.ID.name(), assetId); + } + String jsonSelector = resource.getParameters().get("jsonSelector", String.class); + String jsonValue = resource.getParameters().get("jsonValue", String.class); + if (jsonSelector != null && !jsonSelector.isBlank() && jsonValue != null) { + try { + return STACPathExpression.STACAssetPredicate.fromHMStacAsset(jsonSelector, jsonValue); + } catch (IllegalArgumentException e) { + throw new KlabIllegalArgumentException("Invalid STAC asset JSON selector: " + jsonSelector, e); + } + } else { + throw new KlabIllegalArgumentException("Search parameters didn't exists"); + } + } private boolean isFeatureInTimeRange(Time time2, SimpleFeature f) { Date datetime = (Date) f.getAttribute("datetime"); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index dc768f509..10e324eb5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -43,6 +43,10 @@ public boolean acceptsMultiple() { // TODO Auto-generated method stub return false; } + + private boolean hasAssetSelector(IParameters parameters) { + return (parameters.contains("asset")||(parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); + } private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) throws MalformedURLException { @@ -59,7 +63,7 @@ private void importCollection(List ret, IParameters parameters, boolean isBulkImport = parameters.contains("bulkImport"); parameters.remove("bulkImport"); - if (!parameters.contains("asset") && !isBulkImport) { + if (!hasAssetSelector(parameters) && !isBulkImport) { Builder builder = buildResource(parameters, project, monitor, collectionId); if (builder != null) { ret.add(builder); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 6bdd725ef..c34270bf6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -1,20 +1,31 @@ package org.integratedmodelling.klab.stac; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import kong.unirest.json.JSONObject; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Function; import java.util.function.Predicate; import org.hortonmachine.gears.io.stac.HMStacAsset; public final class STACPathExpression { - private final List path; - - public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; + private final List path; + + /* + * This works: eo:bands[*]>name + * This probably does not behave as a user might expect: eo:bands>name + * because eo:bands resolves to an array node, and then the next path part tries to read field name directly from an array. + * That returns nothing. That is acceptable, but it should be documented: arrays require either [n] or [*]. + */ + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; private STACPathExpression(List path) { this.path = path; @@ -24,14 +35,13 @@ public static STACPathExpression parse(String jsonPath) { if (jsonPath == null || jsonPath.isBlank()) { throw new IllegalArgumentException("JSON path cannot be empty"); } - return new STACPathExpression(parsePath(jsonPath)); } public boolean matches(JsonNode root, String expectedValue) { List resolvedNodes = resolve(root); - for (JsonNode node : resolvedNodes) { + for(JsonNode node : resolvedNodes) { if (jsonValueEquals(node, expectedValue)) { return true; } @@ -44,10 +54,10 @@ public List resolve(JsonNode root) { List currentNodes = new ArrayList<>(); currentNodes.add(root); - for (PathPart part : path) { + for(PathPart part : path) { List nextNodes = new ArrayList<>(); - for (JsonNode current : currentNodes) { + for(JsonNode current : currentNodes) { if (current == null || current.isNull() || current.isMissingNode()) { continue; } @@ -70,7 +80,7 @@ public List resolve(JsonNode root) { } } else if (part.arrayMode() == ArrayMode.WILDCARD) { if (fieldNode.isArray()) { - for (JsonNode arrayElement : fieldNode) { + for(JsonNode arrayElement : fieldNode) { if (arrayElement != null && !arrayElement.isNull() && !arrayElement.isMissingNode()) { nextNodes.add(arrayElement); } @@ -94,7 +104,7 @@ private static List parsePath(String jsonPath) { List parts = new ArrayList<>(); - for (String token : tokens) { + for(String token : tokens) { String trimmed = token.trim(); if (trimmed.isEmpty()) { @@ -106,7 +116,7 @@ private static List parsePath(String jsonPath) { return parts; } - + private static boolean valueEquals(Object actualValue, String expectedValue) { if (actualValue == null) { return expectedValue == null; @@ -127,17 +137,6 @@ private static boolean valueEquals(Object actualValue, String expectedValue) { return Objects.equals(String.valueOf(actualValue), expectedValue); } - private static boolean numberEquals(Number actualValue, String expectedValue) { - try { - BigDecimal actual = new BigDecimal(actualValue.toString()); - BigDecimal expected = new BigDecimal(expectedValue); - - return actual.compareTo(expected) == 0; - } catch (NumberFormatException e) { - return false; - } - } - private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { return expectedValue == null; @@ -148,24 +147,19 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu } if (actualValue.isNumber()) { - return numberEquals(actualValue, expectedValue); + return numberEquals(actualValue.decimalValue(), expectedValue); } if (actualValue.isBoolean()) { - return Boolean.toString(actualValue.booleanValue()) - .equalsIgnoreCase(expectedValue); - } - - if (actualValue.isTextual()) { - return Objects.equals(actualValue.asText(), expectedValue); + return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); } return Objects.equals(actualValue.asText(), expectedValue); } - private static boolean numberEquals(JsonNode actualValue, String expectedValue) { + private static boolean numberEquals(Number actualValue, String expectedValue) { try { - BigDecimal actual = actualValue.decimalValue(); + BigDecimal actual = new BigDecimal(actualValue.toString()); BigDecimal expected = new BigDecimal(expectedValue); return actual.compareTo(expected) == 0; @@ -173,12 +167,8 @@ private static boolean numberEquals(JsonNode actualValue, String expectedValue) return false; } } - - public record PathPart( - String fieldName, - ArrayMode arrayMode, - Integer arrayIndex - ) { + + public record PathPart(String fieldName, ArrayMode arrayMode, Integer arrayIndex) { public static PathPart parse(String token) { int bracketStart = token.indexOf('['); @@ -190,24 +180,18 @@ public static PathPart parse(String token) { int bracketEnd = token.indexOf(']', bracketStart); if (bracketEnd < 0) { - throw new IllegalArgumentException( - "Invalid array syntax in path element: " + token - ); + throw new IllegalArgumentException("Invalid array syntax in path element: " + token); } if (bracketEnd != token.length() - 1) { - throw new IllegalArgumentException( - "Unexpected characters after array syntax in path element: " + token - ); + throw new IllegalArgumentException("Unexpected characters after array syntax in path element: " + token); } String fieldName = token.substring(0, bracketStart).trim(); String indexText = token.substring(bracketStart + 1, bracketEnd).trim(); if (fieldName.isEmpty()) { - throw new IllegalArgumentException( - "Field name cannot be empty in path element: " + token - ); + throw new IllegalArgumentException("Field name cannot be empty in path element: " + token); } if ("*".equals(indexText)) { @@ -219,52 +203,72 @@ public static PathPart parse(String token) { try { index = Integer.parseInt(indexText); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "Invalid array index in path element: " + token, - e - ); + throw new IllegalArgumentException("Invalid array index in path element: " + token, e); } if (index < 0) { - throw new IllegalArgumentException( - "Array index cannot be negative in path element: " + token - ); + throw new IllegalArgumentException("Array index cannot be negative in path element: " + token); } return new PathPart(fieldName, ArrayMode.INDEX, index); } } - + public enum ArrayMode { - NONE, - INDEX, - WILDCARD + NONE, INDEX, WILDCARD } - - public final class STACAssetPredicate { + + public static final class STACAssetPredicate { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private STACAssetPredicate() { } - - public static Predicate fromJsonPath( - String jsonPath, - String expectedValue - ) { + + public static Predicate fromJsonPath(String jsonPath, String expectedValue, + Function jsonNodeExtractor) { STACPathExpression expression = STACPathExpression.parse(jsonPath); - return asset -> { - if (asset == null || asset.getAssetNode() == null) { + return object -> { + if (object == null) { + return false; + } + + JsonNode node = jsonNodeExtractor.apply(object); + + if (node == null || node.isNull() || node.isMissingNode()) { return false; } - return expression.matches(asset.getAssetNode(), expectedValue); + return expression.matches(node, expectedValue); }; } - public static Predicate fromAssetAttribute( - String attributeName, - String expectedValue - ) { + public static Predicate fromHMStacAsset(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, asset -> asset == null ? null : asset.getAssetNode()); + } + + public static Predicate fromJsonNode(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, node -> node); + } + + public static Predicate fromKongJsonObject(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, STACAssetPredicate::toJsonNode); + } + + private static JsonNode toJsonNode(JSONObject jsonObject) { + if (jsonObject == null) { + return null; + } + + try { + return OBJECT_MAPPER.readTree(jsonObject.toString()); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Cannot convert JSONObject to JsonNode", e); + } + } + + public static Predicate fromHMStacAssetAttribute(String attributeName, String expectedValue) { AssetAttribute attribute = AssetAttribute.fromName(attributeName); return asset -> { @@ -277,14 +281,8 @@ public static Predicate fromAssetAttribute( return valueEquals(actualValue, expectedValue); }; } - - public static Predicate from(String jsonPath, String expectedValue) { - STACPathExpression expression = STACPathExpression.parse(jsonPath); - - return asset -> expression.matches(asset.getAssetNode(), expectedValue); - } } - + public enum AssetAttribute { ID("id") { @@ -342,17 +340,15 @@ public static AssetAttribute fromName(String name) { throw new IllegalArgumentException("Asset attribute name cannot be empty"); } - for (AssetAttribute attribute : values()) { + for(AssetAttribute attribute : values()) { if (attribute.name.equalsIgnoreCase(name.trim())) { return attribute; } } - throw new IllegalArgumentException( - "Unsupported HMStacAsset attribute: " + name - + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason" - ); + throw new IllegalArgumentException("Unsupported HMStacAsset attribute: " + name + + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason"); } } - + } \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index a01646ac0..61a9ddcf4 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import org.integratedmodelling.kim.api.IParameters; import org.integratedmodelling.klab.api.data.IGeometry; @@ -20,6 +21,7 @@ import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor; import org.integratedmodelling.klab.data.resources.Resource; import org.integratedmodelling.klab.data.resources.ResourceBuilder; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnimplementedException; import org.integratedmodelling.klab.rest.CodelistReference; import org.integratedmodelling.klab.rest.MappingReference; @@ -62,18 +64,25 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); + generateCodeList(builder, assetId, asset); + } else if (userData.contains("jsonSelector")) { + if (userData.contains("jsonValue")) { + JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromKongJsonObject( + userData.get("jsonSelector", String.class), + userData.get("jsonValue", String.class) + ); + + for (String assetId : assets.keySet()) { + JSONObject asset = assets.getJSONObject(assetId); + + if (predicate.test(asset)) { + generateCodeList(builder, assetId, asset); + } } - builder.addCodeList(codelist); - } - if (type != null) { - builder.withType(type); + } else { + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue have to be filled"); } } @@ -87,6 +96,22 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni return builder; } + private void generateCodeList(Builder builder, String assetId, JSONObject asset) { + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); + } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); + } + } + private Type readRasterDataType(JSONObject asset) { if (!asset.has("raster:bands")) { return null; diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index 0404a17d6..5ccdd7c9a 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -15,9 +15,17 @@ optional number 'band' "Relevant only for raster resources. - The band for a multi-band raster. Default is 0." + The band index number for a multi-band raster. Default is 0." default 0 + optional text 'jsonSelector' + "JSON selector" + default "" + + optional text 'jsonValue' + "JSON value" + default "" + optional text 'cog' "Relevant only for Resources served as Cloud Optimized GeoTiff, with Public Access" default "" diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java index d24b147f3..8f840105d 100644 --- a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java @@ -18,40 +18,86 @@ class STACPathExpressionAndAttributeTest { private static JsonNode testJson() throws Exception { String json = """ - { - "node1": { - "node2": [ - { - "name": "something", - "count": 10, - "active": true - }, - { - "name": "other", - "count": 20, - "active": false - } - ] - }, - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - }, - { - "name": "B08", - "common_name": "nir" - } - ], - "properties": { - "cloud_cover": 12.5, - "enabled": true - } - } + "{assets": { + "peat_thickness": { + "href": "https://s3.waw4-1.cloudferro.com/ecdc-waw4-1-ekqouvq3otv8hmw0njzuvo0g4dy0ys8r985n7dggjis3erkpn5o/ECDC/Soil/Soil_V2_global_20010101_20211231_peat_thickness.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data" + ], + "Method to produce": "Spatial distribution is based on the prediction using the Quantile Random Forest algorithm, informed by land surface data (soil, climate, organisms, and topography), to develop regional models over six regions for peat thickness. Peat thickness models are based on approximately 27,000 data points. Highest accuracy observed in African peatlands.", + "Link to resource": [ + "https://zenodo.org/records/14183473" + ], + "proj:code": "EPSG:4326", + "proj:geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 180, + -60.007461 + ], + [ + 180, + 88.007948 + ], + [ + -180, + 88.007948 + ], + [ + -180, + -60.007461 + ], + [ + 180, + -60.007461 + ] + ] + ] + }, + "proj:bbox": [ + -180, + -60.007461, + 180, + 88.007948 + ], + "proj:shape": [ + 16477, + 40076 + ], + "proj:transform": [ + 0.008983152841201715, + 0, + -180.004416632, + 0, + -0.008983152841172543, + 88.007948385, + 0, + 0, + 1 + ], + "eo:bands": [ + { + "name": "peat_thickness" + } + ], + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32", + "statistics": { + "minimum": 1.9351852, + "maximum": 1006.69586 + }, + "unit": "cm", + "scale": 1, + "offset": 0 + } + ] + } + }} """; return MAPPER.readTree(json); @@ -62,13 +108,13 @@ void shouldResolveIndexedArrayElement() throws Exception { JsonNode root = testJson(); STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>name"); + STACPathExpression.parse("eo:bands[*]>name"); List resolved = expression.resolve(root); assertEquals(1, resolved.size()); assertEquals("something", resolved.get(0).asText()); - assertTrue(expression.matches(root, "something")); + assertTrue(expression.matches(root, "peat_thickness")); assertFalse(expression.matches(root, "other")); } diff --git a/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java b/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java index cff1a099b..a9a4b028f 100644 --- a/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java +++ b/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java @@ -39,17 +39,28 @@ public KlabIllegalArgumentException() { /** * Instantiates a new klab illegal status exception. * - * @param arg0 the arg 0 + * @param message the message */ - public KlabIllegalArgumentException(String arg0) { - super(arg0); + public KlabIllegalArgumentException(String message) { + super(message); // TODO Auto-generated constructor stub } + + /** + * Instantiates a new klab illegal status exception. + * + * @param message the message + * @param e the exception + */ + public KlabIllegalArgumentException(String message, Throwable e) { + super(message, e); + // TODO Auto-generated constructor stub + } /** * Instantiates a new klab illegal status exception. * - * @param e the e + * @param e the exception */ public KlabIllegalArgumentException(Throwable e) { super(e); From 06d5d511815ac5f1b03f41dfe94277c6cac31739 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 8 May 2026 13:42:30 +0200 Subject: [PATCH 29/74] Small improvements, still not test --- .../klab/stac/STACEncoder.java | 5 +++-- .../klab/stac/STACPathExpression.java | 15 ++++++++++++--- .../klab/stac/STACValidator.java | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 1e38894e4..3284d5517 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -473,7 +473,7 @@ public boolean test(HMStacItem item) { private Predicate getAssetPredicate(IResource resource){ String assetId = resource.getParameters().get("asset", String.class); if (assetId != null) { - return STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute(AssetAttribute.ID.name(), assetId); + return STACPathExpression.STACAssetPredicate.fromHMStacAssetId(assetId); } String jsonSelector = resource.getParameters().get("jsonSelector", String.class); String jsonValue = resource.getParameters().get("jsonValue", String.class); @@ -484,7 +484,8 @@ private Predicate getAssetPredicate(IResource resource){ throw new KlabIllegalArgumentException("Invalid STAC asset JSON selector: " + jsonSelector, e); } } else { - throw new KlabIllegalArgumentException("Search parameters didn't exists"); + throw new KlabIllegalArgumentException( + "Either asset or both jsonSelector and jsonValue must be provided"); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index c34270bf6..7799ef556 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -142,7 +142,7 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu return expectedValue == null; } - if (expectedValue == null) { + if (expectedValue == null || !actualValue.isValueNode()) { return false; } @@ -151,10 +151,15 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu } if (actualValue.isBoolean()) { - return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); + return Boolean.toString(actualValue.booleanValue()) + .equalsIgnoreCase(expectedValue); + } + + if (actualValue.isTextual()) { + return Objects.equals(actualValue.asText(), expectedValue); } - return Objects.equals(actualValue.asText(), expectedValue); + return false; } private static boolean numberEquals(Number actualValue, String expectedValue) { @@ -281,6 +286,10 @@ public static Predicate fromHMStacAssetAttribute(String attributeNa return valueEquals(actualValue, expectedValue); }; } + + public static Predicate fromHMStacAssetId(String expectedValue) { + return fromHMStacAssetAttribute("id", expectedValue); + } } public enum AssetAttribute { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 61a9ddcf4..041c5243e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -82,7 +82,7 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni } } } else { - throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue have to be filled"); + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); } } From 8653a22f3aab7887f16a3e5fde2144e96c7e6d74 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Sat, 9 May 2026 11:43:55 +0200 Subject: [PATCH 30/74] adds support to get the features from stac, and to rasterize based on presence --- .../klab/stac/STACEncoder.java | 83 +++++++----- .../klab/stac/STACValidator.java | 35 ++--- .../stac/extensions/STACFeatureExtension.java | 121 ++++++++++++++++++ .../main/resources/ogc/prototypes/stac.kdl | 10 +- .../src/main/resources/ogc/prototypes/wfs.kdl | 2 +- 5 files changed, 196 insertions(+), 55 deletions(-) create mode 100644 adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9c331a850..e227cf93e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -12,6 +12,7 @@ import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.api.data.FeatureSource; import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.CRS; import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; import org.hortonmachine.gears.io.stac.HMStacItem; @@ -58,23 +59,31 @@ import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; +import org.integratedmodelling.klab.stac.extensions.STACFeatureExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; import org.geotools.coverage.processing.Operations; +import org.geotools.data.geojson.GeoJSONReader; +import org.geotools.data.memory.MemoryDataStore; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; import org.geotools.api.feature.simple.SimpleFeature; import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.api.geometry.Position; -import org.geotools.referencing.CRS; -import org.geotools.api.referencing.FactoryException; +import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaOptions.Item; + import java.time.*; import java.time.format.DateTimeFormatter; +import org.locationtech.jts.operation.union.UnaryUnionOp; +import org.geotools.feature.simple.SimpleFeatureBuilder; +import org.geotools.feature.simple.SimpleFeatureTypeBuilder; +import org.geotools.data.DataUtilities; +import java.util.ArrayList; import kong.unirest.json.JSONObject; @@ -246,7 +255,6 @@ public void getEncodedData(IResource resource, Map urnParameters JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); String assetId = resource.getParameters().get("asset", String.class); Integer bandIndex = resource.getParameters().get("band", Integer.class); - boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); // This is part of a WIP that will be removed in the future boolean isIIASA = catalogUrl.contains("iiasa.blob"); @@ -266,8 +274,8 @@ public void getEncodedData(IResource resource, Map urnParameters ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } - - // These are the static STAC catalogs + + // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); Time time2 = time; //TODO make the time and query time different @@ -355,7 +363,8 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); return; } - + + System.out.println("Found the Search Option!"); LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -385,10 +394,26 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } ITimeInstant start = time.getStart(); ITimeInstant end = time.getEnd(); + + if (assetId == null) { + scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; + try { + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, start, end); + } catch (Exception e) { + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + } + encoder = new VectorEncoder(); + ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + return; + } + //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) - + GridCoverage2D coverage = null; try { + System.out.println("Searcing STAC.."); List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -422,7 +447,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou var p = new Predicate() { @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates var bands = asset.getAssetNode().get("eo:bands"); if (bands != null && bands.isArray()) { var bandsArray = (ArrayNode) bands; @@ -438,34 +463,30 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou }; // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } - return false; - } - }).collect(Collectors.toList()); - + items = items.stream().filter(item -> { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + return start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd; + }).collect(Collectors.toList()); if (items.size() == 0) { throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } + // Once the support for customized predicate is added, we can apply for features as well + Set EPSGAtAssets = items.stream() diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index a01646ac0..26219e6a5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -59,28 +59,29 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (userData.contains("asset")) { String assetId = userData.get("asset", String.class); - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); - JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); + JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); + + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); + } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); } - builder.addCodeList(codelist); - } - if (type != null) { - builder.withType(type); } - } - + if (userData.contains("cog")) { if (userData.get("cog") != null) { builder.withType(Type.NUMBER); - } + } + } readMetadata(collectionData, builder); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java new file mode 100644 index 000000000..bc43afe08 --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -0,0 +1,121 @@ +package org.integratedmodelling.klab.stac.extensions; + +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.geotools.api.data.FeatureSource; +import org.geotools.data.geojson.GeoJSONReader; +import org.geotools.data.memory.MemoryDataStore; +import org.integratedmodelling.klab.stac.STACUtils; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.hortonmachine.gears.io.stac.HMStacItem; + +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; +import org.integratedmodelling.klab.api.observations.scale.IScale; +import org.integratedmodelling.klab.api.observations.scale.space.IEnvelope; +import org.integratedmodelling.klab.api.observations.scale.space.IGrid; +import org.integratedmodelling.klab.api.observations.scale.time.ITimeInstant; +import org.integratedmodelling.klab.stac.STACUtils; +import java.time.format.DateTimeFormatter; +import java.time.*; + +public class STACFeatureExtension { + public static FeatureSource getFeatures(JSONObject catalogData, String collectionId, List bbox, ITimeInstant start, ITimeInstant end) throws Exception { + + System.out.println("Getting features from STAC!"); + String searchEndpoint = STACUtils.getLinkTo(catalogData, "search") + .orElseThrow(() -> new Exception("Search Link not found for the Catalog")); + + List featureList = new ArrayList<>(); + System.out.println(searchEndpoint); + + JSONArray bboxArray = new JSONArray(); + for (Double v : bbox) { + bboxArray.put(v); + } + + + JSONObject searchPayload = new JSONObject() + .put("limit", 100) + .put("bbox", bboxArray) + .put("collections", new JSONArray().put(collectionId)); + + while (searchEndpoint != null) { + + HttpResponse response = Unirest + .post(searchEndpoint) + .header("Content-Type", "application/json") + .body(searchPayload) + .asJson(); + + JSONObject body = response.getBody().getObject(); + System.out.println(body); + + JSONArray features = body.getJSONArray("features"); + System.out.println(features.length()); + + Iterator featureIterator = features.iterator(); + + while (featureIterator.hasNext()) { + try { + JSONObject feature = (JSONObject) featureIterator.next(); + SimpleFeature feat = GeoJSONReader.parseFeature(feature.toString()); + HMStacItem item = HMStacItem.fromSimpleFeature(feat); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + if (item.getStartTimestamp() != null && item.getEndTimestamp() != null) { + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { + featureList.add(feat); + } + } else { + featureList.add(feat); + } + + + } catch (Exception e) { + e.printStackTrace(); + } + } + searchEndpoint = null; + if (body.has("links")) { + JSONArray links = body.getJSONArray("links"); + + for (Object obj : links) { + JSONObject link = (JSONObject) obj; + + if ("next".equalsIgnoreCase(link.optString("rel"))) { + String searchEndpointNew = link.getString("href"); + if (searchEndpointNew.equals(searchEndpoint)) { + searchEndpoint = null; + } + break; + } + } + } + } + System.out.println(featureList.size()); + SimpleFeatureType type = featureList.get(0).getType(); + MemoryDataStore dataStore = new org.geotools.data.memory.MemoryDataStore(type); + dataStore.addFeatures(featureList); + return dataStore.getFeatureSource(type.getTypeName()); + } +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index 0404a17d6..e1d8d2b7c 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -8,19 +8,17 @@ { final text 'collection' - "The URL pointing to the STAC collection file that contains the resource dataset." + "[REQUIRED] The URL pointing to the STAC collection file that contains the resource dataset." optional text 'asset' - "The asset that is going to be retrieved from the items. Left it blank when the information is stored in the feature." + "The asset that is going to be retrieved from the items. Should be left it blank when the information is stored in the feature." optional number 'band' "Relevant only for raster resources. - The band for a multi-band raster. Default is 0." - default 0 + The band for a multi-band raster. Default it will pick up the first one." optional text 'cog' - "Relevant only for Resources served as Cloud Optimized GeoTiff, with Public Access" - default "" + "Relevant only for Resources served as Cloud Optimised GeoTiff, with Public Access" optional enum 'bandmixer' "Relevant only for raster resources.\n diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl index 24e2f0a11..dad0ee466 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl @@ -35,6 +35,6 @@ optional enum 'axisOrder' "Select the order of the XY axis: 'lat_lon' (default), or 'lon_lat'." - default "lat_lon" + default lat_lon values lat_lon, lon_lat } \ No newline at end of file From 83e4476f54ab709b0f841479c9b608567bf35f15 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Sat, 9 May 2026 11:49:11 +0200 Subject: [PATCH 31/74] removes prints, and adds some sanity checks --- .../org/integratedmodelling/klab/stac/STACEncoder.java | 2 -- .../klab/stac/extensions/STACFeatureExtension.java | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index e227cf93e..6b0296b2d 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -364,7 +364,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou return; } - System.out.println("Found the Search Option!"); LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -413,7 +412,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou GridCoverage2D coverage = null; try { - System.out.println("Searcing STAC.."); List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java index bc43afe08..aa668f566 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -30,12 +30,10 @@ public class STACFeatureExtension { public static FeatureSource getFeatures(JSONObject catalogData, String collectionId, List bbox, ITimeInstant start, ITimeInstant end) throws Exception { - System.out.println("Getting features from STAC!"); String searchEndpoint = STACUtils.getLinkTo(catalogData, "search") .orElseThrow(() -> new Exception("Search Link not found for the Catalog")); List featureList = new ArrayList<>(); - System.out.println(searchEndpoint); JSONArray bboxArray = new JSONArray(); for (Double v : bbox) { @@ -57,10 +55,7 @@ public static FeatureSource getFeatures(JSONOb .asJson(); JSONObject body = response.getBody().getObject(); - System.out.println(body); - JSONArray features = body.getJSONArray("features"); - System.out.println(features.length()); Iterator featureIterator = features.iterator(); @@ -112,7 +107,10 @@ public static FeatureSource getFeatures(JSONOb } } } - System.out.println(featureList.size()); + if (featureList.isEmpty()) { + throw new Exception("No features found for the given parameters"); + } + SimpleFeatureType type = featureList.get(0).getType(); MemoryDataStore dataStore = new org.geotools.data.memory.MemoryDataStore(type); dataStore.addFeatures(featureList); From 9a814630d7a058dd97cf2cb977c71b66b4c53b37 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 11:00:50 +0200 Subject: [PATCH 32/74] fixes stac validator and importer to allow import --- .../klab/stac/STACAssetMapParser.java | 7 ++- .../klab/stac/STACCollectionParser.java | 38 +++++++++++---- .../klab/stac/STACEncoder.java | 2 +- .../klab/stac/STACImporter.java | 48 +++++++++++++++++-- .../klab/stac/STACValidator.java | 4 +- 5 files changed, 83 insertions(+), 16 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java index 3be8c4a50..7dacca18b 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java @@ -10,6 +10,11 @@ public static Set readAssetNames(JSONObject assets) { } public static JSONObject getAsset(JSONObject assetMap, String assetId) { - return assetMap.getJSONObject(assetId); + /* + * Not failing if the assetId is not found, the reason being that in + * some collections the items have different assets, and the Asset naming related conventions that MSFT + * Planetary follows is not necessarily followed by evryone! + */ + return assetMap.optJSONObject(assetId); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index c351c324c..a7bcb0915 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -16,6 +16,7 @@ import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; public class STACCollectionParser { @@ -62,7 +63,7 @@ public static IGeometry readGeometry(JSONObject collection) { * @return The asset list as a JSON * @throws KlabResourceAccessException */ - public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObject collection) throws KlabResourceAccessException { + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) throws KlabResourceAccessException { String collectionId = collection.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); @@ -91,21 +92,42 @@ public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObje return itemData.getJSONObject("assets"); } throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); - } + } - // TODO Move the query to another place. - String parameters = "?collections=" + collectionId + "&limit=1"; - HttpResponse response = Unirest.get(searchEndpoint.get() + parameters).asJson(); + JSONObject searchPayload = new JSONObject() + .put("limit", 100) + .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) + .put("collections", new JSONArray().put(collectionId)); + + HttpResponse response = Unirest + .post(searchEndpoint.get()) + .header("Content-Type", "application/json") + .body(searchPayload) + .asJson(); if (!response.isSuccess()) { - throw new KlabResourceAccessException(); //TODO set message + throw new KlabResourceAccessException("Unable to import collection, Search failed"); //TODO set message } JSONObject searchResponse = response.getBody().getObject(); if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException(); // TODO set message there is no feature + throw new KlabResourceAccessException("No features were found in the collection to be imported"); // TODO set message there is no feature + } + + JSONArray features = searchResponse.getJSONArray("features"); + + for (int i = 0; i < features.length(); i++) { + JSONObject feature = features.getJSONObject(i); + + JSONObject assetInfo = feature + .getJSONObject("assets") + .optJSONObject(assetId); + + if (assetInfo != null) { + return assetInfo; + } } - return searchResponse.getJSONArray("features").getJSONObject(0).getJSONObject("assets"); + throw new KlabResourceAccessException("No Asset with ID: " + assetId + " was found in the collection"); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 6b0296b2d..4e7dd55f7 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -53,7 +53,7 @@ import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.ogc.STACAdapter; -import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; +import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index dc768f509..1f085b4ed 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -68,7 +68,42 @@ private void importCollection(List ret, IParameters parameters, } return; } - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + + String assetId = parameters.get("asset"); + JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + + /* + * If the particular asset Id wasn't found, then + * still proceed to create the resource since it can happen + * for a number of reasons. Pagination, STAC Backend etc. + * In that case however, a better handling of S3 URL needs to figured out + */ + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + throw new Exception("Unsupported media type for the asset"); + } + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + } + + + parameters.put("asset", assetId); + String resourceUrn = collectionId + "-" + assetId; + Builder builder = buildResource(parameters, project, monitor, resourceUrn); + if (builder != null) { + ret.add(builder); + } else { + monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); + } + + // Think of a way to do the REGEX matching. Possibly not worth it to support + // this for something as diverse as STAC + + /* Set assetIds = STACAssetMapParser.readAssetNames(assets); for(String assetId : assetIds) { if (regex != null && !assetId.matches(regex)) { @@ -77,10 +112,13 @@ private void importCollection(List ret, IParameters parameters, } JSONObject assetData = STACAssetMapParser.getAsset(assets, assetId); - if (!STACAssetParser.isSupportedMediaType(assetData)) { - Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); - continue; + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); + continue; + } } + parameters.put("asset", assetId); String resourceUrn = collectionId + "-" + assetId; String href = assetData.getString("href"); @@ -97,6 +135,8 @@ private void importCollection(List ret, IParameters parameters, monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); } } + + */ } private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 26219e6a5..4b73caf0e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -59,8 +59,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (userData.contains("asset")) { String assetId = userData.get("asset", String.class); - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); - JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); + JSONObject asset = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + //JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); Type type = readRasterDataType(asset); // Currently, only files:values is supported. If needed, the classification extension could be used too. From a27c8de7bc2fba92b7cad577e14cdf7ddaebd825 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 11:35:41 +0200 Subject: [PATCH 33/74] adds conversion to str --- .../java/org/integratedmodelling/klab/stac/STACImporter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 1f085b4ed..5b6e8c6c9 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -45,7 +45,7 @@ public boolean acceptsMultiple() { } private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) - throws MalformedURLException { + throws Exception { String collectionUrl = parameters.get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = STACCollectionParser.readCollectionId(collectionData); @@ -69,7 +69,7 @@ private void importCollection(List ret, IParameters parameters, return; } - String assetId = parameters.get("asset"); + String assetId = (String) parameters.get("asset"); JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); /* From b453fd1e1b9ac7ed202563a1c70a997bd65ba48a Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 11 May 2026 11:53:38 +0200 Subject: [PATCH 34/74] Changes to assume assets is not an array but a list of json object --- .../klab/stac/STACPathExpression.java | 135 ++-- .../STACPathExpressionAndAttributeTest.java | 591 ------------------ .../ogc/stac/test/STACPathExpressionTest.java | 407 ++++++++++++ 3 files changed, 496 insertions(+), 637 deletions(-) delete mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java create mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 7799ef556..0a16ae462 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.TextNode; import kong.unirest.json.JSONObject; @@ -19,13 +20,8 @@ public final class STACPathExpression { private final List path; - /* - * This works: eo:bands[*]>name - * This probably does not behave as a user might expect: eo:bands>name - * because eo:bands resolves to an array node, and then the next path part tries to read field name directly from an array. - * That returns nothing. That is acceptable, but it should be documented: arrays require either [n] or [*]. - */ - public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands.name"; + public static final String MEDIA_TYPE_BANDS_NAME = "type"; private STACPathExpression(List path) { this.path = path; @@ -54,39 +50,15 @@ public List resolve(JsonNode root) { List currentNodes = new ArrayList<>(); currentNodes.add(root); - for(PathPart part : path) { + for (PathPart part : path) { List nextNodes = new ArrayList<>(); - for(JsonNode current : currentNodes) { + for (JsonNode current : currentNodes) { if (current == null || current.isNull() || current.isMissingNode()) { continue; } - JsonNode fieldNode = current.get(part.fieldName()); - - if (fieldNode == null || fieldNode.isNull() || fieldNode.isMissingNode()) { - continue; - } - - if (part.arrayMode() == ArrayMode.NONE) { - nextNodes.add(fieldNode); - } else if (part.arrayMode() == ArrayMode.INDEX) { - if (fieldNode.isArray()) { - JsonNode indexedNode = fieldNode.get(part.arrayIndex()); - - if (indexedNode != null && !indexedNode.isNull() && !indexedNode.isMissingNode()) { - nextNodes.add(indexedNode); - } - } - } else if (part.arrayMode() == ArrayMode.WILDCARD) { - if (fieldNode.isArray()) { - for(JsonNode arrayElement : fieldNode) { - if (arrayElement != null && !arrayElement.isNull() && !arrayElement.isMissingNode()) { - nextNodes.add(arrayElement); - } - } - } - } + resolvePart(current, part, nextNodes); } currentNodes = nextNodes; @@ -99,8 +71,87 @@ public List resolve(JsonNode root) { return currentNodes; } + private static void resolvePart(JsonNode current, PathPart part, List nextNodes) { + if (current.isArray()) { + resolveFromArray(current, part, nextNodes); + } else if (current.isObject()) { + resolveFromObject(current, part, nextNodes); + } + } + + private static void resolveFromArray(JsonNode arrayNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + JsonNode indexedNode = arrayNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + resolvePart(indexedNode, new PathPart(part.fieldName(), ArrayMode.NONE, null), nextNodes); + } + + + } else { + for (JsonNode element : arrayNode) { + if (isUsable(element)) { + resolvePart(element, part, nextNodes); + } + } + } + } + + private static void resolveFromObject(JsonNode objectNode, PathPart part, List nextNodes) { + JsonNode directField = objectNode.get(part.fieldName()); + + if (isUsable(directField)) { + addResolvedField(directField, part, nextNodes); + } else { + objectNode.fields().forEachRemaining(entry -> { + String mapKey = entry.getKey(); + JsonNode mapValue = entry.getValue(); + + if (!isUsable(mapValue) || !mapValue.isObject()) { + return; + } + + if ("id".equalsIgnoreCase(part.fieldName())) { + nextNodes.add(TextNode.valueOf(mapKey)); + return; + } + + JsonNode nestedField = mapValue.get(part.fieldName()); + + if (isUsable(nestedField)) { + addResolvedField(nestedField, part, nextNodes); + } + }); + + } + } + + private static void addResolvedField(JsonNode fieldNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + if (fieldNode.isArray()) { + JsonNode indexedNode = fieldNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + nextNodes.add(indexedNode); + } + } + } else if (fieldNode.isArray()) { + for (JsonNode element : fieldNode) { + if (isUsable(element)) { + nextNodes.add(element); + } + } + } else { + nextNodes.add(fieldNode); + } + } + + private static boolean isUsable(JsonNode node) { + return node != null && !node.isNull() && !node.isMissingNode(); + } + private static List parsePath(String jsonPath) { - String[] tokens = jsonPath.split(">"); + String[] tokens = jsonPath.split("\\."); List parts = new ArrayList<>(); @@ -151,8 +202,7 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu } if (actualValue.isBoolean()) { - return Boolean.toString(actualValue.booleanValue()) - .equalsIgnoreCase(expectedValue); + return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); } if (actualValue.isTextual()) { @@ -199,28 +249,21 @@ public static PathPart parse(String token) { throw new IllegalArgumentException("Field name cannot be empty in path element: " + token); } - if ("*".equals(indexText)) { - return new PathPart(fieldName, ArrayMode.WILDCARD, null); - } - int index; - try { index = Integer.parseInt(indexText); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid array index in path element: " + token, e); } - if (index < 0) { throw new IllegalArgumentException("Array index cannot be negative in path element: " + token); } - return new PathPart(fieldName, ArrayMode.INDEX, index); } } public enum ArrayMode { - NONE, INDEX, WILDCARD + NONE, INDEX } public static final class STACAssetPredicate { @@ -286,7 +329,7 @@ public static Predicate fromHMStacAssetAttribute(String attributeNa return valueEquals(actualValue, expectedValue); }; } - + public static Predicate fromHMStacAssetId(String expectedValue) { return fromHMStacAssetAttribute("id", expectedValue); } diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java deleted file mode 100644 index 8f840105d..000000000 --- a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java +++ /dev/null @@ -1,591 +0,0 @@ -package org.integratedmodelling.klab.ogc.stac.test; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.hortonmachine.gears.io.stac.HMStacAsset; -import org.integratedmodelling.klab.stac.STACPathExpression; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.function.Predicate; - -import static org.junit.jupiter.api.Assertions.*; - -class STACPathExpressionAndAttributeTest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private static JsonNode testJson() throws Exception { - String json = """ - "{assets": { - "peat_thickness": { - "href": "https://s3.waw4-1.cloudferro.com/ecdc-waw4-1-ekqouvq3otv8hmw0njzuvo0g4dy0ys8r985n7dggjis3erkpn5o/ECDC/Soil/Soil_V2_global_20010101_20211231_peat_thickness.tif", - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "roles": [ - "data" - ], - "Method to produce": "Spatial distribution is based on the prediction using the Quantile Random Forest algorithm, informed by land surface data (soil, climate, organisms, and topography), to develop regional models over six regions for peat thickness. Peat thickness models are based on approximately 27,000 data points. Highest accuracy observed in African peatlands.", - "Link to resource": [ - "https://zenodo.org/records/14183473" - ], - "proj:code": "EPSG:4326", - "proj:geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 180, - -60.007461 - ], - [ - 180, - 88.007948 - ], - [ - -180, - 88.007948 - ], - [ - -180, - -60.007461 - ], - [ - 180, - -60.007461 - ] - ] - ] - }, - "proj:bbox": [ - -180, - -60.007461, - 180, - 88.007948 - ], - "proj:shape": [ - 16477, - 40076 - ], - "proj:transform": [ - 0.008983152841201715, - 0, - -180.004416632, - 0, - -0.008983152841172543, - 88.007948385, - 0, - 0, - 1 - ], - "eo:bands": [ - { - "name": "peat_thickness" - } - ], - "raster:bands": [ - { - "sampling": "area", - "data_type": "float32", - "statistics": { - "minimum": 1.9351852, - "maximum": 1006.69586 - }, - "unit": "cm", - "scale": 1, - "offset": 0 - } - ] - } - }} - """; - - return MAPPER.readTree(json); - } - - @Test - void shouldResolveIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertTrue(expression.matches(root, "peat_thickness")); - assertFalse(expression.matches(root, "other")); - } - - @Test - void shouldResolveSecondIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[1]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("other", resolved.get(0).asText()); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "something")); - } - - @Test - void shouldResolveWildcardArrayElements() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(2, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertEquals("other", resolved.get(1).asText()); - - assertTrue(expression.matches(root, "something")); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "missing")); - } - - @Test - void shouldMatchEoBandsUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(3, resolved.size()); - assertEquals("B01", resolved.get(0).asText()); - assertEquals("B04", resolved.get(1).asText()); - assertEquals("B08", resolved.get(2).asText()); - - assertTrue(expression.matches(root, "B01")); - assertTrue(expression.matches(root, "B04")); - assertTrue(expression.matches(root, "B08")); - assertFalse(expression.matches(root, "B99")); - } - - @Test - void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>common_name"); - - assertTrue(expression.matches(root, "coastal")); - assertTrue(expression.matches(root, "red")); - assertTrue(expression.matches(root, "nir")); - assertFalse(expression.matches(root, "green")); - } - - @Test - void shouldMatchNumbers() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>count"); - - assertTrue(expression.matches(root, "10")); - assertTrue(expression.matches(root, "10.0")); - assertFalse(expression.matches(root, "20")); - assertFalse(expression.matches(root, "not-a-number")); - } - - @Test - void shouldMatchBooleans() throws Exception { - JsonNode root = testJson(); - - STACPathExpression firstExpression = - STACPathExpression.parse("node1>node2[0]>active"); - - STACPathExpression secondExpression = - STACPathExpression.parse("node1>node2[1]>active"); - - assertTrue(firstExpression.matches(root, "true")); - assertTrue(firstExpression.matches(root, "TRUE")); - assertFalse(firstExpression.matches(root, "false")); - - assertTrue(secondExpression.matches(root, "false")); - assertTrue(secondExpression.matches(root, "FALSE")); - assertFalse(secondExpression.matches(root, "true")); - } - - @Test - void shouldMatchTopLevelProperties() throws Exception { - JsonNode root = testJson(); - - STACPathExpression cloudCoverExpression = - STACPathExpression.parse("properties>cloud_cover"); - - STACPathExpression enabledExpression = - STACPathExpression.parse("properties>enabled"); - - assertTrue(cloudCoverExpression.matches(root, "12.5")); - assertTrue(cloudCoverExpression.matches(root, "12.50")); - assertFalse(cloudCoverExpression.matches(root, "13")); - - assertTrue(enabledExpression.matches(root, "true")); - assertFalse(enabledExpression.matches(root, "false")); - } - - @Test - void shouldReturnEmptyListForMissingPath() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>missing"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "anything")); - } - - @Test - void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("properties[*]>cloud_cover"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "12.5")); - } - - @Test - void shouldRejectInvalidWildcardSyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[abc]>name") - ); - - assertTrue(exception.getMessage().contains("Invalid array index")); - } - - @Test - void shouldRejectNegativeArrayIndex() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[-1]>name") - ); - - assertTrue(exception.getMessage().contains("Array index cannot be negative")); - } - - @Test - void shouldRejectUnexpectedCharactersAfterArraySyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[0]extra>name") - ); - - assertTrue(exception.getMessage().contains("Unexpected characters")); - } - - @Test - void shouldRejectEmptyPathElement() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>>name") - ); - - assertTrue(exception.getMessage().contains("Invalid empty path element")); - } - - // Test by attributes - - @Test - void shouldMatchAssetByIdAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldNotMatchAssetByDifferentIdAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B08"); - - assertFalse(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByTitleAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Surface reflectance band 4", - "type": "image/tiff; application=geotiff" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute( - "title", - "Surface reflectance band 4" - ); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByTypeAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute( - "type", - "image/tiff; application=geotiff" - ); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByEpsgAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632"); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByEpsgAttributeUsingDecimalEquivalent() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632.0"); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldNotMatchAssetByDifferentEpsgAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "4326"); - - assertFalse(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByValidAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("valid", "true"); - - /* - * One caveat: HMStacAsset computes isValid through HMStacAssetHandlers.getHandler(this). - * Depending on your test classpath and supported MIME types, asset.isValid() - * may be either true or false. That is why the validity test uses: - * assertEquals(asset.isValid(), predicate.test(asset)); - */ - assertEquals(asset.isValid(), predicate.test(asset)); - } - - // JSON-path-through-asset tests - @Test - void shouldMatchAssetUsingJsonPathPredicate() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - } - ] - } - """); - - HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromJsonPath( - "eo:bands[*]>name", - "B04" - ); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldNotMatchAssetUsingJsonPathPredicateWhenValueIsMissing() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - } - ] - } - """); - - HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromJsonPath( - "eo:bands[*]>name", - "B08" - ); - - assertFalse(predicate.test(asset)); - } - - @Test - void shouldMatchAssetUsingDefaultJsonPathPredicateFactory() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "eo:bands": [ - { - "name": "B04" - } - ] - } - """); - - HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.from( - "eo:bands[*]>name", - "B04" - ); - - assertTrue(predicate.test(asset)); - } - - // negative/null-safety tests: - @Test - void shouldReturnFalseWhenAssetIsNullForAttributePredicate() { - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); - - assertFalse(predicate.test(null)); - } - - @Test - void shouldReturnFalseWhenAssetIsNullForJsonPathPredicate() { - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromJsonPath( - "eo:bands[*]>name", - "B04" - ); - - assertFalse(predicate.test(null)); - } - - @Test - void shouldRejectUnsupportedAssetAttribute() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( - "unsupportedAttribute", - "value" - ) - ); - - assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); - } - - @Test - void shouldRejectEmptyAssetAttributeName() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( - " ", - "value" - ) - ); - - assertTrue(exception.getMessage().contains("Asset attribute name cannot be empty")); - } - - -} \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java new file mode 100644 index 000000000..11c75358e --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java @@ -0,0 +1,407 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +class STACPathExpressionTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static JsonNode stacItemNode() throws Exception { + String json = """ + { + "assets": { + "CHELSA_bio12_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_bio12_1981-2010_V.2.1.tif", + "title": "Climate Annual Precipitation (CHELSA bio12)", + "description": "Annual precipitation data from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "proj:bbox": [ + -180.0, + -90.0, + 180.0, + 90.0 + ], + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "bio12", + "description": "Annual precipitation" + } + ], + "file:size": 2608605060, + "raster:bands": [ + { + "nodata": -99999, + "data_type": "int32" + } + ] + }, + "CHELSA_gdd5_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gdd5_1981-2010_V.2.1.tif", + "title": "Climate Growing Degree Days > 5°C (CHELSA gdd5)", + "description": "Growing degree days (T > 5°C) from the CHELSA Climate dataset for the period 1981-2010", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gdd5", + "description": "Growing degree days with temperature > 5°C" + } + ], + "file:size": 3520234749, + "raster:bands": [ + { + "nodata": 2147483647, + "data_type": "int32" + } + ] + }, + "CHELSA_gsp_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gsp_1981-2010_V.2.1.tif", + "title": "Climate Precipitation in Growing Season (CHELSA gsp)", + "description": "Precipitation during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:3857", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gsp", + "description": "Precipitation in growing season" + } + ], + "file:size": 2447532198, + "raster:bands": [ + { + "nodata": 2147483648, + "data_type": "float32" + } + ] + }, + "CHELSA_gst_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gst_1981-2010_V.2.1.tif", + "title": "Climate Mean Temperature in Growing Season (CHELSA gst)", + "description": "Mean temperature during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gst", + "description": "Mean temperature in growing season" + } + ], + "file:size": 308275230, + "raster:bands": [ + { + "nodata": 65535, + "data_type": "int32" + } + ] + } + } + } + """; + + return OBJECT_MAPPER.readTree(json); + } + + private static JsonNode assetNode(String assetId) throws Exception { + return stacItemNode().get("assets").get(assetId); + } + + private static HMStacAsset bio12Asset() throws Exception { + return new HMStacAsset( + "CHELSA_bio12_1981-2010_V.2.1", + assetNode("CHELSA_bio12_1981-2010_V.2.1") + ); + } + + private static HMStacAsset gstAsset() throws Exception { + return new HMStacAsset( + "CHELSA_gst_1981-2010_V.2.1", + assetNode("CHELSA_gst_1981-2010_V.2.1") + ); + } + + @Test + void stacPathExpressionMatchesAssetByVirtualIdFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.id"); + + assertTrue(expression.matches(root, "CHELSA_gst_1981-2010_V.2.1")); + assertTrue(expression.matches(root, "CHELSA_bio12_1981-2010_V.2.1")); + assertFalse(expression.matches(root, "CHELSA_unknown")); + } + + @Test + void stacPathExpressionResolvesAllAssetIdsFromAssetsObjectMapKeys() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.id").resolve(root); + + assertEquals(4, resolved.size()); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_bio12_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gdd5_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gsp_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gst_1981-2010_V.2.1"))); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingImplicitArrayExpansion() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands.name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gdd5")); + assertTrue(expression.matches(root, "gsp")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingExplicitArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands[0].name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesAssetMediaTypeFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.type"); + + assertTrue(expression.matches(root, "image/tiff")); + assertFalse(expression.matches(root, "application/json")); + } + + @Test + void stacPathExpressionMatchesRasterBandDataTypeUsingArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.raster:bands[0].data_type"); + + assertTrue(expression.matches(root, "int32")); + assertTrue(expression.matches(root, "float32")); + assertFalse(expression.matches(root, "uint8")); + } + + @Test + void stacPathExpressionMatchesNumericValuesUsingNumericComparison() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.file:size"); + + assertTrue(expression.matches(root, "2608605060")); + assertTrue(expression.matches(root, "308275230.0")); + assertFalse(expression.matches(root, "123")); + } + + @Test + void hmStacAssetPredicateMatchesJsonExpressionAgainstAssetNode() throws Exception { + HMStacAsset asset = bio12Asset(); + + Predicate bandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate indexedBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands[0].name", "bio12"); + + Predicate typePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("type", "image/tiff"); + + Predicate rasterDataTypePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("raster:bands[0].data_type", "int32"); + + assertTrue(bandPredicate.test(asset)); + assertTrue(indexedBandPredicate.test(asset)); + assertTrue(typePredicate.test(asset)); + assertTrue(rasterDataTypePredicate.test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "gst") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("type", "application/json") + .test(asset)); + } + + @Test + void hmStacAssetPredicateMatchesJavaAssetAttributes() throws Exception { + HMStacAsset asset = bio12Asset(); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Climate Annual Precipitation (CHELSA bio12)") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "image/tiff") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326.0") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("valid", Boolean.toString(asset.isValid())) + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Wrong title") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "application/json") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "3857") + .test(asset)); + } + + @Test + void hmStacAssetAttributePredicateDistinguishesDifferentAssets() throws Exception { + HMStacAsset bio12 = bio12Asset(); + HMStacAsset gst = gstAsset(); + + Predicate bio12IdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1"); + + Predicate gstIdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1"); + + Predicate bio12BandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate gstBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "gst"); + + assertTrue(bio12IdPredicate.test(bio12)); + assertFalse(bio12IdPredicate.test(gst)); + + assertTrue(gstIdPredicate.test(gst)); + assertFalse(gstIdPredicate.test(bio12)); + + assertTrue(bio12BandPredicate.test(bio12)); + assertFalse(bio12BandPredicate.test(gst)); + + assertTrue(gstBandPredicate.test(gst)); + assertFalse(gstBandPredicate.test(bio12)); + } + + @Test + void predicatesReturnFalseForNullHMStacAsset() { + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "bio12") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + } + + @Test + void stacPathExpressionReturnsEmptyResultForMissingPath() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.missing.attribute").resolve(root); + + assertTrue(resolved.isEmpty()); + } + + @Test + void parseRejectsOldWildcardArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[*].name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void oldGreaterThanSeparatorDoesNotMatchBecauseDotIsNowTheSeparator() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets>eo:bands>name"); + + assertFalse(expression.matches(root, "bio12")); + } + + @Test + void parseRejectsEmptyPath() { + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(null)); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse("")); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(" ")); + } + + @Test + void parseRejectsNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[-1].name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void parseRejectsUnsupportedHMStacAssetAttribute() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute("unsupported", "value") + ); + + assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); + } +} \ No newline at end of file From 7ea3b47c293d58ce025c7690fe99f96a06c7c8f2 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 14:52:06 +0200 Subject: [PATCH 35/74] adds some try/catch --- .../integratedmodelling/klab/stac/STACEncoder.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index b483f39d7..ec93c0358 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -445,6 +445,7 @@ public void getEncodedData(IResource resource, Map urnParameters return start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd; }).collect(Collectors.toList()); if (items.size() == 0) { + manager.close(); throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); @@ -474,7 +475,13 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou }; } else if (resource.getParameters().get("jsonSelector", String.class) != null) { // based on the JSON Expression on JSONSelector and JSONValue - assetPredicate = getAssetPredicate(resource); + try { + assetPredicate = getAssetPredicate(resource); + } catch (Exception e) { + manager.close(); + throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); + } + } else { // NO JSONSelector and JSONValue found, NO assetID was passed as well scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); @@ -483,10 +490,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou try { source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, start, end); } catch (Exception e) { + manager.close(); throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); } encoder = new VectorEncoder(); ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + manager.close(); return; } // Once the support for customized predicate is added, we can apply for features as well From 7de1fe892a3edff317b76666e4e67d0a928a3e51 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 15:46:18 +0200 Subject: [PATCH 36/74] fixes --- .../klab/stac/STACEncoder.java | 204 +++++++++--------- 1 file changed, 100 insertions(+), 104 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index ec93c0358..8cf223eda 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -5,10 +5,6 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -92,7 +88,7 @@ import org.geotools.feature.simple.SimpleFeatureBuilder; import org.geotools.feature.simple.SimpleFeatureTypeBuilder; import org.geotools.data.DataUtilities; -import java.util.ArrayList; +import java.util.*; import kong.unirest.json.JSONObject; @@ -185,6 +181,9 @@ private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor } } + /* + Helper to Sort Items (of type HMStacItem) based on their timestamp + */ private void sortByDate(List items, IMonitor monitor) { if (items.stream().anyMatch(i -> i.getTimestamp() == null)) { throw new KlabIllegalStateException("STAC items are lacking a timestamp and could not be sorted by date."); @@ -224,6 +223,8 @@ public void getEncodedData(IResource resource, Map urnParameters IEnvelope envelope = space.getEnvelope(); List bbox = List.of(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + // Only for Backward Compatiability + // A new COG Adapter would be added if (resource.getParameters().get("cog") != null) { COGURL = resource.getParameters().get("cog", String.class); scope.getMonitor().info("Getting requested extent from the COG Asset from url" + COGURL); @@ -263,16 +264,24 @@ public void getEncodedData(IResource resource, Map urnParameters String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collectionData); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); Integer bandIndex = resource.getParameters().get("band", Integer.class); - boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); - // This is part of a WIP that will be removed in the future - boolean isIIASA = catalogUrl.contains("iiasa.blob"); String assetId = resource.getParameters().get("asset", String.class); - - Time time = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time) - .findFirst().orElseThrow(); + boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); + final boolean allowTransform = true; + Time ctxTime = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time).findFirst().orElseThrow(); Time resourceTime = (Time) Scale.create(resource.getGeometry()).getDimension(Type.TIME); + + Time effectiveTime = ctxTime; + if (resourceTime != null + && resourceTime.getStart() != null + && resourceTime.getEnd() != null + && resourceTime.getCoveredExtent() > 0) { - if (isIIASA) { + effectiveTime = validateTemporalDimension(ctxTime, resourceTime); + } + + + // This is part of a WIP that will be removed in the future + if (catalogUrl.contains("iiasa.blob")) { FeatureSource source; try { source = STACIIASAExtension.getFeatures(collectionData, bbox); @@ -283,27 +292,55 @@ public void getEncodedData(IResource resource, Map urnParameters ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } + + /* + Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue + */ + + Predicate assetPredicate = null; + if (assetId != null) { + assetPredicate = new Predicate() { + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + var bandsArray = (ArrayNode) bands; + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } + return false; + } + }; + } else if (resource.getParameters().get("jsonSelector", String.class) != null) { + // based on the JSON Expression on JSONSelector and JSONValue + try { + assetPredicate = getAssetPredicate(resource); + } catch (Exception e){ + throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); + } + + } // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); - Time time2 = time; //TODO make the time and query time different features = features.stream().filter(f -> { Geometry fGeometry = (Geometry) f.getDefaultGeometry(); return fGeometry.intersects(space.getShape().getJTSGeometry()); }).toList(); - features = features.stream().filter(f -> isFeatureInTimeRange(time2, f)).toList(); - if (features.isEmpty()) { - throw new KlabResourceNotFoundException("There are no items in this context for the collection " + collectionId); - } CoordinateReferenceSystem crs = features.get(0).getFeatureType().getCoordinateReferenceSystem(); if (crs == null) { crs = CrsUtilities.getCrsFromSrid(4326); // We go to the standard } - + var time2 = effectiveTime; // TODO merge with similar code from below IGrid grid = space.getGrid(); - RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); @@ -313,15 +350,17 @@ public void getEncodedData(IResource resource, Map urnParameters RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); // end //TODO - - List items = features.stream().map(f -> { - try { - return HMStacItem.fromSimpleFeature(f); - } catch (Exception e) { - scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); - } - return null; - }).filter(i -> i != null).toList(); + List items = features.stream() + .map(f -> { + try { + return HMStacItem.fromSimpleFeature(f); + } catch (Exception e) { + scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); + return null; + }}) + .filter(Objects::nonNull) + .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) + .toList(); GridCoverage2D coverage = null; @@ -388,14 +427,7 @@ public void getEncodedData(IResource resource, Map urnParameters Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - - - if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { - time = validateTemporalDimension(time, resourceTime); - } - ITimeInstant start = time.getStart(); - ITimeInstant end = time.getEnd(); + collection.setGeometryFilter(poly); //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) GridCoverage2D coverage = null; @@ -426,24 +458,12 @@ public void getEncodedData(IResource resource, Map urnParameters Client s3Client = buildS3Client(bucketRegion); collection.setS3Client(s3Client); } - + var time = effectiveTime; // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(item -> { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - return start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd; - }).collect(Collectors.toList()); + items = items.stream() + .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) + .collect(Collectors.toList()); + if (items.size() == 0) { manager.close(); throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); @@ -452,43 +472,13 @@ public void getEncodedData(IResource resource, Map urnParameters } // Allow transform ensures the process to finish, but I would not bet on the resulting data - Predicate assetPredicate = null; - final boolean allowTransform = true; - if (assetId != null) { - assetPredicate = new Predicate() { - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id - return asset.getId().equals(assetId); - } - return false; - } - }; - } else if (resource.getParameters().get("jsonSelector", String.class) != null) { - // based on the JSON Expression on JSONSelector and JSONValue - try { - assetPredicate = getAssetPredicate(resource); - } catch (Exception e) { - manager.close(); - throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); - } - - } else { + if (assetPredicate == null) { // NO JSONSelector and JSONValue found, NO assetID was passed as well scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); // Only get the features from STAC Collection, no need to interact with Rasters FeatureSource source; try { - source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, start, end); + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); } catch (Exception e) { manager.close(); throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); @@ -498,6 +488,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.close(); return; } + // Once the support for customized predicate is added, we can apply for features as well Set EPSGAtAssets = items.stream() @@ -550,26 +541,31 @@ private Predicate getAssetPredicate(IResource resource){ } } - private boolean isFeatureInTimeRange(Time time2, SimpleFeature f) { - Date datetime = (Date) f.getAttribute("datetime"); - if (datetime != null) { - if (isDateWithinRange(time2, datetime)) { - return true; - } - } - Date itemStart = (Date) f.getAttribute("start_datetime"); - if (itemStart == null) { - return false; - } - Date itemEnd = (Date) f.getAttribute("end_datetime"); - if (itemEnd == null) { - return itemStart.toInstant().getEpochSecond() <= time2.getStart().getMilliseconds(); - } - if (isDateWithinRange(time2, itemStart) || isDateWithinRange(time2, itemEnd)) { - return true; - } - return false; + /* + To check if an Item (of type HMStacItem) is within a time range + */ + private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + try { + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + return startMillis >= itemStart && endMillis <= itemEnd; + + } catch (Exception e) { + e.printStackTrace(); + return false; + } } private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, String collectionId) { From 4a40970aceb8a70fd1423f468f42fbbcb9a4b56e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 16:34:20 +0200 Subject: [PATCH 37/74] epsg at assets getting epsg at asset level if not, then at item level --- .../integratedmodelling/klab/stac/STACEncoder.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 8cf223eda..ea5cade2f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -490,12 +490,17 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } // Once the support for customized predicate is added, we can apply for features as well + + var pred = assetPredicate; Set EPSGAtAssets = items.stream() - .flatMap(item -> item.getAssets().stream()) - .filter(assetPredicate) - .map(HMStacAsset::getEpsg) + .flatMap(item -> item.getAssets().stream() + .filter(pred) + .map(asset -> asset.getEpsg() != null + ? asset.getEpsg() + : item.getEpsg())) .collect(Collectors.toUnmodifiableSet()); + if (EPSGAtAssets.size() > 1) { scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." From 1c853c9e95dfef2c24aa9dc5fbd01a61ac9357f0 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 16:41:49 +0200 Subject: [PATCH 38/74] finds first, and update the time check --- .../integratedmodelling/klab/stac/STACEncoder.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index ea5cade2f..9f01de4c1 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -496,9 +496,11 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou items.stream() .flatMap(item -> item.getAssets().stream() .filter(pred) + .findFirst() .map(asset -> asset.getEpsg() != null ? asset.getEpsg() - : item.getEpsg())) + : item.getEpsg()) + .stream()) .collect(Collectors.toUnmodifiableSet()); if (EPSGAtAssets.size() > 1) { @@ -552,7 +554,15 @@ To check if an Item (of type HMStacItem) is within a time range */ private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String startTimestamp = item.getStartTimestamp(); + String endTimestamp = item.getEndTimestamp(); + + if (startTimestamp == null || endTimestamp == null) { + return true; // Assume the time part is ok + } + try { + long itemStart = LocalDateTime .parse(item.getStartTimestamp(), formatter) .atZone(ZoneOffset.UTC) From 44b09a726df187cdb6d22374acdcb9e7b2948f15 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 16:52:22 +0200 Subject: [PATCH 39/74] updates string conv --- .../java/org/integratedmodelling/klab/stac/STACImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 100f999f7..055b12cea 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -73,7 +73,7 @@ private void importCollection(List ret, IParameters parameters, return; } - String assetId = (String) parameters.get("asset"); + String assetId = parameters.get("asset", String.class); JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); /* From 4d58c7aecb9d6bfbb80c577e848e6a45b6151f17 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 14:38:16 +0200 Subject: [PATCH 40/74] avoid making search twice --- .../klab/stac/STACEncoder.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9f01de4c1..33c43999b 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -432,6 +432,25 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou GridCoverage2D coverage = null; try { + + // Allow transform ensures the process to finish, but I would not bet on the resulting data + if (assetPredicate == null) { + // NO JSONSelector and JSONValue found, NO assetID was passed as well + scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; + try { + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); + } catch (Exception e) { + manager.close(); + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + } + encoder = new VectorEncoder(); + ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + manager.close(); + return; + } + List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -470,24 +489,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } - - // Allow transform ensures the process to finish, but I would not bet on the resulting data - if (assetPredicate == null) { - // NO JSONSelector and JSONValue found, NO assetID was passed as well - scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); - // Only get the features from STAC Collection, no need to interact with Rasters - FeatureSource source; - try { - source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); - } catch (Exception e) { - manager.close(); - throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); - } - encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); - manager.close(); - return; - } // Once the support for customized predicate is added, we can apply for features as well From e3ba3dcc6570ebee3f4468bc64ab0ecae0b15e14 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 14:41:05 +0200 Subject: [PATCH 41/74] fix grammar --- adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index d17441ac1..899adae92 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -11,7 +11,7 @@ "[REQUIRED] The URL pointing to the STAC collection file that contains the resource dataset." optional text 'asset' - "The asset that is going to be retrieved from the items. Should be left it blank when the information is stored in the feature." + "The asset that is going to be retrieved from the items. Should be left blank when the information is stored in the feature." optional number 'band' "Relevant only for raster resources. From 3a0dfa98b34ca52320d191d9b7eed7b9a953529b Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 14:55:56 +0200 Subject: [PATCH 42/74] remove dual cli logs --- .../org/integratedmodelling/klab/stac/STACEncoder.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 33c43999b..872842377 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -293,6 +293,7 @@ public void getEncodedData(IResource resource, Map urnParameters return; } + /* Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue */ @@ -509,13 +510,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou + "." + "The transformation process could affect the data."); } - - if (items.size() == 0) { - manager.close(); - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); - } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); - } HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); From e2eea1751b34221374ffabd0930f7ddf1061421a Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 15:18:01 +0200 Subject: [PATCH 43/74] fail fast on resource access problems for collection --- .../klab/stac/STACEncoder.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 872842377..836d09373 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -305,16 +305,16 @@ public void getEncodedData(IResource resource, Map urnParameters public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates var bands = asset.getAssetNode().get("eo:bands"); if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id - return asset.getId().equals(assetId); - } + var bandsArray = (ArrayNode) bands; + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } return false; } }; @@ -419,6 +419,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou if (collection == null) { scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + throw new KlabResourceAccessException("Collection" + resource.getParameters().get("collection", String.class) +" cannot be accessed."); // Fail fast } IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation From 87717976c5b363cd3f1eb573bc112a3fe80d44d8 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 15:22:40 +0200 Subject: [PATCH 44/74] put under a single try/catch --- .../klab/stac/STACEncoder.java | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 836d09373..920b75837 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -407,33 +407,23 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou try { manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e) { - try { - manager.close(); - } catch (Exception e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl + ". Reason :" + e.getMessage()); - } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - throw new KlabResourceAccessException("Collection" + resource.getParameters().get("collection", String.class) +" cannot be accessed."); // Fail fast - } - IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation - ? ((Observation) scope.getTargetArtifact()).getObservable() - : null; - HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); - - Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); - Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) - - GridCoverage2D coverage = null; - try { + if (collection == null) { + scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + manager.close(); + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail fast + } + + IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation + ? ((Observation) scope.getTargetArtifact()).getObservable() + : null; + HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); + Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); + collection.setGeometryFilter(poly); + //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) + + GridCoverage2D coverage = null; // Allow transform ensures the process to finish, but I would not bet on the resulting data if (assetPredicate == null) { From 7b922c1d9dc58c812db9836a28ae91eab9fc483e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 15:48:00 +0200 Subject: [PATCH 45/74] put under a single try/catch --- .../java/org/integratedmodelling/klab/stac/STACEncoder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 920b75837..bd4e5f7e1 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -508,12 +508,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } manager.close(); + encoder = new RasterEncoder(); + ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } - encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } private Predicate getAssetPredicate(IResource resource){ From 57be8c606ba5e1d348d1e63fd9985d3732cc537c Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 12 May 2026 16:47:06 +0200 Subject: [PATCH 46/74] Reorganize code for asset search in import --- .../klab/stac/STACCollectionParser.java | 203 ++++++++---------- .../klab/stac/STACValidator.java | 106 ++++----- 2 files changed, 144 insertions(+), 165 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index a750c5b46..3fbacccc5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -3,7 +3,6 @@ import java.time.Instant; import java.util.*; import java.util.function.Predicate; -import java.util.stream.IntStream; import org.integratedmodelling.klab.api.data.IGeometry; import org.integratedmodelling.klab.common.Geometry; @@ -12,8 +11,6 @@ import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; -import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema.Items; - import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; @@ -37,7 +34,7 @@ public static String readCollectionId(JSONObject collection) { */ public static IGeometry readGeometry(JSONObject collection) { GeometryBuilder gBuilder = Geometry.builder(); - + JSONObject extent = collection.getJSONObject("extent"); List bbox = extent.getJSONObject("spatial").getJSONArray("bbox").getJSONArray(0).toList(); gBuilder.space().boundingBox(Double.valueOf(bbox.get(0).toString()), Double.valueOf(bbox.get(1).toString()), @@ -58,90 +55,46 @@ public static IGeometry readGeometry(JSONObject collection) { return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("grid"); } - /** - * Reads the asset of a STAC collection and returns them as a JSON based on Asset Key. - * @param collection as a JSON - * @return The asset matching the assetId as a JSONObject - * @throws KlabResourceAccessException - */ - public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) throws KlabResourceAccessException { - String collectionId = collection.getString("id"); - String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); - JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - - Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") - ? STACUtils.getLinkTo(catalogData, "search") - : STACUtils.getLinkTo(collection, "search"); - - // Static catalogs should have their assets on the Collection - if (searchEndpoint.isEmpty()) { - // Check the assets - if (collection.has("assets")) { - return collection.getJSONObject("assets"); - } - // Try to get the assets from a link that has type `item` - Optional itemHref = STACUtils.getLinkTo(collection, "item"); - if (itemHref.isEmpty()) { - throw new KlabResourceNotFoundException("Cannot find items at STAC collection \"" + collectionUrl + "\""); - } - String itemUrl = itemHref.get().startsWith(".") - ? collectionUrl.replace("collection.json", "") + itemHref.get().replace("./", "") - : itemHref.get(); - // TODO get assets from the item - JSONObject itemData = STACUtils.requestMetadata(itemUrl, "feature"); - if (itemData.has("assets")) { - return itemData.getJSONObject("assets"); - } - throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); - } - - JSONObject searchPayload = new JSONObject() - .put("limit", 100) - .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) - .put("collections", new JSONArray().put(collectionId)); - - HttpResponse response = Unirest - .post(searchEndpoint.get()) - .header("Content-Type", "application/json") - .body(searchPayload) - .asJson(); - - if (!response.isSuccess()) { - throw new KlabResourceAccessException("Unable to import collection, Search failed"); //TODO set message + private static JSONObject findAsset(JSONObject assets, String assetId, Predicate predicate) { + if (assets == null) { + return null; } - JSONObject searchResponse = response.getBody().getObject(); - if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException("No features were found in the collection to be imported"); // TODO set message there is no feature + if (assetId != null) { + JSONObject asset = assets.optJSONObject(assetId); + if (asset == null) { + return null; + } + JSONObject result = new JSONObject(); + result.put(assetId, asset); + return result; } - - JSONArray features = searchResponse.getJSONArray("features"); - - for (int i = 0; i < features.length(); i++) { - JSONObject feature = features.getJSONObject(i); - - JSONObject assetInfo = feature - .getJSONObject("assets") - .optJSONObject(assetId); - if (assetInfo != null) { - return assetInfo; - } + if (predicate != null) { + return assets.keySet().stream().map(key -> { + JSONObject asset = assets.optJSONObject(key); + if (asset == null || !predicate.test(asset)) { + return null; + } + JSONObject result = new JSONObject(); + result.put(key, asset); + return result; + }).filter(Objects::nonNull).findFirst().orElse(null); } - throw new KlabResourceAccessException("No Asset with ID: " + assetId + " was found in the collection"); + return null; } - - /* - Retrieves the asset matching the Predicate - */ - public static JSONObject readAssetInformationFromCollectionWithPredicate(String collectionUrl, JSONObject collection, Predicate p) throws KlabResourceAccessException { + private static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId, + Predicate predicate) throws KlabResourceAccessException { + if ((assetId == null) == (predicate == null)) { + throw new KlabResourceAccessException("Exactly one of assetId or predicate must be provided"); + } String collectionId = collection.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") + Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") ? STACUtils.getLinkTo(catalogData, "search") : STACUtils.getLinkTo(collection, "search"); @@ -149,7 +102,10 @@ public static JSONObject readAssetInformationFromCollectionWithPredicate(String if (searchEndpoint.isEmpty()) { // Check the assets if (collection.has("assets")) { - return collection.getJSONObject("assets"); + JSONObject assetInfo = findAsset(collection.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; + } } // Try to get the assets from a link that has type `item` Optional itemHref = STACUtils.getLinkTo(collection, "item"); @@ -159,55 +115,74 @@ public static JSONObject readAssetInformationFromCollectionWithPredicate(String String itemUrl = itemHref.get().startsWith(".") ? collectionUrl.replace("collection.json", "") + itemHref.get().replace("./", "") : itemHref.get(); - // TODO get assets from the item JSONObject itemData = STACUtils.requestMetadata(itemUrl, "feature"); - if (itemData.has("assets")) { - return itemData.getJSONObject("assets"); + JSONObject assetInfo = findAsset(itemData.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; } throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); - } - - JSONObject searchPayload = new JSONObject() - .put("limit", 100) - .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) - .put("collections", new JSONArray().put(collectionId)); - - HttpResponse response = Unirest - .post(searchEndpoint.get()) - .header("Content-Type", "application/json") - .body(searchPayload) - .asJson(); + } + + JSONObject searchPayload = new JSONObject().put("limit", 100) + .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) + .put("collections", new JSONArray().put(collectionId)); + + HttpResponse response = Unirest.post(searchEndpoint.get()).header("Content-Type", "application/json") + .body(searchPayload).asJson(); if (!response.isSuccess()) { - throw new KlabResourceAccessException("Unable to import collection, Search failed"); //TODO set message + throw new KlabResourceAccessException("Unable to import collection, Search failed"); } JSONObject searchResponse = response.getBody().getObject(); - if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException("No features were found in the collection to be imported"); // TODO set message there is no feature + JSONArray features = searchResponse.optJSONArray("features"); + if (features == null || features.length() == 0) { + throw new KlabResourceAccessException("No features were found in the collection to be imported"); } - - JSONArray features = searchResponse.getJSONArray("features"); - JSONObject matchingFeature = IntStream.range(0, features.length()) - .mapToObj(i -> features.getJSONObject(i)) - .filter(p) - .findFirst() - .orElse(null); + for (int i = 0; i < features.length(); i++) { + JSONObject feature = features.optJSONObject(i); + if (feature == null) { + continue; + } - if (matchingFeature == null) { - throw new KlabResourceAccessException("No Feature found containing an asset matching the predicate based on JSONSelector and JSONValidator was found in the collection"); + JSONObject assetInfo = findAsset(feature.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; + } } - JSONObject assets = matchingFeature.getJSONObject("assets"); - JSONObject assetInfo = assets.keySet().stream() - .map(assets::getJSONObject) - .filter(asset -> p.test(asset)) - .findFirst() - .orElse(null); - - if (assetInfo == null) { - throw new KlabResourceAccessException("No Asset was found in the matching Fearure"); // shouldn't happen + if (assetId != null) { + throw new KlabResourceAccessException("No asset with ID \"" + assetId + "\" was found in the collection"); } - return assetInfo; + + throw new KlabResourceAccessException("No asset matching the predicate was found in the collection"); + } + + /** + * Reads an asset from a STAC collection by asset key. + * + * @param collectionUrl URL of the STAC collection + * @param collection collection metadata as JSON + * @param assetId asset key to find inside each feature's assets object + * @return a JSONObject containing one entry: assetId -> asset JSON object + * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found + */ + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) + throws KlabResourceAccessException { + return readAssetInformationFromCollection(collectionUrl, collection, assetId, null); + } + + /** + * Reads the first asset in a STAC collection whose asset JSON object matches the predicate. + * + * @param collectionUrl URL of the STAC collection + * @param collection collection metadata as JSON + * @param predicate predicate evaluated against each asset JSON object + * @return a JSONObject containing one entry: assetId -> asset JSON object + * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found + */ + public static JSONObject readAssetInformationFromCollectionWithPredicate(String collectionUrl, JSONObject collection, + Predicate predicate) throws KlabResourceAccessException { + return readAssetInformationFromCollection(collectionUrl, collection, null, predicate); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index b3d10d2f0..eed2dc509 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -45,61 +45,62 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni String collectionUrl = userData.get("collection", String.class); String collectionId = userData.get("collectionId", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); - if (collectionId == null) { + if (collectionId == null) { collectionId = collectionData.getString("id"); userData.put("collectionId", collectionId); } IGeometry geometry = STACCollectionParser.readGeometry(collectionData); - Builder builder = new ResourceBuilder(urn) - .withParameters(userData) - .withGeometry(geometry) - .withType(Type.OBJECT); + Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); - // The default URL of the resource is the collection endpoint. May be overwritten. + // The default URL of the resource is the collection endpoint. May be overwritten. builder.withMetadata(IMetadata.DC_URL, collectionUrl); - JSONObject asset = null; - String assetId = null; + JSONObject assetNode; if (userData.contains("asset")) { - assetId = userData.get("asset", String.class); - asset = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - //JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - } else if (userData.contains("jsonSelector")) { - if (userData.contains("jsonValue")) { - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromKongJsonObject( - userData.get("jsonSelector", String.class), - userData.get("jsonValue", String.class) - ); - asset = STACCollectionParser.readAssetInformationFromCollectionWithPredicate(collectionUrl, collectionData, predicate); - assetId = (String) asset.get("id"); // Since the Asset Id is not directly defined - } - } else { - throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); - } - - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); - } - builder.addCodeList(codelist); - } - if (type != null) { - builder.withType(type); - } - generateCodeList(builder, assetId, asset); - - if (userData.contains("cog")) { - if (userData.get("cog") != null) { - builder.withType(Type.NUMBER); - } + String requestedAssetId = userData.get("asset", String.class); + + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); + + } else if (userData.contains("jsonSelector")) { + if (!userData.contains("jsonValue")) { + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); + } + + Predicate predicate = STACPathExpression.STACAssetPredicate + .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); + + assetNode = STACCollectionParser.readAssetInformationFromCollectionWithPredicate(collectionUrl, collectionData, + predicate); + + } else { + throw new KlabIllegalArgumentException("Either asset or jsonSelector/jsonValue must be provided"); + } + + String assetId = assetNode.keys().next(); + JSONObject asset = assetNode.getJSONObject(assetId); + + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); + } + generateCodeList(builder, assetId, asset); + + if (userData.contains("cog")) { + if (userData.get("cog") != null) { + builder.withType(Type.NUMBER); + } + } readMetadata(collectionData, builder); return builder; @@ -107,7 +108,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni private void generateCodeList(Builder builder, String assetId, JSONObject asset) { Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. Map vals = STACAssetParser.getFileValues(asset); if (!vals.isEmpty()) { CodelistReference codelist = populateCodelist(assetId, vals); @@ -125,7 +127,7 @@ private Type readRasterDataType(JSONObject asset) { if (!asset.has("raster:bands")) { return null; } - + if (asset.getJSONArray("raster:bands").isEmpty() || !asset.getJSONArray("raster:bands").getJSONObject(0).has("data_type")) { // We assume that most rasters are numeric. When in doubt, we set the default to Number @@ -133,7 +135,8 @@ private Type readRasterDataType(JSONObject asset) { } String type = asset.getJSONArray("raster:bands").getJSONObject(0).getString("data_type"); // https://github.com/stac-extensions/raster?tab=readme-ov-file#data-types - final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", "float16", "float32", "float64"); + final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", + "float16", "float32", "float64"); if (NUMERIC_DATA_TYPES.contains(type)) { return Type.NUMBER; } @@ -155,8 +158,8 @@ private CodelistReference populateCodelist(String assetId, Map v MappingReference direct = new MappingReference(); MappingReference inverse = new MappingReference(); vals.entrySet().forEach(code -> { - direct.getMappings().add(new Pair<>(code.getKey(), (String)code.getValue())); - codelist.getCodeDescriptions().put(code.getKey(), (String)code.getValue()); + direct.getMappings().add(new Pair<>(code.getKey(), (String) code.getValue())); + codelist.getCodeDescriptions().put(code.getKey(), (String) code.getValue()); }); Type type = STACUtils.inferValueType(vals.entrySet().stream().findFirst().get().getKey()); codelist.setType(type); @@ -166,7 +169,8 @@ private CodelistReference populateCodelist(String assetId, Map v } private void readMetadata(final JSONObject json, Builder builder) { - // We could check the doi only if the Scientific Notation extension is provided, but we can try anyway + // We could check the doi only if the Scientific Notation extension is provided, but we can + // try anyway String doi = STACUtils.readDOI(json); if (doi != null && !doi.isBlank()) { builder.withMetadata(IMetadata.DC_URL, doi); @@ -233,7 +237,7 @@ public Collection getAllFilesForResource(File file) { } @Override - public Map describeResource(IResource resource) { + public Map< ? extends String, ? extends Object> describeResource(IResource resource) { // TODO Auto-generated method stub return null; } From 4df97a89e88a515121731cb40e45a48ec48ac99f Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 12 May 2026 16:49:00 +0200 Subject: [PATCH 47/74] Align code with changes --- .../java/org/integratedmodelling/klab/stac/STACImporter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 055b12cea..c8a689020 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -74,8 +74,9 @@ private void importCollection(List ret, IParameters parameters, } String assetId = parameters.get("asset", String.class); - JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + JSONObject assetData = assetNode.getJSONObject(assetId); /* * If the particular asset Id wasn't found, then * still proceed to create the resource since it can happen From 2abae11ff08126dd60db1ea947818c1e1a4cf395 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Wed, 13 May 2026 09:24:25 +0200 Subject: [PATCH 48/74] Rename function to be coherent --- .../org/integratedmodelling/klab/stac/STACCollectionParser.java | 2 +- .../java/org/integratedmodelling/klab/stac/STACValidator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index 3fbacccc5..77f1482a5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -181,7 +181,7 @@ public static JSONObject readAssetInformationFromCollection(String collectionUrl * @return a JSONObject containing one entry: assetId -> asset JSON object * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found */ - public static JSONObject readAssetInformationFromCollectionWithPredicate(String collectionUrl, JSONObject collection, + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, Predicate predicate) throws KlabResourceAccessException { return readAssetInformationFromCollection(collectionUrl, collection, null, predicate); } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index eed2dc509..0384fc9c2 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -71,7 +71,7 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni Predicate predicate = STACPathExpression.STACAssetPredicate .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); - assetNode = STACCollectionParser.readAssetInformationFromCollectionWithPredicate(collectionUrl, collectionData, + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, predicate); } else { From 8633543fa0d7dc648d04f13a00874e325ad04108 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Wed, 13 May 2026 10:50:45 +0200 Subject: [PATCH 49/74] Clean and format code --- .../klab/stac/STACCollectionParser.java | 2 +- .../klab/stac/STACEncoder.java | 403 +++++++++--------- .../klab/stac/STACImporter.java | 51 ++- .../klab/stac/STACPathExpression.java | 9 +- .../klab/stac/STACUtils.java | 22 +- .../klab/stac/STACValidator.java | 6 +- 6 files changed, 235 insertions(+), 258 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index 77f1482a5..6dd160b9f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -140,7 +140,7 @@ private static JSONObject readAssetInformationFromCollection(String collectionUr throw new KlabResourceAccessException("No features were found in the collection to be imported"); } - for (int i = 0; i < features.length(); i++) { + for(int i = 0; i < features.length(); i++) { JSONObject feature = features.optJSONObject(i); if (feature == null) { continue; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index bd4e5f7e1..c64465b84 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -5,6 +5,11 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -15,8 +20,8 @@ import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.processing.Operations; import org.geotools.geometry.jts.ReferencedEnvelope; -import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; @@ -53,28 +58,17 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalStateException; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; -import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.ogc.STACAdapter; -import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; +import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; -import org.integratedmodelling.klab.stac.STACPathExpression.AssetAttribute; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; -import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.stac.extensions.STACFeatureExtension; +import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; -import org.geotools.coverage.processing.Operations; -import org.geotools.data.geojson.GeoJSONReader; -import org.geotools.data.memory.MemoryDataStore; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.Polygon; -import org.geotools.api.feature.simple.SimpleFeature; -import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.data.simple.SimpleFeatureCollection; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; @@ -82,19 +76,11 @@ import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; -import java.time.*; -import java.time.format.DateTimeFormatter; -import org.locationtech.jts.operation.union.UnaryUnionOp; -import org.geotools.feature.simple.SimpleFeatureBuilder; -import org.geotools.feature.simple.SimpleFeatureTypeBuilder; -import org.geotools.data.DataUtilities; -import java.util.*; - import kong.unirest.json.JSONObject; public class STACEncoder implements IResourceEncoder { - - /** + + /** * The raster or vector encoder that does the actual work after we get our coverage from the service. */ IResourceEncoder encoder; @@ -131,7 +117,8 @@ public ICodelist categorize(IResource resource, String attribute, IMonitor monit private Time refitTime(Time contextTime, Time resourceTime) { if (resourceTime.getCoveredExtent() < contextTime.getCoveredExtent()) { - throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted."); + throw new KlabContextualizationException( + "Current observation is outside the bounds of the STAC resource and cannot be reffitted."); } if (contextTime.getStart().isBefore(resourceTime.getStart())) { ITimeInstant newEnd = TimeInstant.create(resourceTime.getStart().getMilliseconds() + contextTime.getLength()); @@ -141,7 +128,8 @@ private Time refitTime(Time contextTime, Time resourceTime) { ITimeInstant newStart = TimeInstant.create(resourceTime.getEnd().getMilliseconds() - contextTime.getLength()); return Time.create(newStart.getMilliseconds(), resourceTime.getEnd().getMilliseconds()); } - throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted."); + throw new KlabContextualizationException( + "Current observation is outside the bounds of the STAC resource and cannot be reffitted."); } /** @@ -171,7 +159,7 @@ private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor case NUMBER: if (Observables.INSTANCE.isExtensive(targetSemantics)) { monitor.debug("Using sum as merge mode"); - return HMRaster.MergeMode.SUM; + return HMRaster.MergeMode.SUM; } monitor.debug("Using substitute as merge mode"); return HMRaster.MergeMode.SUBSTITUTE; @@ -189,8 +177,8 @@ private void sortByDate(List items, IMonitor monitor) { throw new KlabIllegalStateException("STAC items are lacking a timestamp and could not be sorted by date."); } items.sort((i1, i2) -> i1.getTimestamp().compareTo(i2.getTimestamp())); - monitor.debug( - "Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + items.get(items.size() - 1).getTimestamp() + "]"); + monitor.debug("Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + + items.get(items.size() - 1).getTimestamp() + "]"); } private Client buildS3Client(String bucketRegion) throws IOException { @@ -201,15 +189,13 @@ private Client buildS3Client(String bucketRegion) throws IOException { } catch (Exception e) { throw new KlabIOException("Error defining S3 credenetials. " + e.getMessage()); } - return Client.s3() - .regionFromEnvironment() // TODO get region from other sources if needed - .credentials(credentials) - .build(); + return Client.s3().regionFromEnvironment() // TODO get region from other sources if needed + .credentials(credentials).build(); } private boolean isDateWithinRange(Time rangeTime, Date date) { Date start = new Date(rangeTime.getStart().getMilliseconds()); - Date end = new Date(rangeTime.getEnd().getMilliseconds()); + Date end = new Date(rangeTime.getEnd().getMilliseconds()); return date.after(start) && date.before(end); } @@ -218,8 +204,7 @@ public void getEncodedData(IResource resource, Map urnParameters IContextualizationScope scope) { String COGURL = null; - Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space) - .findFirst().orElseThrow(); + Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space).findFirst().orElseThrow(); IEnvelope envelope = space.getEnvelope(); List bbox = List.of(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); @@ -229,35 +214,32 @@ public void getEncodedData(IResource resource, Map urnParameters COGURL = resource.getParameters().get("cog", String.class); scope.getMonitor().info("Getting requested extent from the COG Asset from url" + COGURL); GridCoverage2D coverage = COGAssetExtension.getCOGWindowCoverage(bbox, COGURL); - - String rcrs = geometry.getDimension(IGeometry.Dimension.Type.SPACE).getParameters().get( - org.integratedmodelling.klab.common.Geometry.PARAMETER_SPACE_PROJECTION, - String.class); - - Projection crs = Projection.create(rcrs); - org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(bbox.get(0), - bbox.get(1), bbox.get(2), bbox.get(3)); - - HMRaster raster = HMRaster.fromGridCoverage(coverage); - HMRaster outRaster = new HMRasterWritableBuilder().setRegion(RegionMap.fromEnvelopeAndGrid(requestedExtend, - (int) space.shape()[0], - (int) space.shape()[1])).setCrs(crs.getCoordinateReferenceSystem()) - .setNoValue(raster.getNovalue()) - .build(); - - GridCoverage2D adjCoverage = null; - try { - outRaster.mapRaster(null, raster, null); - adjCoverage = outRaster.buildCoverage(); - } catch (Exception e) { - throw new KlabResourceAccessException("Cannot build COG Output " + e.getMessage()); - } - - encoder = new RasterEncoder(); + + String rcrs = geometry.getDimension(IGeometry.Dimension.Type.SPACE).getParameters() + .get(org.integratedmodelling.klab.common.Geometry.PARAMETER_SPACE_PROJECTION, String.class); + + Projection crs = Projection.create(rcrs); + org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(bbox.get(0), bbox.get(1), + bbox.get(2), bbox.get(3)); + + HMRaster raster = HMRaster.fromGridCoverage(coverage); + HMRaster outRaster = new HMRasterWritableBuilder() + .setRegion(RegionMap.fromEnvelopeAndGrid(requestedExtend, (int) space.shape()[0], (int) space.shape()[1])) + .setCrs(crs.getCoordinateReferenceSystem()).setNoValue(raster.getNovalue()).build(); + + GridCoverage2D adjCoverage = null; + try { + outRaster.mapRaster(null, raster, null); + adjCoverage = outRaster.buildCoverage(); + } catch (Exception e) { + throw new KlabResourceAccessException("Cannot build COG Output " + e.getMessage()); + } + + encoder = new RasterEncoder(); ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, adjCoverage, geometry, builder, scope); return; } - + String collectionUrl = resource.getParameters().get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = collectionData.getString("id"); @@ -269,52 +251,51 @@ public void getEncodedData(IResource resource, Map urnParameters final boolean allowTransform = true; Time ctxTime = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time).findFirst().orElseThrow(); Time resourceTime = (Time) Scale.create(resource.getGeometry()).getDimension(Type.TIME); - + Time effectiveTime = ctxTime; - if (resourceTime != null - && resourceTime.getStart() != null - && resourceTime.getEnd() != null + if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { effectiveTime = validateTemporalDimension(ctxTime, resourceTime); } - // This is part of a WIP that will be removed in the future if (catalogUrl.contains("iiasa.blob")) { FeatureSource source; try { source = STACIIASAExtension.getFeatures(collectionData, bbox); } catch (IOException e) { - throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage()); + throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage()); } encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } - - + /* Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue */ Predicate assetPredicate = null; if (assetId != null) { - assetPredicate = new Predicate() { - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates + assetPredicate = new Predicate(){ + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would + // be there, adding support for customised + // predicates var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id - return asset.getId().equals(assetId); - } + if (bands != null && bands.isArray()) { + var bandsArray = (ArrayNode) bands; + for(var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this + // case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } return false; } }; @@ -322,12 +303,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // based on the JSON Expression on JSONSelector and JSONValue try { assetPredicate = getAssetPredicate(resource); - } catch (Exception e){ + } catch (Exception e) { throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); } - + } - + // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); @@ -343,64 +324,66 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // TODO merge with similar code from below IGrid grid = space.getGrid(); RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); // end //TODO - List items = features.stream() - .map(f -> { - try { - return HMStacItem.fromSimpleFeature(f); - } catch (Exception e) { - scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); - return null; - }}) - .filter(Objects::nonNull) - .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) - .toList(); + List items = features.stream().map(f -> { + try { + return HMStacItem.fromSimpleFeature(f); + } catch (Exception e) { + scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); + return null; + } + }).filter(Objects::nonNull) + .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) + .toList(); GridCoverage2D coverage = null; try { - // TODO see if we can access to the same readRasterBandOnRegion without using a collection + // TODO see if we can access to the same readRasterBandOnRegion without using a + // collection LogProgressMonitor lpm = new LogProgressMonitor(); try (HMStacManager manager = new HMStacManager(catalogUrl, lpm)) { - HMStacCollection collection = null; - try { - manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e1) { - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); - } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - } - Predicate predicate; - try { - predicate = getAssetPredicate(resource); - } catch (KlabIllegalArgumentException e) { - manager.close(); - throw e; - } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); - } + HMStacCollection collection = null; + try { + manager.open(); + collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + } catch (Exception e1) { + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); + } + + if (collection == null) { + scope.getMonitor().error( + "Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + } + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, + MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + } if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } } catch (Exception e) { - throw new KlabResourceAccessException("Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); + throw new KlabResourceAccessException( + "Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); } encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); return; } - + LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -408,41 +391,46 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - manager.close(); - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail fast - } - - IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation - ? ((Observation) scope.getTargetArtifact()).getObservable() - : null; - HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); - Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); - Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) - - GridCoverage2D coverage = null; - - // Allow transform ensures the process to finish, but I would not bet on the resulting data + if (collection == null) { + scope.getMonitor() + .error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + manager.close(); + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail + // fast + } + + IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation + ? ((Observation) scope.getTargetArtifact()).getObservable() + : null; + HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); + Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); + collection.setGeometryFilter(poly); + // collection.setTimestampFilter(new Date(start.getMilliseconds()), new + // Date(end.getMilliseconds())); --> Filter later :) + + GridCoverage2D coverage = null; + + // Allow transform ensures the process to finish, but I would not bet on the resulting + // data if (assetPredicate == null) { // NO JSONSelector and JSONValue found, NO assetID was passed as well scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); - // Only get the features from STAC Collection, no need to interact with Rasters - FeatureSource source; + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; try { - source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), + effectiveTime.getEnd()); } catch (Exception e) { manager.close(); - throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); } encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); manager.close(); return; } - + List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -456,8 +444,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou IGrid grid = space.getGrid(); RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); @@ -470,53 +457,49 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou collection.setS3Client(s3Client); } var time = effectiveTime; - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC + // Filter here based on time, since in some STAC collections they don't yet support + // temporal filtering :( like ECDC items = items.stream() - .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) - .collect(Collectors.toList()); + .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) + .collect(Collectors.toList()); if (items.size() == 0) { manager.close(); - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); + throw new KlabIllegalStateException( + "No STAC items found covering the entire time duration of the context requested"); } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } - + // Once the support for customized predicate is added, we can apply for features as well - + var pred = assetPredicate; - Set EPSGAtAssets = - items.stream() - .flatMap(item -> item.getAssets().stream() - .filter(pred) - .findFirst() - .map(asset -> asset.getEpsg() != null - ? asset.getEpsg() - : item.getEpsg()) - .stream()) - .collect(Collectors.toUnmodifiableSet()); - + Set EPSGAtAssets = items.stream() + .flatMap(item -> item.getAssets().stream().filter(pred).findFirst() + .map(asset -> asset.getEpsg() != null ? asset.getEpsg() : item.getEpsg()).stream()) + .collect(Collectors.toUnmodifiableSet()); + if (EPSGAtAssets.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() - + "." - + "The transformation process could affect the data."); + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." + + "The transformation process could affect the data."); } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, + MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } manager.close(); encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { - e.printStackTrace(); + e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } } - - private Predicate getAssetPredicate(IResource resource){ + + private Predicate getAssetPredicate(IResource resource) { String assetId = resource.getParameters().get("asset", String.class); if (assetId != null) { return STACPathExpression.STACAssetPredicate.fromHMStacAssetId(assetId); @@ -534,44 +517,40 @@ private Predicate getAssetPredicate(IResource resource){ } } - /* To check if an Item (of type HMStacItem) is within a time range */ private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - String startTimestamp = item.getStartTimestamp(); - String endTimestamp = item.getEndTimestamp(); - - if (startTimestamp == null || endTimestamp == null) { - return true; // Assume the time part is ok - } - - try { - - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - return startMillis >= itemStart && endMillis <= itemEnd; - - } catch (Exception e) { - e.printStackTrace(); - return false; - } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String startTimestamp = item.getStartTimestamp(); + String endTimestamp = item.getEndTimestamp(); + + if (startTimestamp == null || endTimestamp == null) { + return true; // Assume the time part is ok + } + + try { + + long itemStart = LocalDateTime.parse(item.getStartTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + return startMillis >= itemStart && endMillis <= itemEnd; + + } catch (Exception e) { + e.printStackTrace(); + return false; + } } - private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, String collectionId) { - List links = collectionData.getJSONArray("links").toList().stream().filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("item")).toList(); - List urlOfLinks = links.stream().map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); + private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, + String collectionId) { + List links = collectionData.getJSONArray("links").toList().stream() + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("item")).toList(); + List urlOfLinks = links.stream() + .map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); return urlOfLinks.stream().map(i -> { try { return STACUtils.getItemAsFeature(i); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index c8a689020..dbaace6c9 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -9,10 +9,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import org.integratedmodelling.kim.api.IParameters; -import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Resources; import org.integratedmodelling.klab.api.data.ILocator; import org.integratedmodelling.klab.api.data.IResource; @@ -43,9 +41,9 @@ public boolean acceptsMultiple() { // TODO Auto-generated method stub return false; } - + private boolean hasAssetSelector(IParameters parameters) { - return (parameters.contains("asset")||(parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); + return (parameters.contains("asset") || (parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); } private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) @@ -55,12 +53,6 @@ private void importCollection(List ret, IParameters parameters, String collectionId = STACCollectionParser.readCollectionId(collectionData); parameters.put("collectionId", collectionId); - String regex = null; - if (parameters.contains("regex")) { - regex = parameters.get(Resources.REGEX_ENTRY, String.class); - parameters.remove(Resources.REGEX_ENTRY); - } - boolean isBulkImport = parameters.contains("bulkImport"); parameters.remove("bulkImport"); if (!hasAssetSelector(parameters) && !isBulkImport) { @@ -72,10 +64,10 @@ private void importCollection(List ret, IParameters parameters, } return; } - + String assetId = parameters.get("asset", String.class); JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - + JSONObject assetData = assetNode.getJSONObject(assetId); /* * If the particular asset Id wasn't found, then @@ -84,18 +76,17 @@ private void importCollection(List ret, IParameters parameters, * In that case however, a better handling of S3 URL needs to figured out */ if (assetData != null) { - if (!STACAssetParser.isSupportedMediaType(assetData)) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { throw new Exception("Unsupported media type for the asset"); - } - String href = assetData.getString("href"); + } + String href = assetData.getString("href"); if (S3URLUtils.isS3Endpoint(href)) { - String[] bucketAndObject = href.split("://")[1].split("/", 2); + // String[] bucketAndObject = href.split("://")[1].split("/", 2); String s3Region = "unknown"; // TODO resolve the region parameters.put("awsRegion", s3Region); } } - - + parameters.put("asset", assetId); String resourceUrn = collectionId + "-" + assetId; Builder builder = buildResource(parameters, project, monitor, resourceUrn); @@ -104,10 +95,16 @@ private void importCollection(List ret, IParameters parameters, } else { monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); } - - // Think of a way to do the REGEX matching. Possibly not worth it to support + + // Think of a way to do the REGEX matching. Possibly not worth it to support // this for something as diverse as STAC - + + // String regex = null; + if (parameters.contains("regex")) { + // regex = parameters.get(Resources.REGEX_ENTRY, String.class); + parameters.remove(Resources.REGEX_ENTRY); + } + /* Set assetIds = STACAssetMapParser.readAssetNames(assets); for(String assetId : assetIds) { @@ -115,7 +112,7 @@ private void importCollection(List ret, IParameters parameters, Logging.INSTANCE.info("Asset " + assetId + " doesn't match REGEX, skipped"); continue; } - + JSONObject assetData = STACAssetMapParser.getAsset(assets, assetId); if (assetData != null) { if (!STACAssetParser.isSupportedMediaType(assetData)) { @@ -132,7 +129,7 @@ private void importCollection(List ret, IParameters parameters, String s3Region = "unknown"; // TODO resolve the region parameters.put("awsRegion", s3Region); } - + Builder builder = buildResource(parameters, project, monitor, resourceUrn); if (builder != null) { ret.add(builder); @@ -144,10 +141,10 @@ private void importCollection(List ret, IParameters parameters, */ } - private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException { - Builder builder = validator.validate( - Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), new URL(parameters.get("collection", String.class)), - parameters, monitor); + private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) + throws MalformedURLException { + Builder builder = validator.validate(Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), + new URL(parameters.get("collection", String.class)), parameters, monitor); if (builder == null) { return null; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 0a16ae462..dfdfdc726 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -50,10 +50,10 @@ public List resolve(JsonNode root) { List currentNodes = new ArrayList<>(); currentNodes.add(root); - for (PathPart part : path) { + for(PathPart part : path) { List nextNodes = new ArrayList<>(); - for (JsonNode current : currentNodes) { + for(JsonNode current : currentNodes) { if (current == null || current.isNull() || current.isMissingNode()) { continue; } @@ -87,9 +87,8 @@ private static void resolveFromArray(JsonNode arrayNode, PathPart part, List keywords = json.getJSONArray("keywords").toList(); - return keywords.isEmpty() ? null : - keywords.stream().collect(Collectors.joining(",")); + return keywords.isEmpty() ? null : keywords.stream().collect(Collectors.joining(",")); } - final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", "properties.sci:doi", "item_assets.sci:doi"); + final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", + "properties.sci:doi", "item_assets.sci:doi"); public static String readDOI(JSONObject json) { - Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)).findFirst(); + Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)) + .findFirst(); return doi.isPresent() ? doi.get() : null; } @@ -61,13 +62,13 @@ public static String readDOIAuthors(String doi) { */ public static boolean containsLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .anyMatch(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)); + .anyMatch(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)); } public static Optional getLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)) - .map(link -> ((JSONObject)link).getString("href")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)) + .map(link -> ((JSONObject) link).getString("href")).findFirst(); } public static JSONObject requestMetadata(String collectionUrl, String type) { @@ -87,7 +88,7 @@ public static String readLicense(JSONObject collection) { return null; } JSONArray links = collection.getJSONArray("links"); - for (int i = 0; i < links.length(); i++) { + for(int i = 0; i < links.length(); i++) { JSONObject link = links.getJSONObject(i); if (!link.has("rel") || !link.getString("rel").equals("license")) { continue; @@ -121,11 +122,12 @@ public static Type inferValueType(String key) { public static String getCatalogUrl(String collectionUrl, String collectionId, JSONObject collectionData) { // The URL of the catalog is the root if (!collectionData.has("links")) { - throw new KlabResourceAccessException("STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); + throw new KlabResourceAccessException( + "STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); } JSONArray links = collectionData.getJSONArray("links"); Optional rootLink = links.toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("root")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("root")).findFirst(); if (rootLink.isEmpty()) { throw new KlabResourceAccessException("STAC collection is missing a relationship to the root catalog"); } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 0384fc9c2..f046da579 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -71,8 +71,7 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni Predicate predicate = STACPathExpression.STACAssetPredicate .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); - assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, - predicate); + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, predicate); } else { throw new KlabIllegalArgumentException("Either asset or jsonSelector/jsonValue must be provided"); @@ -82,7 +81,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni JSONObject asset = assetNode.getJSONObject(assetId); Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. Map vals = STACAssetParser.getFileValues(asset); if (!vals.isEmpty()) { CodelistReference codelist = populateCodelist(assetId, vals); From 2703a2cd14800c92ecfe5e4a8b782aa125d60487 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Wed, 13 May 2026 14:40:19 +0200 Subject: [PATCH 50/74] Solve NPE if no asset match all the conditions --- .../java/org/integratedmodelling/klab/stac/STACEncoder.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index c64465b84..0570919d6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -486,6 +486,10 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + if (outRaster == null) { + scope.getMonitor().error("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + throw new KlabIllegalStateException("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + } coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); From 3c7ed198d78c60bc3f25df03a5b3ad2ec5ad31a9 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Wed, 13 May 2026 15:52:46 +0200 Subject: [PATCH 51/74] final fixes --- .../klab/stac/STACImporter.java | 56 ++++++++++++------- .../klab/stac/STACValidator.java | 9 ++- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index c8a689020..cb29f1996 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -74,30 +74,48 @@ private void importCollection(List ret, IParameters parameters, } String assetId = parameters.get("asset", String.class); - JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - - JSONObject assetData = assetNode.getJSONObject(assetId); + String resourceUrn = collectionId; + JSONObject assetData = null; /* - * If the particular asset Id wasn't found, then - * still proceed to create the resource since it can happen - * for a number of reasons. Pagination, STAC Backend etc. - * In that case however, a better handling of S3 URL needs to figured out + This is only for the part, to check the Asset is under an S3 Bucket */ - if (assetData != null) { - if (!STACAssetParser.isSupportedMediaType(assetData)) { - throw new Exception("Unsupported media type for the asset"); - } - String href = assetData.getString("href"); - if (S3URLUtils.isS3Endpoint(href)) { - String[] bucketAndObject = href.split("://")[1].split("/", 2); - String s3Region = "unknown"; // TODO resolve the region - parameters.put("awsRegion", s3Region); + if (assetId != null) { + JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + assetData = assetNode.getJSONObject(assetId); + /* + * If the particular asset Id wasn't found, then + * still proceed to create the resource since it can happen + * for a number of reasons. Pagination, STAC Backend etc. + * In that case however, a better handling of S3 URL needs to figured out + */ + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + throw new Exception("Unsupported media type for the asset"); + } + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } } - } + resourceUrn = collectionId + "-" + assetId; + } + //TODO: Check if assetId is null, and instead a jsonSelector and Value is passed! + //else if (parameters.get("jsonSelector", String.class) != null) { + // String selector = parameters.get("jsonSelector", String.class); + // String val = parameters.get("jsonValue", String.class); + // assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, null, STACPathExpression.STACAssetPredicate.fromKongJsonObject(selector, val)); + //} - parameters.put("asset", assetId); - String resourceUrn = collectionId + "-" + assetId; + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + Builder builder = buildResource(parameters, project, monitor, resourceUrn); if (builder != null) { ret.add(builder); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 0384fc9c2..b89bf7e94 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -49,8 +49,9 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni collectionId = collectionData.getString("id"); userData.put("collectionId", collectionId); } - IGeometry geometry = STACCollectionParser.readGeometry(collectionData); + + IGeometry geometry = STACCollectionParser.readGeometry(collectionData); Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); // The default URL of the resource is the collection endpoint. May be overwritten. @@ -60,7 +61,6 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (userData.contains("asset")) { String requestedAssetId = userData.get("asset", String.class); - assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); } else if (userData.contains("jsonSelector")) { @@ -75,7 +75,10 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni predicate); } else { - throw new KlabIllegalArgumentException("Either asset or jsonSelector/jsonValue must be provided"); + // Just import Features + monitor.info("import STAC Collection for Features"); + readMetadata(collectionData, builder); + return builder; } String assetId = assetNode.keys().next(); From 30bb415b3ce80d781edd05f83290be2f6d18eca9 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Thu, 14 May 2026 11:05:24 +0200 Subject: [PATCH 52/74] fixes for time --- .../klab/stac/STACCollectionParser.java | 4 ++-- .../org/integratedmodelling/klab/stac/STACValidator.java | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index 6dd160b9f..bbb50c95e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -32,7 +32,7 @@ public static String readCollectionId(JSONObject collection) { * @param parameters * @return geometry */ - public static IGeometry readGeometry(JSONObject collection) { + public static IGeometry readGeometry(JSONObject collection, boolean featureOnly) { GeometryBuilder gBuilder = Geometry.builder(); JSONObject extent = collection.getJSONObject("extent"); @@ -49,7 +49,7 @@ public static IGeometry readGeometry(JSONObject collection) { } // TODO find non-ad-hoc cases - if (collection.getString("id").equals("slovak_SK_v5_reference-points_EUNIS2012")) { + if (collection.getString("id").equals("slovak_SK_v5_reference-points_EUNIS2012") || featureOnly) { return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical"); } return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("grid"); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index d02bc6687..e61c55089 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -50,8 +50,12 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni userData.put("collectionId", collectionId); } - - IGeometry geometry = STACCollectionParser.readGeometry(collectionData); + IGeometry geometry = null; + if (!userData.contains("asset") && !userData.contains("jsonSelector")) { + geometry = STACCollectionParser.readGeometry(collectionData, true); + } else { + geometry = STACCollectionParser.readGeometry(collectionData, false); + } Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); // The default URL of the resource is the collection endpoint. May be overwritten. From 8dbb873859e50de69f3be0a056a2a284dcb36452 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Thu, 14 May 2026 15:55:27 +0200 Subject: [PATCH 53/74] adds check on assetId itself, adds S3EndpointUrl on import --- .../klab/stac/STACCollectionParser.java | 34 ++++++++++++++++--- .../klab/stac/STACImporter.java | 6 +++- .../klab/stac/STACValidator.java | 12 +++++++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index bbb50c95e..fcab6242f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -62,12 +62,36 @@ private static JSONObject findAsset(JSONObject assets, String assetId, Predicate if (assetId != null) { JSONObject asset = assets.optJSONObject(assetId); - if (asset == null) { - return null; + if (asset != null) { + JSONObject result = new JSONObject(); + result.put(assetId, asset); + return result; + } else { + return assets.keySet().stream() + .map(key -> { + JSONObject assetNode = assets.optJSONObject(key); + if (assetNode == null) { + return null; + } + if (!assetNode.has("eo:bands")) { + return null; + } + JSONArray bandsArray = assetNode.optJSONArray("eo:bands"); + for (var bandObj:bandsArray) { + JSONObject band = (JSONObject) bandObj; + if (assetId.equals(band.get("name"))) { + JSONObject result = new JSONObject(); + result.put(key, assetNode); + return result; + } + } + + return null; + }) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } - JSONObject result = new JSONObject(); - result.put(assetId, asset); - return result; } if (predicate != null) { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 3dfab9173..1d640dd5a 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -73,7 +73,11 @@ private void importCollection(List ret, IParameters parameters, */ if (assetId != null) { JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - assetData = assetNode.getJSONObject(assetId); + //assetData = assetNode.getJSONObject(assetId); + assetData = (JSONObject) assetNode.toMap() + .values() + .iterator() + .next(); /* * If the particular asset Id wasn't found, then * still proceed to create the resource since it can happen diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index e61c55089..8b44dad9a 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -27,6 +27,7 @@ import org.integratedmodelling.klab.rest.MappingReference; import org.integratedmodelling.klab.rest.ResourceCRUDRequest; import org.integratedmodelling.klab.utils.Pair; +import org.integratedmodelling.klab.utils.s3.S3URLUtils; import kong.unirest.json.JSONObject; @@ -86,6 +87,17 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni String assetId = assetNode.keys().next(); JSONObject asset = assetNode.getJSONObject(assetId); + String href = asset.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + if (href.contains("waw3")) { + userData.put("s3EndpointUrl", "https://s3.waw3-1.cloudferro.com"); + } else if (href.contains("waw4")) { + userData.put("s3EndpointUrl", "https://s3.waw4-1.cloudferro.com"); + } else { + userData.put("s3EndpointUrl", "unknown"); + } + + } Type type = readRasterDataType(asset); // Currently, only files:values is supported. If needed, the classification extension could From 6d82bf4014bfaec6cffbed13d4b3efa5cf6dc358 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Fri, 15 May 2026 11:03:40 +0200 Subject: [PATCH 54/74] adds aws deps --- adapters/klab.ogc/pom.xml | 18 ++++++++++++++++++ .../klab/stac/STACEncoder.java | 10 +++++----- pom.xml | 1 + 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/adapters/klab.ogc/pom.xml b/adapters/klab.ogc/pom.xml index bf4eef970..87c4b1481 100644 --- a/adapters/klab.ogc/pom.xml +++ b/adapters/klab.ogc/pom.xml @@ -89,6 +89,24 @@ aws-lightweight-client-java ${s3.version} + + + + apache-client + software.amazon.awssdk + 2.29.14 + + + commons-logging + commons-logging + + + + + software.amazon.awssdk + protocol-core + 2.29.14 + - - apache-client - software.amazon.awssdk - 2.29.14 - - - commons-logging - commons-logging - - - - - software.amazon.awssdk - protocol-core - 2.29.14 - - 34.0 - 0.11.1-SNAPSHOT + 0.11.3-SNAPSHOT 1.20.0 1.5.0 17 From 9b3875e9d0dd953c938738241f1c15bbab06183e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 18 May 2026 10:54:04 +0200 Subject: [PATCH 57/74] adds resampling to fix for the shift --- .../integratedmodelling/klab/stac/STACEncoder.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index b96c97346..c9c805096 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -377,10 +377,10 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); } + CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326"); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } - CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326"); if (!CRS.equalsIgnoreMetadata( coverage.getCoordinateReferenceSystem(), targetCRS)) { @@ -509,8 +509,17 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } + CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326"); manager.close(); encoder = new RasterEncoder(); + if (!CRS.equalsIgnoreMetadata( + coverage.getCoordinateReferenceSystem(), + targetCRS)) { + + coverage = (GridCoverage2D) Operations.DEFAULT.resample( + coverage, + targetCRS); + } ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { e.printStackTrace(); From 0c3e90e21d67455ef0b1a40a55cbe72c53edbe12 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 18 May 2026 15:28:11 +0200 Subject: [PATCH 58/74] use HMCRSRegistry instead of .Decode --- .../org/integratedmodelling/klab/stac/STACEncoder.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index c9c805096..7ed6103dc 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -72,6 +72,7 @@ import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; +import org.hortonmachine.gears.utils.crs.HMCrsRegistry; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; @@ -377,7 +378,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); } - CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326"); + CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } @@ -413,7 +414,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail // fast } - + IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation ? ((Observation) scope.getTargetArtifact()).getObservable() : null; @@ -509,12 +510,15 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } - CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326"); + CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); manager.close(); encoder = new RasterEncoder(); if (!CRS.equalsIgnoreMetadata( coverage.getCoordinateReferenceSystem(), targetCRS)) { + + System.out.println("Resampling! Found " + coverage.getCoordinateReferenceSystem() + + " Resampling to " + targetCRS); coverage = (GridCoverage2D) Operations.DEFAULT.resample( coverage, From b487c1e275020494b0ee91ea7b9bad672d31ab23 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 18 May 2026 15:29:53 +0200 Subject: [PATCH 59/74] fixes --- .../java/org/integratedmodelling/klab/stac/STACEncoder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 7ed6103dc..24a242336 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -516,9 +516,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou if (!CRS.equalsIgnoreMetadata( coverage.getCoordinateReferenceSystem(), targetCRS)) { - - System.out.println("Resampling! Found " + coverage.getCoordinateReferenceSystem() - + " Resampling to " + targetCRS); coverage = (GridCoverage2D) Operations.DEFAULT.resample( coverage, From 776b39bb56c18baa063b82a5cfa1d5e8fd1d9638 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 18 May 2026 15:39:32 +0200 Subject: [PATCH 60/74] fixes --- .../klab/stac/STACEncoder.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 24a242336..b7caf89fc 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -32,6 +32,9 @@ import org.hortonmachine.gears.utils.RegionMap; import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; +import org.hortonmachine.gears.utils.crs.HMCrsRegistry; +import org.hortonmachine.gears.utils.crs.HMCrsTransformer; + import org.integratedmodelling.klab.Authentication; import org.integratedmodelling.klab.Observables; import org.integratedmodelling.klab.api.data.IGeometry; @@ -72,7 +75,6 @@ import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.hortonmachine.gears.utils.crs.HMCrsRegistry; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; @@ -376,20 +378,18 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, MergeMode.SUBSTITUTE, lpm); + CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); + if (!HMCrsRegistry.crsEquals(outRaster.getCrs(),targetCRS)) { + var transformer = new HMCrsTransformer(outRaster.getCrs(), targetCRS); + transformer.setAcceptLenientDatumShift(true); + outRaster = transformer.transform(outRaster); + } + coverage = outRaster.buildCoverage(); } - CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } - if (!CRS.equalsIgnoreMetadata( - coverage.getCoordinateReferenceSystem(), - targetCRS)) { - - coverage = (GridCoverage2D) Operations.DEFAULT.resample( - coverage, - targetCRS); - } } catch (Exception e) { throw new KlabResourceAccessException( "Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); @@ -506,21 +506,20 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou scope.getMonitor().error("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); throw new KlabIllegalStateException("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); } + CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); + if (!HMCrsRegistry.crsEquals(outRaster.getCrs(),targetCRS)) { + var transformer = new HMCrsTransformer(outRaster.getCrs(), targetCRS); + transformer.setAcceptLenientDatumShift(true); + outRaster = transformer.transform(outRaster); + } + coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } - CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); + manager.close(); encoder = new RasterEncoder(); - if (!CRS.equalsIgnoreMetadata( - coverage.getCoordinateReferenceSystem(), - targetCRS)) { - - coverage = (GridCoverage2D) Operations.DEFAULT.resample( - coverage, - targetCRS); - } ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { e.printStackTrace(); From dfd2c455fecc4e4640b06c8e5ce5658358a43d1e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Wed, 20 May 2026 11:55:28 +0200 Subject: [PATCH 61/74] fix shift in stac resources --- .../klab/stac/STACEncoder.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index b7caf89fc..53e597351 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -374,7 +374,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou predicate = getAssetPredicate(resource); } catch (KlabIllegalArgumentException e) { manager.close(); - throw e; + throw e; } HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, MergeMode.SUBSTITUTE, lpm); @@ -450,7 +450,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); - throw new KlabIllegalStateException("No STAC items found for this context."); + throw new KlabIllegalStateException("No STAC items found for this context, check the Spatial/ Temporal bounds of items and the Context"); } if (mergeMode == HMRaster.MergeMode.SUBSTITUTE) { @@ -501,10 +501,10 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, - MergeMode.SUBSTITUTE, lpm); + MergeMode.SUBSTITUTE, lpm); if (outRaster == null) { - scope.getMonitor().error("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); - throw new KlabIllegalStateException("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + scope.getMonitor().error("Unable to build the output from the STAC Resource"); + throw new KlabIllegalStateException("Unable to build the output from the STAC Resource"); } CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); if (!HMCrsRegistry.crsEquals(outRaster.getCrs(),targetCRS)) { @@ -513,8 +513,13 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou outRaster = transformer.transform(outRaster); } - coverage = outRaster.buildCoverage(); - if (bandIndex != null) { // Which means theat it's a Multi Band COG + + HMRaster paddedRaster = new HMRasterWritableBuilder().setName("padded").setRegion(region) + .setCrs(targetCRS).setNoValue(outRaster.getNovalue()).build(); + paddedRaster.mapRaster(null, outRaster, null); + coverage = paddedRaster.buildCoverage(); + + if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } From da12df0a3e41f2cc9a95cf094c9f06970cefd0ca Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Thu, 18 Jun 2026 10:01:01 +0200 Subject: [PATCH 62/74] Avoid NPE skipping strange null observable --- .../klab/engine/runtime/APIObservationTask.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java index f8dbc1383..1a595663f 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java @@ -204,7 +204,9 @@ private long observe() { * make every observation */ for (IObservable observable : observables) { - + if (observable == null) { + continue; + } IObservation observation = null; ITask observationTask = ((ISubject) context).observe(observable); observationTask.addScenarios(scenarios); From f577e88da2ff67f588c990b39ca7e0280cd64da2 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 23 Jun 2026 13:00:10 +0200 Subject: [PATCH 63/74] Align code with changes in HM --- .../integratedmodelling/geoprocessing/core/KrigingResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java index 375a05867..3484285c1 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java @@ -13,7 +13,7 @@ import org.hortonmachine.gears.libs.modules.HMConstants; import org.hortonmachine.gears.utils.coverage.CoverageUtilities; import org.hortonmachine.gears.utils.math.matrixes.MatrixException; -import org.hortonmachine.hmachine.modules.statistics.kriging.old.OmsKriging; +import org.hortonmachine.hmachine.modules.statistics.krigingexp.old.OmsKriging; import org.integratedmodelling.geoprocessing.TaskMonitor; import org.integratedmodelling.kim.api.IKimConcept; import org.integratedmodelling.klab.Concepts; From 9e97e7fc78ca58cfe43f6d42192d056a6fd7a242 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 29 Jun 2026 12:18:03 +0200 Subject: [PATCH 64/74] Modification to CALIPER authority to make it works --- .../authorities/caliper/CaliperAuthority.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java b/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java index 111ed6711..1e1e5cda3 100644 --- a/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java +++ b/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java @@ -61,7 +61,7 @@ public class CaliperAuthority implements IAuthority { + " ?concept skos:prefLabel ?label_en . FILTER(contains(lcase(str(?label_en)), '{QUERY_STRING}')) .\r\n" + " ?concept skos:notation ?code .\r\n" + " ?concept skos:broader ?broader .\r\n" + "} order by ?code"; - private static final String SPARQL_ENDPOINT = "https://stats.fao.org/caliper/AllVocs/"; + private static final String SPARQL_ENDPOINT = "https://caliper.integratedmodelling.org/caliper/sparql"; private static final Map CALIPER_SCHEMES = new HashMap<>(); private static final Map CALIPER_DESCRIPTIONS = new HashMap<>(); private static final Map CALIPER_URLS = new HashMap<>(); @@ -70,26 +70,26 @@ public class CaliperAuthority implements IAuthority { // TODO all this should come from caliper, filtered as needed and cached -// CALIPER_SCHEMES.put("ISIC", "http://unstats.un.org/classifications/ISIC/rev4/scheme"); + CALIPER_SCHEMES.put("ISIC", "https://unstats.un.org/classifications/ISIC/rev4/scheme"); // CALIPER_SCHEMES.put("ICC10", "http://stats-class.fao.uniroma2.it/ICC/v1.0/scheme"); - CALIPER_SCHEMES.put("ICC", "http://stats.fao.org/classifications/ICC/v1.1/scheme"); + CALIPER_SCHEMES.put("ICC", "https://stats.fao.org/classifications/ICC/v1.1/scheme"); // CALIPER_SCHEMES.put("M49", "http://stats.fao.org/classifications/geo/M49"); // CALIPER_SCHEMES.put("SDGEO", "http://stats.fao.org/classifications/geo/M49/SDG-groups"); // CALIPER_SCHEMES.put("FOODEX2", "http://stats.fao.org/classifications/foodex2/all"); // CALIPER_SCHEMES.put("CPC20", "http://stats-class.fao.uniroma2.it/CPC/v2.0/scheme"); - CALIPER_SCHEMES.put("CPC", "http://unstats.un.org/classifications/CPC/v2.1/core"); + CALIPER_SCHEMES.put("CPC", "https://unstats.un.org/classifications/CPC/v2.1/core"); // CALIPER_SCHEMES.put("CPC21FERT", "http://stats-class.fao.uniroma2.it/CPC/v2.1/fert"); // CALIPER_SCHEMES.put("FCL", "http://stats-class.fao.uniroma2.it/FCL/v2019/scheme"); - CALIPER_SCHEMES.put("HS", "http://stats.fao.org/classifications/HS/fao_mapping_targets/scheme"); + //CALIPER_SCHEMES.put("HS", "http://stats.fao.org/classifications/HS/fao_mapping_targets/scheme"); - CALIPER_URLS.put("ISIC", "http://unstats.un.org/classifications/ISIC/rev4"); + CALIPER_URLS.put("ISIC", "https://unstats.un.org/classifications/ISIC/rev4"); // CALIPER_SCHEMES.put("ICC10", "http://stats-class.fao.uniroma2.it/ICC/v1.0"); - CALIPER_URLS.put("ICC", "http://stats.fao.org/classifications/ICC/v1.1"); + CALIPER_URLS.put("ICC", "https://stats.fao.org/classifications/ICC/v1.1"); // CALIPER_URLS.put("M49", "http://stats.fao.org/classifications/geo/m49"); // CALIPER_URLS.put("SDGEO", "http://stats.fao.org/classifications/geo/M49/SDG-groups"); // CALIPER_URLS.put("FOODEX2", "http://stats.fao.org/classifications/foodex2"); // CALIPER_SCHEMES.put("CPC20", "http://stats-class.fao.uniroma2.it/CPC/v2.0"); - CALIPER_URLS.put("CPC", "http://stats.fao.org/classifications/CPC/v2.1"); + CALIPER_URLS.put("CPC", "https://stats.fao.org/classifications/CPC/v2.1"); // CALIPER_SCHEMES.put("CPC21FERT", "http://stats-class.fao.uniroma2.it/CPC/v2.1/fert"); // CALIPER_SCHEMES.put("FCL", "http://stats-class.fao.uniroma2.it/FCL/v2019"); // CALIPER_URLS.put("HS", "http://stats.fao.org/classifications/HS/fao_mapping_targets"); From 4daf00356b8b7208c80448bfe02664be2b350d61 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 29 Jun 2026 12:19:02 +0200 Subject: [PATCH 65/74] Format --- .../authorities/caliper/CaliperAuthority.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java b/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java index 1e1e5cda3..a629c10cf 100644 --- a/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java +++ b/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java @@ -80,7 +80,8 @@ public class CaliperAuthority implements IAuthority { CALIPER_SCHEMES.put("CPC", "https://unstats.un.org/classifications/CPC/v2.1/core"); // CALIPER_SCHEMES.put("CPC21FERT", "http://stats-class.fao.uniroma2.it/CPC/v2.1/fert"); // CALIPER_SCHEMES.put("FCL", "http://stats-class.fao.uniroma2.it/FCL/v2019/scheme"); - //CALIPER_SCHEMES.put("HS", "http://stats.fao.org/classifications/HS/fao_mapping_targets/scheme"); + // CALIPER_SCHEMES.put("HS", + // "http://stats.fao.org/classifications/HS/fao_mapping_targets/scheme"); CALIPER_URLS.put("ISIC", "https://unstats.un.org/classifications/ISIC/rev4"); // CALIPER_SCHEMES.put("ICC10", "http://stats-class.fao.uniroma2.it/ICC/v1.0"); @@ -163,7 +164,7 @@ public Identity getIdentity(String identityId, String catalog) { Set parents = new HashSet<>(); source = new AuthorityIdentity(); - for (Statement statement : model) { + for(Statement statement : model) { // System.out.println("CIAPA EL STATEMENT: " + statement); @@ -199,7 +200,7 @@ public Identity getIdentity(String identityId, String catalog) { ((AuthorityIdentity) source).setConceptName(sanitize(catalog, ((AuthorityIdentity) source).getId())); ((AuthorityIdentity) source).setLocator(ID + "." + catalog + ":" + ((AuthorityIdentity) source).getId()); - for (String parent : parents) { + for(String parent : parents) { if (((AuthorityIdentity) source).getParentIds() == null) { ((AuthorityIdentity) source).setParentIds(new ArrayList<>()); } @@ -236,7 +237,7 @@ public Capabilities getCapabilities() { ret.setFuzzy(true); ret.setName(ID); ret.setDescription(DESCRIPTION); - for (String s : CALIPER_DESCRIPTIONS.keySet()) { + for(String s : CALIPER_DESCRIPTIONS.keySet()) { ret.getSubAuthorities().add(new Pair<>(s, CALIPER_DESCRIPTIONS.get(s))); } @@ -259,7 +260,7 @@ public List search(String query, String catalog) { if (response.isSuccess()) { try { JSONObject result = response.getBody().getObject(); - for (Object zoz : result.getJSONObject("results").getJSONArray("bindings")) { + for(Object zoz : result.getJSONObject("results").getJSONArray("bindings")) { JSONObject res = (JSONObject) zoz; String code = res.getJSONObject("code").getString("value"); @@ -299,7 +300,7 @@ public static void main(String[] args) { try (InputStream input = new URL("http://unstats.un.org/classifications/CPC/v2.0/0.ttl").openStream()) { Model model = Rio.parse(input, RDFFormat.TURTLE); - for (Statement statement : model) { + for(Statement statement : model) { System.out.println("CIAPA EL STATEMENT: " + statement); } } catch (Exception e) { @@ -308,15 +309,15 @@ public static void main(String[] args) { } - @Override - public String getName() { - return ID; - } + @Override + public String getName() { + return ID; + } - @Override - public ICodelist getCodelist() { - // TODO this may be less than obvious with Caliper - return null; - } + @Override + public ICodelist getCodelist() { + // TODO this may be less than obvious with Caliper + return null; + } } From de6eb95363560f1b5c68bfd53cc48c1bdadba57b Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 7 Jul 2026 14:43:17 +0200 Subject: [PATCH 66/74] adds slow requests, fixes vectors --- .../klab/stac/STACEncoder.java | 65 ++++++++++++++++++- .../stac/extensions/STACFeatureExtension.java | 39 ++++++----- .../klab/Authentication.java | 2 +- 3 files changed, 83 insertions(+), 23 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 53e597351..09006d1c9 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -2,9 +2,11 @@ import java.io.IOException; import java.io.OutputStream; +import java.net.URL; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; @@ -63,23 +65,30 @@ import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; import org.integratedmodelling.klab.exceptions.KlabValidationException; +import org.integratedmodelling.klab.ogc.OpenEOAdapter; import org.integratedmodelling.klab.ogc.STACAdapter; import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; +import org.integratedmodelling.klab.openeo.OpenEO; +import org.integratedmodelling.klab.openeo.OpenEO.OpenEOFuture; +import org.integratedmodelling.klab.openeo.OpenEO.Process; import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACFeatureExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; +import org.integratedmodelling.klab.utils.JsonUtils; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.operation.union.UnaryUnionOp; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; +import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; public class STACEncoder implements IResourceEncoder { @@ -360,7 +369,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou HMStacCollection collection = null; try { manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + collection = manager.getCollectionById(collectionId); } catch (Exception e1) { throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); } @@ -405,7 +414,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou HMStacCollection collection = null; try { manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + collection = manager.getCollectionById(collectionId); if (collection == null) { scope.getMonitor() @@ -446,6 +455,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.close(); return; } + List items = collection.searchItems(); if (items.isEmpty()) { @@ -499,6 +509,57 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." + "The transformation process could affect the data."); } + + + // Specific Implementation for the Slow Requests flow in WEED + if (collection.getId().contains("EU_modelV2-1-MECE") + && resource.getUrn().contains("im.resources-main")) { + Geometry unionMLStacInference = UnaryUnionOp.union( + items.stream() + .map(item -> item.getGeometry()) + .filter(g -> g != null && !g.isEmpty()) + .collect(Collectors.toList()) + ); + + if (!unionMLStacInference.contains(poly)) { + scope.getMonitor().warn("The requested extend for ML inferences is not completely contained in STAC, Starting ML Inference Request"); + + OpenEO service = OpenEOAdapter.getClient("openeo_weed.dataspace.copernicus.eu"); + List processes = new ArrayList<>(); + String processNamespace = "https://raw.githubusercontent.com/ESA-WEED-project/OpenEO-UDP-UDF-catalogue/refs/heads/main/UDP/json/udp_starter.json"; + String processID = "udp_starter"; + + Process process = JsonUtils.load(new URL(processNamespace), + Process.class); + process.encodeSelf(processNamespace); + processes.add(process); + + JSONObject arguments = new JSONObject() + .put("bbox", new JSONObject() + .put("crs", 4326) + .put("west", bbox.get(0)) + .put("south", bbox.get(3)) + .put("east", bbox.get(1)) + .put("north", bbox.get(2))) + .put("digitalId", "AM1729") // Forms the STAC coordinate later + .put("scenarioId", "DT_SLOW_FLOW") // Forms the STAC coordinate later + .put("year", ctxTime.getEnd().getYear()) + .put("onnx_model", "EUNIS2021plus_panEU_v201_2024_OneZone") // Hardcoding for now only for Europe, until the "BEST" model is decided! + .put("dt_url", "https://services.integratedmodelling.org/runtime/main/api/v1/dt/ESA_INSTITUTIONAL.3vh554o6h6c"); + + + OpenEOFuture job = service.submit(processID, arguments, + scope.getMonitor(), processes.toArray(new Process[processes.size()])); + + if (job.isCancelled()) { + scope.getMonitor().warn("job canceled"); + } else if (job.getError() != null) { + scope.getMonitor().error(job.getError()); + } else { + scope.getMonitor().info("Inference Request has been submitted to the ML Workflows"); + } + } + } HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java index aa668f566..b422e8122 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -42,7 +42,7 @@ public static FeatureSource getFeatures(JSONOb JSONObject searchPayload = new JSONObject() - .put("limit", 100) + .put("limit", 1000) .put("bbox", bboxArray) .put("collections", new JSONArray().put(collectionId)); @@ -66,26 +66,25 @@ public static FeatureSource getFeatures(JSONOb HMStacItem item = HMStacItem.fromSimpleFeature(feat); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - if (item.getStartTimestamp() != null && item.getEndTimestamp() != null) { - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { - featureList.add(feat); - } - } else { - featureList.add(feat); - } - + if (item.getStartTimestamp() == null || item.getEndTimestamp() == null) { // Assume best case scenario + featureList.add(feat); + continue; + } + + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { + featureList.add(feat); + } } catch (Exception e) { e.printStackTrace(); } diff --git a/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java b/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java index 2d980bbdf..9c7761261 100644 --- a/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java +++ b/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java @@ -463,7 +463,7 @@ public ExternalAuthenticationCredentials getCredentials(String endpoint) { * @return the credential or null if not present */ private ExternalAuthenticationCredentials getCredentialsByUrl(String hostUrl) { - return externalCredentials.values().stream().filter(credential -> credential.getURL().equals(hostUrl)).sorted() + return externalCredentials.values().stream().filter(credential -> credential.getURL().equals(hostUrl)) .findFirst().orElse(null); } From 483fb7de9d54dcde1f62ef494968852cf2f9292a Mon Sep 17 00:00:00 2001 From: AM1729 Date: Thu, 9 Jul 2026 13:05:30 +0200 Subject: [PATCH 67/74] gets data from Copernicus S3 --- .../klab/stac/STACEncoder.java | 15 +++++++++++---- .../klab/stac/STACValidator.java | 8 ++++++++ pom.xml | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 09006d1c9..0661f5407 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -430,11 +430,18 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); + //collection.setGeometryFilter(poly); // collection.setTimestampFilter(new Date(start.getMilliseconds()), new // Date(end.getMilliseconds())); --> Filter later :) GridCoverage2D coverage = null; + + collection.setBboxFilter(new double[] { + bbox.get(0), + bbox.get(2), + bbox.get(1), + bbox.get(3) + }); // Allow transform ensures the process to finish, but I would not bet on the resulting // data @@ -456,7 +463,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou return; } - List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -487,7 +493,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // temporal filtering :( like ECDC items = items.stream() .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) - .collect(Collectors.toList()); + .collect(Collectors.toList()); if (items.size() == 0) { manager.close(); @@ -631,7 +637,8 @@ private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() .toEpochMilli(); - return startMillis >= itemStart && endMillis <= itemEnd; + return (startMillis >= itemStart || Math.abs(startMillis - itemStart) < 10000) + && (endMillis <= itemEnd || Math.abs(endMillis - itemEnd) < 10000); // Allow some small tolerance } catch (Exception e) { e.printStackTrace(); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 8b44dad9a..d87727ed7 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -93,6 +93,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni userData.put("s3EndpointUrl", "https://s3.waw3-1.cloudferro.com"); } else if (href.contains("waw4")) { userData.put("s3EndpointUrl", "https://s3.waw4-1.cloudferro.com"); + } else if (collectionUrl.contains("https://stac.dataspace.copernicus.eu/")){ + userData.put("s3EndpointUrl", "https://eodata.dataspace.copernicus.eu/"); } else { userData.put("s3EndpointUrl", "unknown"); } @@ -143,6 +145,12 @@ private void generateCodeList(Builder builder, String assetId, JSONObject asset) } private Type readRasterDataType(JSONObject asset) { + + if (asset.has("type")) { + if(asset.get("type").toString().contains("image/tiff")) { // Numbers + return Type.NUMBER; + } + } if (!asset.has("raster:bands")) { return null; } diff --git a/pom.xml b/pom.xml index 8e2739378..f33933d1c 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ 17 - 34.0 + 34.4 0.11.3-SNAPSHOT 1.20.0 1.5.0 From 60433f821f02b3f025052469770c35d723e14294 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 13 Jul 2026 11:33:22 +0200 Subject: [PATCH 68/74] Avoid NPE in case of not local resource --- .../org/integratedmodelling/klab/ide/views/ResourceEditor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java b/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java index 5d64c3f11..7cdf4406f 100644 --- a/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java +++ b/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java @@ -560,6 +560,9 @@ public void loadResource(ResourceReference resource) { private File getResourcePath(ResourceReference resource) { String path = resource.getLocalPath(); + if (path == null) { + return null; + } String project = Path.getFirst(path, "/"); IKimProject prj = Kim.INSTANCE.getProject(project); if (project != null) { From 713097d40de3daea54c97903d4bec7954395db9f Mon Sep 17 00:00:00 2001 From: AM1729 Date: Wed, 15 Jul 2026 12:27:44 +0200 Subject: [PATCH 69/74] adds some ruleguards while importing STAC resources --- .../org/integratedmodelling/klab/stac/STACValidator.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index d87727ed7..3e3f0d6e6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -65,6 +65,9 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni JSONObject assetNode; if (userData.contains("asset")) { + if(userData.contains("cog")) { + throw new KlabIllegalArgumentException("STAC asset and cog URL both shouldn't be provided while importing"); + } String requestedAssetId = userData.get("asset", String.class); assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); @@ -72,6 +75,10 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (!userData.contains("jsonValue")) { throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); } + + if(userData.contains("cog")) { + throw new KlabIllegalArgumentException("jsonSelector and jsonValue shouldn't be provided along with cog URL while importing"); + } Predicate predicate = STACPathExpression.STACAssetPredicate .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); From 34517bcc9cdfcc2e3edb836e871cc015980d53fb Mon Sep 17 00:00:00 2001 From: AM1729 Date: Thu, 23 Jul 2026 14:31:55 +0200 Subject: [PATCH 70/74] fixes bbox for stac resources, observing outisde bounds message --- .../klab/stac/STACCollectionParser.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index fcab6242f..79d202376 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -33,26 +33,34 @@ public static String readCollectionId(JSONObject collection) { * @return geometry */ public static IGeometry readGeometry(JSONObject collection, boolean featureOnly) { - GeometryBuilder gBuilder = Geometry.builder(); + + GeometryBuilder gBuilder = Geometry.builder(); JSONObject extent = collection.getJSONObject("extent"); List bbox = extent.getJSONObject("spatial").getJSONArray("bbox").getJSONArray(0).toList(); - gBuilder.space().boundingBox(Double.valueOf(bbox.get(0).toString()), Double.valueOf(bbox.get(1).toString()), - Double.valueOf(bbox.get(2).toString()), Double.valueOf(bbox.get(3).toString())); - - List interval = extent.getJSONObject("temporal").getJSONArray("interval").getJSONArray(0).toList(); - if (interval.get(0) != null) { - gBuilder.time().start(Instant.parse(interval.get(0).toString()).toEpochMilli()); - } - if (interval.size() > 1 && interval.get(1) != null) { - gBuilder.time().end(Instant.parse(interval.get(1).toString()).toEpochMilli()); - } + gBuilder.space().boundingBox(Double.valueOf(bbox.get(0).toString()), Double.valueOf(bbox.get(2).toString()), + Double.valueOf(bbox.get(1).toString()), Double.valueOf(bbox.get(3).toString())); // In STAC, the geometry obj should necessarily be in WGS84 so it's good + + // IMPORTANT: In k.LAB it's minX, maxX, minY, maxY; but in STAC or geoserver the bbox is + // minX, minY, maxX, maxY + + // Avoid setting the time since, they get filtered later on + +// List interval = extent.getJSONObject("temporal").getJSONArray("interval").getJSONArray(0).toList(); +// if (interval.get(0) != null) { +// gBuilder.time().start(Instant.parse(interval.get(0).toString()).toEpochMilli()); +// } +// if (interval.size() > 1 && interval.get(1) != null) { +// gBuilder.time().end(Instant.parse(interval.get(1).toString()).toEpochMilli()); +// } + + //TODO: Figure out a way to know if time would be logical or grid // TODO find non-ad-hoc cases - if (collection.getString("id").equals("slovak_SK_v5_reference-points_EUNIS2012") || featureOnly) { - return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical"); - } - return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("grid"); +// if (collection.getString("id").equals("slovak_SK_v5_reference-points_EUNIS2012") || featureOnly) { +// return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical"); +// } + return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical"); } private static JSONObject findAsset(JSONObject assets, String assetId, Predicate predicate) { From 0daad81f6dd857a78229733b7cf7f4acb5ba3222 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Thu, 23 Jul 2026 14:34:09 +0200 Subject: [PATCH 71/74] removes logical without a time --- .../org/integratedmodelling/klab/stac/STACCollectionParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index 79d202376..130d83008 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -60,7 +60,7 @@ public static IGeometry readGeometry(JSONObject collection, boolean featureOnly) // if (collection.getString("id").equals("slovak_SK_v5_reference-points_EUNIS2012") || featureOnly) { // return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical"); // } - return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical"); + return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE); } private static JSONObject findAsset(JSONObject assets, String assetId, Predicate predicate) { From a2bfa7635d3534dca1c38f531d687a5b7efb2f3e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 27 Jul 2026 13:11:40 +0200 Subject: [PATCH 72/74] filters based on AssetID AND JsonSeletor; also adds the funtionality to check the assets based on the exact key match --- .../klab/stac/STACEncoder.java | 20 ++++++++++--------- .../klab/stac/STACPathExpression.java | 13 ++++++++++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 0661f5407..b1bd4cd61 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -316,10 +316,16 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou return false; } }; - } else if (resource.getParameters().get("jsonSelector", String.class) != null) { - // based on the JSON Expression on JSONSelector and JSONValue + } + + if (resource.getParameters().get("jsonSelector", String.class) != null) { + // based on the JSON Expression on JSONSelector and JSONValue; try { - assetPredicate = getAssetPredicate(resource); + if (assetPredicate != null) { + assetPredicate = assetPredicate.and(getAssetPredicateFromJSONSelector(resource)); // Predicate based on JSONPath AND the AssetID + } else { + assetPredicate = getAssetPredicateFromJSONSelector(resource); + } } catch (Exception e) { throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); } @@ -380,7 +386,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } Predicate predicate; try { - predicate = getAssetPredicate(resource); + predicate = getAssetPredicateFromJSONSelector(resource); } catch (KlabIllegalArgumentException e) { manager.close(); throw e; @@ -599,11 +605,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } } - private Predicate getAssetPredicate(IResource resource) { - String assetId = resource.getParameters().get("asset", String.class); - if (assetId != null) { - return STACPathExpression.STACAssetPredicate.fromHMStacAssetId(assetId); - } + private Predicate getAssetPredicateFromJSONSelector(IResource resource) { String jsonSelector = resource.getParameters().get("jsonSelector", String.class); String jsonValue = resource.getParameters().get("jsonValue", String.class); if (jsonSelector != null && !jsonSelector.isBlank() && jsonValue != null) { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index dfdfdc726..bc4f73bd6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -282,10 +282,19 @@ public static Predicate fromJsonPath(String jsonPath, String expectedValu } JsonNode node = jsonNodeExtractor.apply(object); - + if (node == null || node.isNull() || node.isMissingNode()) { return false; } + + /* + * Check if {"a.b": expectedValue} exists and also {"a": { "b": expectedValue }} + */ + + JsonNode directValue = node.get(jsonPath); + if (directValue != null && !directValue.isNull()) { + return expectedValue.equals(directValue.asText()); + } return expression.matches(node, expectedValue); }; @@ -300,7 +309,7 @@ public static Predicate fromJsonNode(String jsonPath, String expectedV } public static Predicate fromKongJsonObject(String jsonPath, String expectedValue) { - return fromJsonPath(jsonPath, expectedValue, STACAssetPredicate::toJsonNode); + return fromJsonPath(jsonPath, expectedValue, STACAssetPredicate::toJsonNode); } private static JsonNode toJsonNode(JSONObject jsonObject) { From 3412f373f0129aab24e327a5653d29b1fd4f57bd Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 28 Jul 2026 15:32:00 +0200 Subject: [PATCH 73/74] Add user agent to avoid 403 from Nominatim --- .../klab/communication/client/Client.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java b/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java index 512530c3f..f7bc4dd3e 100644 --- a/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java +++ b/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java @@ -76,6 +76,7 @@ import org.springframework.web.client.ResponseErrorHandler; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.client.UnknownContentTypeException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -101,6 +102,7 @@ public class Client extends RestTemplate implements IClient { public static final String KLAB_VERSION_HEADER = "KlabVersion"; + public static final String KLAB_USER_AGENT = "k.LAB/0.11.0"; public static final String KLAB_CONNECTION_TIMEOUT = "klab.connection.timeout"; ObjectMapper objectMapper; @@ -597,6 +599,8 @@ public T get(String url, Class< ? extends T> cls, Object... parameters) { HttpHeaders headers = new HttpHeaders(); headers.set("Accept", cls.equals(String.class) ? "text/plain" : "application/json"); headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + // add user-agent to avoid 403 from Nominatim + headers.set(HttpHeaders.USER_AGENT, KLAB_USER_AGENT); setAuthTokens(headers); HttpEntity entity = new HttpEntity<>(headers); @@ -634,7 +638,7 @@ public T get(String url, Class< ? extends T> cls, Object... parameters) { } else if (String.class.equals(cls)) { response = basicTemplate.exchange(url, HttpMethod.GET, entity, String.class); } else /* if (Map.class.isAssignableFrom(cls)) */ { - response = exchange(url, HttpMethod.GET, entity, Map.class); + response = exchange(url, HttpMethod.GET, entity, Map.class); } switch(response.getStatusCodeValue()) { From 07b29d322a793cd3d8e1aa6f256c286bbd800534 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 28 Jul 2026 15:32:25 +0200 Subject: [PATCH 74/74] Format code --- .../klab/communication/client/Client.java | 1367 +++++++++-------- 1 file changed, 686 insertions(+), 681 deletions(-) diff --git a/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java b/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java index f7bc4dd3e..9655a693a 100644 --- a/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java +++ b/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java @@ -83,17 +83,18 @@ import com.fasterxml.jackson.databind.SerializationFeature; /** - * Helper to avoid having to write 10 lines every time I need to do a GET with headers. It can be - * given authorization objects using the with(...) idiom: + * Helper to avoid having to write 10 lines every time I need to do a GET with + * headers. It can be given authorization objects using the with(...) idiom: * * RestTemplateHelper template = new RestTemplateHelper(); ... * template.with(session).post(....) ... * - * with(..) can be called with any {@link IIdentity} according to the type of authentication - * required by the receiver. Automatically sets the necessary tokens and parameters. + * with(..) can be called with any {@link IIdentity} according to the type of + * authentication required by the receiver. Automatically sets the necessary + * tokens and parameters. * - * The template also configures an objectmapper for optimal use in k.LAB, manages errors and inserts - * user agent headers. + * The template also configures an objectmapper for optimal use in k.LAB, + * manages errors and inserts user agent headers. * * * @author ferdinando.villa @@ -101,680 +102,684 @@ */ public class Client extends RestTemplate implements IClient { - public static final String KLAB_VERSION_HEADER = "KlabVersion"; - public static final String KLAB_USER_AGENT = "k.LAB/0.11.0"; - public static final String KLAB_CONNECTION_TIMEOUT = "klab.connection.timeout"; - - ObjectMapper objectMapper; - String authorizationToken; - String authenticationToken; - MediaType contentType = new MediaType(MediaType.APPLICATION_JSON, StandardCharsets.UTF_8); - RestTemplate basicTemplate; - private Set endpoints = new HashSet<>(); - - private static ClientHttpRequestFactory factory; - - public static Client createCustomTimeoutClient(int timeout) { - HttpComponentsClientHttpRequestFactory custom = new HttpComponentsClientHttpRequestFactory(); - custom.setReadTimeout(timeout); - custom.setConnectTimeout(timeout); - return new Client(custom); - } - - public static synchronized Client create() { - - if (factory == null) { - factory = new HttpComponentsClientHttpRequestFactory(); - if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { - int connectTimeout = 1000 - * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); - ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); - ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); - } - } - - return new Client(factory); - } - - public static Client createJson() { - Client ret = create(); - ret.setup(); - return ret; - } - - /** - * This one will return a client that interprets anything as JSON independent of content type. - * Required for misbehaving services that return JSON with other content types. - * - * @return - */ - public static synchronized Client createUniversalJSON() { - - if (factory == null) { - factory = new HttpComponentsClientHttpRequestFactory(); - if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { - int connectTimeout = 1000 - * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); - ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); - ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); - } - } - - return new UniversalClient(factory); - } - - public static class UniversalClient extends Client { - UniversalClient(ClientHttpRequestFactory factory) { - super(factory); - objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - List> messageConverters = new ArrayList>(); - MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); - /* - * Make this converter process any kind of response, not only application/*json, which - * is the default behaviour - */ - converter.setSupportedMediaTypes(Collections.singletonList(MediaType.ALL)); - messageConverters.add(converter); - this.setMessageConverters(messageConverters); - } - } - - public static class NodeClient extends Client { - - public NodeClient(INodeIdentity node) { - - super(factory); - - objectMapper = new ObjectMapper(); - List> messageConverters = new ArrayList<>(); - MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); - StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); - FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); - ProtobufHttpMessageConverter protobufConverter = new ProtobufHttpMessageConverter(); - // ByteArrayHttpMessageConverter byteConverter = new - // ByteArrayHttpMessageConverter(); - jsonMessageConverter.setObjectMapper(objectMapper); - - setErrorHandler(new JSONResponseErrorHandler()); - messageConverters.add(jsonMessageConverter); - messageConverters.add(utf8); - messageConverters.add(formHttpMessageConverter); - messageConverters.add(protobufConverter); - // messageConverters.add(byteConverter); - setMessageConverters(messageConverters); - this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); - this.authorizationToken = node.getId(); - } - } - - /** - * Send an authentication request to a hub for an engine. - * - * @param url - * @param request - * @return the response. If not authenticated, throw a KlabAuthorizationException. If timeout, - * return null. - */ - public EngineAuthenticationResponse authenticateEngine(String url, EngineAuthenticationRequest request) { - return post(url + API.HUB.AUTHENTICATE_ENGINE, request, EngineAuthenticationResponse.class); - } - - /** - * Send an authentication request to a hub for a node. - * - * @param url - * @param request - * @return the response. If not authenticated, throw a KlabAuthorizationException. If timeout, - * return null. - */ - public NodeAuthenticationResponse authenticateNode(String url, NodeAuthenticationRequest request) { - return post(url + API.HUB.AUTHENTICATE_NODE, request, NodeAuthenticationResponse.class); - } - - /** - * Check an engine's heartbeat. - * - * @param url base engine/node URL - * @return true if alive - */ - public boolean ping(String url, String endpoint) { - try { - ResponseEntity response = basicTemplate.exchange(url + endpoint, HttpMethod.GET, - new HttpEntity(null, null), Object.class); - return response.getStatusCodeValue() == 200; - } catch (Throwable e) { - return false; - } - } - - public boolean ping(String url) { - return ping(url, "/actuator/health"); - } - - public boolean pingService(String url) { - return ping(url, "/public/status"); - } - - private class JSONResponseErrorHandler implements ResponseErrorHandler { - - @Override - public void handleError(ClientHttpResponse response) throws IOException { - } - - @Override - public boolean hasError(ClientHttpResponse response) throws IOException { - HttpStatus status = response.getStatusCode(); - HttpStatus.Series series = status.series(); - return (HttpStatus.Series.CLIENT_ERROR.equals(series) || HttpStatus.Series.SERVER_ERROR.equals(series)); - } - } - - /** - * Add option to set both headers, Authorization and Authentication - * @param headers the headers object to set the new headers - */ - private void setAuthTokens(HttpHeaders headers) { - if (authorizationToken != null) { - headers.set(HttpHeaders.AUTHORIZATION, authorizationToken); - } - if (authenticationToken != null) { - headers.add("Authentication", authenticationToken); - } - } - /** - * Interceptor to add user agents and ensure that the authorization token gets in. - * - * @author ferdinando.villa - * - */ - public class AuthorizationInterceptor implements ClientHttpRequestInterceptor { - - @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) - throws IOException { - - HttpRequestWrapper requestWrapper = new HttpRequestWrapper(request); - HttpHeaders headers = requestWrapper.getHeaders(); - headers.set("Accept", "application/json"); - headers.set("X-User-Agent", "k.LAB " + Version.CURRENT); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - return execution.execute(requestWrapper, body); - } - } - - private void setup() { - - objectMapper = new ObjectMapper(); - List> messageConverters = new ArrayList<>(); - MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); - StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); - FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); - // ByteArrayHttpMessageConverter byteConverter = new - // ByteArrayHttpMessageConverter(); - jsonMessageConverter.setObjectMapper(objectMapper); - - setErrorHandler(new JSONResponseErrorHandler()); - messageConverters.add(jsonMessageConverter); - messageConverters.add(utf8); - messageConverters.add(formHttpMessageConverter); - // messageConverters.add(byteConverter); - setMessageConverters(messageConverters); - this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); - } - - private Client(ClientHttpRequestFactory factory) { - super(factory); - this.basicTemplate = new RestTemplate(); - setup(); - } - - private Client(Client client) { - super(factory); - this.basicTemplate = client.basicTemplate; - this.objectMapper = client.objectMapper; - this.endpoints = client.endpoints; - this.authorizationToken = client.authorizationToken; - this.authenticationToken = client.authenticationToken; - this.contentType = client.contentType; - setup(); - } - - private Client() { - super(factory); - } - - public Client withContentType(MediaType contentType) { - Client ret = new Client(this); - ret.contentType = contentType; - return ret; - } - - /** - * Return a client with authorization set to the passed object. - * - * @param authorizer any identity. - * @return a new - */ - @Override - public Client onBehalfOf(IIdentity authorizer) { - - Client ret = new Client(this); - ret.objectMapper = this.objectMapper; - ret.authorizationToken = authorizer.getId(); - return ret; - } - - public Client withAuthorization(String authorization) { - - Client ret = new Client(this); - ret.objectMapper = this.objectMapper; - ret.authorizationToken = authorization; - return ret; - } - - public Client withAuthentication(String authentication) { - - Client ret = new Client(); - ret.objectMapper = this.objectMapper; - ret.authenticationToken = authentication; - return ret; - } - - @SuppressWarnings("unchecked") - @Override - public T post(String url, Object data, Class< ? extends T> cls) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - if (data != null) { - headers.setContentType(contentType); - } - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - - HttpEntity entity = new HttpEntity<>(data, headers); - - try { - - ResponseEntity response = exchange(url, HttpMethod.POST, entity, Object.class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); - case 500: - throw new KlabResourceAccessException("internal: request " + url + " caused a remote server error"); - } - - if (response.getBody() == null) { - return null; - } - if (response.getBody() instanceof Map && ((Map< ? , ? >) response.getBody()).containsKey("exception") - && ((Map< ? , ? >) response.getBody()).get("exception") != null) { - - Map< ? , ? > map = (Map< ? , ? >) response.getBody(); - Object exception = map.get("exception"); - // Object path = map.get("path"); - Object message = map.get("message"); - // Object error = map.get("error"); - - dumpRequest(url, headers, data); - throw new KlabIOException("remote exception: " + (message == null ? exception : message)); - } - - if (cls.isAssignableFrom(response.getBody().getClass())) { - return (T) response.getBody(); - } - - try { - return objectMapper.convertValue(response.getBody(), cls); - } catch (Throwable t) { - System.out.println("Unrecognized response: " + response.getBody()); - throw t; - } - } catch (RestClientException e) { - - System.out.println("REST exception: " + e.getMessage()); - dumpRequest(url, headers, data); - throw new KlabIOException(e); - } - } - - private void dumpRequest(String url, HttpHeaders headers, Object data) { - - System.out.println("Endpoint: " + url); - System.out.println("Headers:"); - for(String header : headers.keySet()) { - System.out.println(" " + header + " = " + headers.get(header)); - } - System.out.println("Data:\n" + printAsJson(data)); - - } - - public void setUrl(String... url) { - if (url == null || url.length == 0) { - this.endpoints.clear(); - } else { - for(String u : url) { - this.endpoints.add(u); - } - } - } - - private String pickEndpoint() { - // TODO periodically check URLs and choose the first that responds (or the one - // with the smallest load) - return this.endpoints.isEmpty() ? null : this.endpoints.iterator().next(); - } - - @Override - public boolean getDownload(String url, File output) { - - url = checkEndpoint(url); - - RestTemplate restTemplate = new RestTemplate(); - restTemplate.getMessageConverters().add(new ByteArrayHttpMessageConverter()); - - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - - HttpEntity entity = new HttpEntity(headers); - - // HttpHeaders headers = new HttpHeaders(); - // headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); - // HttpEntity entity = new HttpEntity<>(headers); - - ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, entity, byte[].class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); - } - - if (response.getBody() == null) { - return false; - } - if (response.getStatusCode() == HttpStatus.OK) { - try { - Files.write(output.toPath(), response.getBody()); - } catch (IOException e) { - throw new KlabIOException(e); - } - } - - return true; - } - - private String checkEndpoint(String url) { - if (!url.toLowerCase().startsWith("http")) { - String ep = pickEndpoint(); - if (ep != null) { - url = ep + (ep.endsWith("/") || url.startsWith("/") ? "" : "/") + url; - } - } - return url; - } - - /** - * Issue a DELETE request. Called remove to avoid conflict with super. - * - * @param url - * @param parameters - * @return - */ - public Object remove(String url, Object... parameters) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - HttpEntity entity = new HttpEntity<>(headers); - - if (parameters != null) { - String params = ""; - for(int i = 0; i < parameters.length; i++) { - String key = parameters[i].toString(); - String nakedKey = key; - String val = parameters[++i].toString(); - if (!(key.startsWith("{") && key.endsWith("}"))) { - key = "{" + key + "}"; - } else { - nakedKey = key.substring(1, key.length() - 1); - } - if (url.contains(key)) { - url = url.replace(key, val); - } else { - params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + Escape.forURL(val); - } - } - if (!params.isEmpty()) { - url += "?" + params; - } - } - - ResponseEntity< ? > response = exchange(url, HttpMethod.DELETE, entity, Object.class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); - case 503: - throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); - } - - return response.getBody(); - } - - /** - * Instrumented for header communication and error parsing - * - * @param url - * @param cls - * @return the deserialized result - */ - @Override - @SuppressWarnings({"unchecked"}) - public T get(String url, Class< ? extends T> cls, Object... parameters) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", cls.equals(String.class) ? "text/plain" : "application/json"); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - // add user-agent to avoid 403 from Nominatim - headers.set(HttpHeaders.USER_AGENT, KLAB_USER_AGENT); - setAuthTokens(headers); - HttpEntity entity = new HttpEntity<>(headers); - - if (parameters != null) { - String params = ""; - for(int i = 0; i < parameters.length; i++) { - String key = parameters[i].toString(); - String nakedKey = key; - Object nextPar = parameters[++i]; - String val = nextPar == null ? null : nextPar.toString(); - - if (val == null) { - continue; - } - - if (!(key.startsWith("{") && key.endsWith("}"))) { - key = "{" + key + "}"; - } else { - nakedKey = key.substring(1, key.length() - 1); - } - if (url.contains(key)) { - url = url.replace(key, val); - } else { - params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + /* Escape.forURL( */val/* ) */; - } - } - if (!params.isEmpty()) { - url += "?" + params; - } - } - - ResponseEntity< ? > response = null; - if (cls.isArray()) { - response = exchange(url, HttpMethod.GET, entity, Object.class); - } else if (String.class.equals(cls)) { - response = basicTemplate.exchange(url, HttpMethod.GET, entity, String.class); - } else /* if (Map.class.isAssignableFrom(cls)) */ { - response = exchange(url, HttpMethod.GET, entity, Map.class); - } - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); - case 406: - case 503: - throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); - } - - if (response.getBody() instanceof Map) { - - Object exception = ((Map< ? , ? >) response.getBody()).get("exception"); - Object error = ((Map< ? , ? >) response.getBody()).get("error"); - - if (exception != null || error != null) { - Object message = ((Map< ? , ? >) response.getBody()).get("message"); - // Object error = response.getBody().get("error"); - throw new KlabIOException( - "remote exception: " + (message == null ? (exception == null ? error : exception) : message)); - } - - return objectMapper.convertValue(response.getBody(), cls); - - } else if (response.getBody() instanceof List && cls.isArray()) { - - List< ? > list = (List< ? >) response.getBody(); - Object ret = Array.newInstance(cls.getComponentType(), (((List< ? >) response.getBody()).size())); - for(int i = 0; i < list.size(); i++) { - Object object = list.get(i); - if (object instanceof Map) { - object = objectMapper.convertValue(object, cls.getComponentType()); - } - Array.set(ret, i, object); - } - - return (T) ret; - - } else if (response.getBody() != null && cls.isAssignableFrom(response.getBody().getClass())) { - return (T) response.getBody(); - } - - return null; - } - - @Override - public T postFile(String url, File file, Class< ? extends T> cls) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("file", new FileSystemResource(file)); - HttpEntity> entity = new HttpEntity<>(body, headers); - - try { - - final RestTemplate restTemplate = new RestTemplate(); - SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); - // without this, large files will eat up the heap - requestFactory.setBufferRequestBody(false); - restTemplate.setRequestFactory(requestFactory); - ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); - } - - if (response.getBody().containsKey("exception") && response.getBody().get("exception") != null) { - Object exception = response.getBody().get("exception"); - // Object path = response.getBody().get("path"); - Object message = response.getBody().get("message"); - // Object error = response.getBody().get("error"); - throw new KlabIOException("remote exception: " + (message == null ? exception : message)); - } - - return objectMapper.convertValue(response.getBody(), cls); - - } catch (RestClientException e) { - throw new KlabIOException(e); - } - - } - - public static String printAsJson(Object object) { - - ObjectMapper om = new ObjectMapper(); - om.enable(SerializationFeature.INDENT_OUTPUT); // pretty print - om.enable(SerializationFeature.WRITE_NULL_MAP_VALUES); // pretty print - om.enable(SerializationFeature.WRITE_SINGLE_ELEM_ARRAYS_UNWRAPPED); // pretty print - om.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); - - try { - return om.writeValueAsString(object); - } catch (Exception e) { - throw new IllegalArgumentException("serialization failed: " + e.getMessage()); - } - } - - @Override - public boolean put(String url, Object data) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - if (data != null) { - headers.setContentType(MediaType.APPLICATION_JSON); - } - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - - try { - - super.put(url, data); - return true; - - } catch (RestClientException e) { - System.out.println("REST exception: " + e.getMessage()); - dumpRequest(url, headers, data); - return false; - } - - } + public static final String KLAB_VERSION_HEADER = "KlabVersion"; + public static final String KLAB_USER_AGENT = "k.LAB/0.11.0"; + public static final String KLAB_CONNECTION_TIMEOUT = "klab.connection.timeout"; + + ObjectMapper objectMapper; + String authorizationToken; + String authenticationToken; + MediaType contentType = new MediaType(MediaType.APPLICATION_JSON, StandardCharsets.UTF_8); + RestTemplate basicTemplate; + private Set endpoints = new HashSet<>(); + + private static ClientHttpRequestFactory factory; + + public static Client createCustomTimeoutClient(int timeout) { + HttpComponentsClientHttpRequestFactory custom = new HttpComponentsClientHttpRequestFactory(); + custom.setReadTimeout(timeout); + custom.setConnectTimeout(timeout); + return new Client(custom); + } + + public static synchronized Client create() { + + if (factory == null) { + factory = new HttpComponentsClientHttpRequestFactory(); + if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { + int connectTimeout = 1000 + * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); + ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); + ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); + } + } + + return new Client(factory); + } + + public static Client createJson() { + Client ret = create(); + ret.setup(); + return ret; + } + + /** + * This one will return a client that interprets anything as JSON independent of + * content type. Required for misbehaving services that return JSON with other + * content types. + * + * @return + */ + public static synchronized Client createUniversalJSON() { + + if (factory == null) { + factory = new HttpComponentsClientHttpRequestFactory(); + if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { + int connectTimeout = 1000 + * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); + ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); + ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); + } + } + + return new UniversalClient(factory); + } + + public static class UniversalClient extends Client { + UniversalClient(ClientHttpRequestFactory factory) { + super(factory); + objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + List> messageConverters = new ArrayList>(); + MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + /* + * Make this converter process any kind of response, not only application/*json, + * which is the default behaviour + */ + converter.setSupportedMediaTypes(Collections.singletonList(MediaType.ALL)); + messageConverters.add(converter); + this.setMessageConverters(messageConverters); + } + } + + public static class NodeClient extends Client { + + public NodeClient(INodeIdentity node) { + + super(factory); + + objectMapper = new ObjectMapper(); + List> messageConverters = new ArrayList<>(); + MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); + StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); + FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); + ProtobufHttpMessageConverter protobufConverter = new ProtobufHttpMessageConverter(); + // ByteArrayHttpMessageConverter byteConverter = new + // ByteArrayHttpMessageConverter(); + jsonMessageConverter.setObjectMapper(objectMapper); + + setErrorHandler(new JSONResponseErrorHandler()); + messageConverters.add(jsonMessageConverter); + messageConverters.add(utf8); + messageConverters.add(formHttpMessageConverter); + messageConverters.add(protobufConverter); + // messageConverters.add(byteConverter); + setMessageConverters(messageConverters); + this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); + this.authorizationToken = node.getId(); + } + } + + /** + * Send an authentication request to a hub for an engine. + * + * @param url + * @param request + * @return the response. If not authenticated, throw a + * KlabAuthorizationException. If timeout, return null. + */ + public EngineAuthenticationResponse authenticateEngine(String url, EngineAuthenticationRequest request) { + return post(url + API.HUB.AUTHENTICATE_ENGINE, request, EngineAuthenticationResponse.class); + } + + /** + * Send an authentication request to a hub for a node. + * + * @param url + * @param request + * @return the response. If not authenticated, throw a + * KlabAuthorizationException. If timeout, return null. + */ + public NodeAuthenticationResponse authenticateNode(String url, NodeAuthenticationRequest request) { + return post(url + API.HUB.AUTHENTICATE_NODE, request, NodeAuthenticationResponse.class); + } + + /** + * Check an engine's heartbeat. + * + * @param url base engine/node URL + * @return true if alive + */ + public boolean ping(String url, String endpoint) { + try { + ResponseEntity response = basicTemplate.exchange(url + endpoint, HttpMethod.GET, + new HttpEntity(null, null), Object.class); + return response.getStatusCodeValue() == 200; + } catch (Throwable e) { + return false; + } + } + + public boolean ping(String url) { + return ping(url, "/actuator/health"); + } + + public boolean pingService(String url) { + return ping(url, "/public/status"); + } + + private class JSONResponseErrorHandler implements ResponseErrorHandler { + + @Override + public void handleError(ClientHttpResponse response) throws IOException { + } + + @Override + public boolean hasError(ClientHttpResponse response) throws IOException { + HttpStatus status = response.getStatusCode(); + HttpStatus.Series series = status.series(); + return (HttpStatus.Series.CLIENT_ERROR.equals(series) || HttpStatus.Series.SERVER_ERROR.equals(series)); + } + } + + /** + * Add option to set both headers, Authorization and Authentication + * + * @param headers the headers object to set the new headers + */ + private void setAuthTokens(HttpHeaders headers) { + if (authorizationToken != null) { + headers.set(HttpHeaders.AUTHORIZATION, authorizationToken); + } + if (authenticationToken != null) { + headers.add("Authentication", authenticationToken); + } + } + + /** + * Interceptor to add user agents and ensure that the authorization token gets + * in. + * + * @author ferdinando.villa + * + */ + public class AuthorizationInterceptor implements ClientHttpRequestInterceptor { + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) + throws IOException { + + HttpRequestWrapper requestWrapper = new HttpRequestWrapper(request); + HttpHeaders headers = requestWrapper.getHeaders(); + headers.set("Accept", "application/json"); + headers.set("X-User-Agent", "k.LAB " + Version.CURRENT); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + return execution.execute(requestWrapper, body); + } + } + + private void setup() { + + objectMapper = new ObjectMapper(); + List> messageConverters = new ArrayList<>(); + MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); + StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); + FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); + // ByteArrayHttpMessageConverter byteConverter = new + // ByteArrayHttpMessageConverter(); + jsonMessageConverter.setObjectMapper(objectMapper); + + setErrorHandler(new JSONResponseErrorHandler()); + messageConverters.add(jsonMessageConverter); + messageConverters.add(utf8); + messageConverters.add(formHttpMessageConverter); + // messageConverters.add(byteConverter); + setMessageConverters(messageConverters); + this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); + } + + private Client(ClientHttpRequestFactory factory) { + super(factory); + this.basicTemplate = new RestTemplate(); + setup(); + } + + private Client(Client client) { + super(factory); + this.basicTemplate = client.basicTemplate; + this.objectMapper = client.objectMapper; + this.endpoints = client.endpoints; + this.authorizationToken = client.authorizationToken; + this.authenticationToken = client.authenticationToken; + this.contentType = client.contentType; + setup(); + } + + private Client() { + super(factory); + } + + public Client withContentType(MediaType contentType) { + Client ret = new Client(this); + ret.contentType = contentType; + return ret; + } + + /** + * Return a client with authorization set to the passed object. + * + * @param authorizer any identity. + * @return a new + */ + @Override + public Client onBehalfOf(IIdentity authorizer) { + + Client ret = new Client(this); + ret.objectMapper = this.objectMapper; + ret.authorizationToken = authorizer.getId(); + return ret; + } + + public Client withAuthorization(String authorization) { + + Client ret = new Client(this); + ret.objectMapper = this.objectMapper; + ret.authorizationToken = authorization; + return ret; + } + + public Client withAuthentication(String authentication) { + + Client ret = new Client(); + ret.objectMapper = this.objectMapper; + ret.authenticationToken = authentication; + return ret; + } + + @SuppressWarnings("unchecked") + @Override + public T post(String url, Object data, Class cls) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + if (data != null) { + headers.setContentType(contentType); + } + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + + HttpEntity entity = new HttpEntity<>(data, headers); + + try { + + ResponseEntity response = exchange(url, HttpMethod.POST, entity, Object.class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); + case 500: + throw new KlabResourceAccessException("internal: request " + url + " caused a remote server error"); + } + + if (response.getBody() == null) { + return null; + } + if (response.getBody() instanceof Map && ((Map) response.getBody()).containsKey("exception") + && ((Map) response.getBody()).get("exception") != null) { + + Map map = (Map) response.getBody(); + Object exception = map.get("exception"); + // Object path = map.get("path"); + Object message = map.get("message"); + // Object error = map.get("error"); + + dumpRequest(url, headers, data); + throw new KlabIOException("remote exception: " + (message == null ? exception : message)); + } + + if (cls.isAssignableFrom(response.getBody().getClass())) { + return (T) response.getBody(); + } + + try { + return objectMapper.convertValue(response.getBody(), cls); + } catch (Throwable t) { + System.out.println("Unrecognized response: " + response.getBody()); + throw t; + } + } catch (RestClientException e) { + + System.out.println("REST exception: " + e.getMessage()); + dumpRequest(url, headers, data); + throw new KlabIOException(e); + } + } + + private void dumpRequest(String url, HttpHeaders headers, Object data) { + + System.out.println("Endpoint: " + url); + System.out.println("Headers:"); + for (String header : headers.keySet()) { + System.out.println(" " + header + " = " + headers.get(header)); + } + System.out.println("Data:\n" + printAsJson(data)); + + } + + public void setUrl(String... url) { + if (url == null || url.length == 0) { + this.endpoints.clear(); + } else { + for (String u : url) { + this.endpoints.add(u); + } + } + } + + private String pickEndpoint() { + // TODO periodically check URLs and choose the first that responds (or the one + // with the smallest load) + return this.endpoints.isEmpty() ? null : this.endpoints.iterator().next(); + } + + @Override + public boolean getDownload(String url, File output) { + + url = checkEndpoint(url); + + RestTemplate restTemplate = new RestTemplate(); + restTemplate.getMessageConverters().add(new ByteArrayHttpMessageConverter()); + + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + + HttpEntity entity = new HttpEntity(headers); + + // HttpHeaders headers = new HttpHeaders(); + // headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); + // HttpEntity entity = new HttpEntity<>(headers); + + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, entity, byte[].class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); + } + + if (response.getBody() == null) { + return false; + } + if (response.getStatusCode() == HttpStatus.OK) { + try { + Files.write(output.toPath(), response.getBody()); + } catch (IOException e) { + throw new KlabIOException(e); + } + } + + return true; + } + + private String checkEndpoint(String url) { + if (!url.toLowerCase().startsWith("http")) { + String ep = pickEndpoint(); + if (ep != null) { + url = ep + (ep.endsWith("/") || url.startsWith("/") ? "" : "/") + url; + } + } + return url; + } + + /** + * Issue a DELETE request. Called remove to avoid conflict with super. + * + * @param url + * @param parameters + * @return + */ + public Object remove(String url, Object... parameters) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + HttpEntity entity = new HttpEntity<>(headers); + + if (parameters != null) { + String params = ""; + for (int i = 0; i < parameters.length; i++) { + String key = parameters[i].toString(); + String nakedKey = key; + String val = parameters[++i].toString(); + if (!(key.startsWith("{") && key.endsWith("}"))) { + key = "{" + key + "}"; + } else { + nakedKey = key.substring(1, key.length() - 1); + } + if (url.contains(key)) { + url = url.replace(key, val); + } else { + params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + Escape.forURL(val); + } + } + if (!params.isEmpty()) { + url += "?" + params; + } + } + + ResponseEntity response = exchange(url, HttpMethod.DELETE, entity, Object.class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); + case 503: + throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); + } + + return response.getBody(); + } + + /** + * Instrumented for header communication and error parsing + * + * @param url + * @param cls + * @return the deserialized result + */ + @Override + @SuppressWarnings({ "unchecked" }) + public T get(String url, Class cls, Object... parameters) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", cls.equals(String.class) ? "text/plain" : "application/json"); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + // add user-agent to avoid 403 from Nominatim + headers.set(HttpHeaders.USER_AGENT, KLAB_USER_AGENT); + setAuthTokens(headers); + HttpEntity entity = new HttpEntity<>(headers); + + if (parameters != null) { + String params = ""; + for (int i = 0; i < parameters.length; i++) { + String key = parameters[i].toString(); + String nakedKey = key; + Object nextPar = parameters[++i]; + String val = nextPar == null ? null : nextPar.toString(); + + if (val == null) { + continue; + } + + if (!(key.startsWith("{") && key.endsWith("}"))) { + key = "{" + key + "}"; + } else { + nakedKey = key.substring(1, key.length() - 1); + } + if (url.contains(key)) { + url = url.replace(key, val); + } else { + params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + /* Escape.forURL( */val/* ) */; + } + } + if (!params.isEmpty()) { + url += "?" + params; + } + } + + ResponseEntity response = null; + if (cls.isArray()) { + response = exchange(url, HttpMethod.GET, entity, Object.class); + } else if (String.class.equals(cls)) { + response = basicTemplate.exchange(url, HttpMethod.GET, entity, String.class); + } else /* if (Map.class.isAssignableFrom(cls)) */ { + response = exchange(url, HttpMethod.GET, entity, Map.class); + } + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); + case 406: + case 503: + throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); + } + + if (response.getBody() instanceof Map) { + + Object exception = ((Map) response.getBody()).get("exception"); + Object error = ((Map) response.getBody()).get("error"); + + if (exception != null || error != null) { + Object message = ((Map) response.getBody()).get("message"); + // Object error = response.getBody().get("error"); + throw new KlabIOException( + "remote exception: " + (message == null ? (exception == null ? error : exception) : message)); + } + + return objectMapper.convertValue(response.getBody(), cls); + + } else if (response.getBody() instanceof List && cls.isArray()) { + + List list = (List) response.getBody(); + Object ret = Array.newInstance(cls.getComponentType(), (((List) response.getBody()).size())); + for (int i = 0; i < list.size(); i++) { + Object object = list.get(i); + if (object instanceof Map) { + object = objectMapper.convertValue(object, cls.getComponentType()); + } + Array.set(ret, i, object); + } + + return (T) ret; + + } else if (response.getBody() != null && cls.isAssignableFrom(response.getBody().getClass())) { + return (T) response.getBody(); + } + + return null; + } + + @Override + public T postFile(String url, File file, Class cls) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", new FileSystemResource(file)); + HttpEntity> entity = new HttpEntity<>(body, headers); + + try { + + final RestTemplate restTemplate = new RestTemplate(); + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + // without this, large files will eat up the heap + requestFactory.setBufferRequestBody(false); + restTemplate.setRequestFactory(requestFactory); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); + } + + if (response.getBody().containsKey("exception") && response.getBody().get("exception") != null) { + Object exception = response.getBody().get("exception"); + // Object path = response.getBody().get("path"); + Object message = response.getBody().get("message"); + // Object error = response.getBody().get("error"); + throw new KlabIOException("remote exception: " + (message == null ? exception : message)); + } + + return objectMapper.convertValue(response.getBody(), cls); + + } catch (RestClientException e) { + throw new KlabIOException(e); + } + + } + + public static String printAsJson(Object object) { + + ObjectMapper om = new ObjectMapper(); + om.enable(SerializationFeature.INDENT_OUTPUT); // pretty print + om.enable(SerializationFeature.WRITE_NULL_MAP_VALUES); // pretty print + om.enable(SerializationFeature.WRITE_SINGLE_ELEM_ARRAYS_UNWRAPPED); // pretty print + om.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); + + try { + return om.writeValueAsString(object); + } catch (Exception e) { + throw new IllegalArgumentException("serialization failed: " + e.getMessage()); + } + } + + @Override + public boolean put(String url, Object data) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + if (data != null) { + headers.setContentType(MediaType.APPLICATION_JSON); + } + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + + try { + + super.put(url, data); + return true; + + } catch (RestClientException e) { + System.out.println("REST exception: " + e.getMessage()); + dumpRequest(url, headers, data); + return false; + } + + } }