diff --git a/adapters/klab.adapter.datacube/pom.xml b/adapters/klab.adapter.datacube/pom.xml index e02c10f85c..1f460d015e 100644 --- a/adapters/klab.adapter.datacube/pom.xml +++ b/adapters/klab.adapter.datacube/pom.xml @@ -1,6 +1,6 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 org.integratedmodelling diff --git a/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java b/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java index 835ede208d..7e0a232ea6 100644 --- a/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java +++ b/adapters/klab.adapter.datacube/src/main/java/org/integratedmodelling/adapter/datacube/ChunkedDatacubeRepository.java @@ -24,10 +24,10 @@ import java.util.logging.Level; import javax.annotation.Nullable; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; import org.apache.commons.io.FileUtils; +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; import org.geotools.coverage.grid.GridCoverage2D; import org.integratedmodelling.adapter.datacube.api.IDatacube; import org.integratedmodelling.klab.Configuration; @@ -40,8 +40,6 @@ import org.integratedmodelling.klab.api.data.IResource.Availability; import org.integratedmodelling.klab.api.data.adapters.IKlabData.Builder; import org.integratedmodelling.klab.api.data.mediation.IUnit; -import org.integratedmodelling.klab.api.observations.IState; -import org.integratedmodelling.klab.api.observations.scale.IExtent; import org.integratedmodelling.klab.api.observations.scale.IScale; import org.integratedmodelling.klab.api.observations.scale.space.IGrid; import org.integratedmodelling.klab.api.observations.scale.space.ISpace; diff --git a/adapters/klab.ogc/pom.xml b/adapters/klab.ogc/pom.xml index 08cd69ca5e..1c04bee848 100644 --- a/adapters/klab.ogc/pom.xml +++ b/adapters/klab.ogc/pom.xml @@ -47,6 +47,22 @@ 1.68 + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + + + + com.fasterxml.jackson.core + jackson-annotations + + org.hortonmachine hm-gears @@ -56,10 +72,15 @@ javax.measure jsr-275 + org.mongodb mongo-java-driver + + edu.colostate.omslab + oms + @@ -68,7 +89,6 @@ aws-lightweight-client-java ${s3.version} - Filter later :) - if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { - time = validateTemporalDimension(time, resourceTime); - } - ITimeInstant start = time.getStart(); - ITimeInstant end = time.getEnd(); - collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); + GridCoverage2D coverage = null; + + collection.setBboxFilter(new double[] { + bbox.get(0), + bbox.get(2), + bbox.get(1), + bbox.get(3) + }); - GridCoverage2D coverage = null; - try { + // Allow transform ensures the process to finish, but I would not bet on the resulting + // data + if (assetPredicate == null) { + // NO JSONSelector and JSONValue found, NO assetID was passed as well + scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; + try { + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), + effectiveTime.getEnd()); + } catch (Exception e) { + manager.close(); + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + } + encoder = new VectorEncoder(); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + manager.close(); + return; + } + List items = collection.searchItems(); - if (items.isEmpty()) { manager.close(); - throw new KlabIllegalStateException("No STAC items found for this context."); + throw new KlabIllegalStateException("No STAC items found for this context, check the Spatial/ Temporal bounds of items and the Context"); } - scope.getMonitor().debug("Found " + items.size() + " STAC items."); if (mergeMode == HMRaster.MergeMode.SUBSTITUTE) { sortByDate(items, scope.getMonitor()); @@ -374,62 +482,178 @@ public void getEncodedData(IResource resource, Map urnParameters IGrid grid = space.getGrid(); RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); - Set EPSGsAtItems = items.stream().map(i -> i.getEpsg()).collect(Collectors.toUnmodifiableSet()); - if (EPSGsAtItems.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the items " + EPSGsAtItems.toString() + ". The transformation process could affect the data."); - } - if (resource.getParameters().contains("awsRegion")) { - String bucketRegion = resource.getParameters().get("awsRegion", String.class); - Client s3Client = buildS3Client(bucketRegion); + if (resource.getParameters().contains("s3EndpointUrl")) { + String s3EndpointURL = resource.getParameters().get("s3EndpointUrl", String.class); + Client s3Client = buildS3Client(s3EndpointURL); collection.setS3Client(s3Client); } + var time = effectiveTime; + // Filter here based on time, since in some STAC collections they don't yet support + // temporal filtering :( like ECDC + items = items.stream() + .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) + .collect(Collectors.toList()); + + if (items.size() == 0) { + manager.close(); + throw new KlabIllegalStateException( + "No STAC items found covering the entire time duration of the context requested"); + } else { + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + } - // Allow transform ensures the process to finish, but I would not bet on the resulting - // data. - final boolean allowTransform = true; - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetId, items, allowTransform, mergeMode, lpm); - coverage = outRaster.buildCoverage(); + // Once the support for customized predicate is added, we can apply for features as well + + var pred = assetPredicate; + Set EPSGAtAssets = items.stream() + .flatMap(item -> item.getAssets().stream().filter(pred).findFirst() + .map(asset -> asset.getEpsg() != null ? asset.getEpsg() : item.getEpsg()).stream()) + .collect(Collectors.toUnmodifiableSet()); + + if (EPSGAtAssets.size() > 1) { + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." + + "The transformation process could affect the data."); + } + + + // Specific Implementation for the Slow Requests flow in WEED + if (collection.getId().contains("EU_modelV2-1-MECE") + && resource.getUrn().contains("im.resources-main")) { + Geometry unionMLStacInference = UnaryUnionOp.union( + items.stream() + .map(item -> item.getGeometry()) + .filter(g -> g != null && !g.isEmpty()) + .collect(Collectors.toList()) + ); + + if (!unionMLStacInference.contains(poly)) { + scope.getMonitor().warn("The requested extend for ML inferences is not completely contained in STAC, Starting ML Inference Request"); + + OpenEO service = OpenEOAdapter.getClient("openeo_weed.dataspace.copernicus.eu"); + List processes = new ArrayList<>(); + String processNamespace = "https://raw.githubusercontent.com/ESA-WEED-project/OpenEO-UDP-UDF-catalogue/refs/heads/main/UDP/json/udp_starter.json"; + String processID = "udp_starter"; + + Process process = JsonUtils.load(new URL(processNamespace), + Process.class); + process.encodeSelf(processNamespace); + processes.add(process); + + JSONObject arguments = new JSONObject() + .put("bbox", new JSONObject() + .put("crs", 4326) + .put("west", bbox.get(0)) + .put("south", bbox.get(3)) + .put("east", bbox.get(1)) + .put("north", bbox.get(2))) + .put("digitalId", "AM1729") // Forms the STAC coordinate later + .put("scenarioId", "DT_SLOW_FLOW") // Forms the STAC coordinate later + .put("year", ctxTime.getEnd().getYear()) + .put("onnx_model", "EUNIS2021plus_panEU_v201_2024_OneZone") // Hardcoding for now only for Europe, until the "BEST" model is decided! + .put("dt_url", "https://services.integratedmodelling.org/runtime/main/api/v1/dt/ESA_INSTITUTIONAL.3vh554o6h6c"); + + + OpenEOFuture job = service.submit(processID, arguments, + scope.getMonitor(), processes.toArray(new Process[processes.size()])); + + if (job.isCancelled()) { + scope.getMonitor().warn("job canceled"); + } else if (job.getError() != null) { + scope.getMonitor().error(job.getError()); + } else { + scope.getMonitor().info("Inference Request has been submitted to the ML Workflows"); + } + } + } + + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, + MergeMode.SUBSTITUTE, lpm); + if (outRaster == null) { + scope.getMonitor().error("Unable to build the output from the STAC Resource"); + throw new KlabIllegalStateException("Unable to build the output from the STAC Resource"); + } + CoordinateReferenceSystem targetCRS = HMCrsRegistry.INSTANCE.getCrs("4326"); + if (!HMCrsRegistry.crsEquals(outRaster.getCrs(),targetCRS)) { + var transformer = new HMCrsTransformer(outRaster.getCrs(), targetCRS); + transformer.setAcceptLenientDatumShift(true); + outRaster = transformer.transform(outRaster); + } + + + HMRaster paddedRaster = new HMRasterWritableBuilder().setName("padded").setRegion(region) + .setCrs(targetCRS).setNoValue(outRaster.getNovalue()).build(); + paddedRaster.mapRaster(null, outRaster, null); + coverage = paddedRaster.buildCoverage(); + + if (bandIndex != null) { // Which means theat it's a Multi Band COG + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + } + manager.close(); + encoder = new RasterEncoder(); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { + e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } - encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } - private boolean isFeatureInTimeRange(Time time2, SimpleFeature f) { - Date datetime = (Date) f.getAttribute("datetime"); - if (datetime != null) { - if (isDateWithinRange(time2, datetime)) { - return true; + private Predicate getAssetPredicateFromJSONSelector(IResource resource) { + String jsonSelector = resource.getParameters().get("jsonSelector", String.class); + String jsonValue = resource.getParameters().get("jsonValue", String.class); + if (jsonSelector != null && !jsonSelector.isBlank() && jsonValue != null) { + try { + return STACPathExpression.STACAssetPredicate.fromHMStacAsset(jsonSelector, jsonValue); + } catch (IllegalArgumentException e) { + throw new KlabIllegalArgumentException("Invalid STAC asset JSON selector: " + jsonSelector); } + } else { + throw new KlabIllegalArgumentException("Either asset or both jsonSelector and jsonValue must be provided"); } + } - Date itemStart = (Date) f.getAttribute("start_datetime"); - if (itemStart == null) { - return false; - } - Date itemEnd = (Date) f.getAttribute("end_datetime"); - if (itemEnd == null) { - return itemStart.toInstant().getEpochSecond() <= time2.getStart().getMilliseconds(); + /* + To check if an Item (of type HMStacItem) is within a time range + */ + private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String startTimestamp = item.getStartTimestamp(); + String endTimestamp = item.getEndTimestamp(); + + if (startTimestamp == null || endTimestamp == null) { + return true; // Assume the time part is ok } - if (isDateWithinRange(time2, itemStart) || isDateWithinRange(time2, itemEnd)) { - return true; + + try { + + long itemStart = LocalDateTime.parse(item.getStartTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + return (startMillis >= itemStart || Math.abs(startMillis - itemStart) < 10000) + && (endMillis <= itemEnd || Math.abs(endMillis - itemEnd) < 10000); // Allow some small tolerance + + } catch (Exception e) { + e.printStackTrace(); + return false; } - return false; } - private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, String collectionId) { - List links = collectionData.getJSONArray("links").toList().stream().filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("item")).toList(); - List urlOfLinks = links.stream().map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); + private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, + String collectionId) { + List links = collectionData.getJSONArray("links").toList().stream() + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("item")).toList(); + List urlOfLinks = links.stream() + .map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); return urlOfLinks.stream().map(i -> { try { return STACUtils.getItemAsFeature(i); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index dc768f5093..1d640dd5a3 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -9,10 +9,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import org.integratedmodelling.kim.api.IParameters; -import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Resources; import org.integratedmodelling.klab.api.data.ILocator; import org.integratedmodelling.klab.api.data.IResource; @@ -44,22 +42,20 @@ public boolean acceptsMultiple() { return false; } + private boolean hasAssetSelector(IParameters parameters) { + return (parameters.contains("asset") || (parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); + } + private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) - throws MalformedURLException { + throws Exception { String collectionUrl = parameters.get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = STACCollectionParser.readCollectionId(collectionData); parameters.put("collectionId", collectionId); - String regex = null; - if (parameters.contains("regex")) { - regex = parameters.get(Resources.REGEX_ENTRY, String.class); - parameters.remove(Resources.REGEX_ENTRY); - } - boolean isBulkImport = parameters.contains("bulkImport"); parameters.remove("bulkImport"); - if (!parameters.contains("asset") && !isBulkImport) { + if (!hasAssetSelector(parameters) && !isBulkImport) { Builder builder = buildResource(parameters, project, monitor, collectionId); if (builder != null) { ret.add(builder); @@ -68,19 +64,86 @@ private void importCollection(List ret, IParameters parameters, } return; } - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + + String assetId = parameters.get("asset", String.class); + String resourceUrn = collectionId; + JSONObject assetData = null; + /* + This is only for the part, to check the Asset is under an S3 Bucket + */ + if (assetId != null) { + JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + //assetData = assetNode.getJSONObject(assetId); + assetData = (JSONObject) assetNode.toMap() + .values() + .iterator() + .next(); + /* + * If the particular asset Id wasn't found, then + * still proceed to create the resource since it can happen + * for a number of reasons. Pagination, STAC Backend etc. + * In that case however, a better handling of S3 URL needs to figured out + */ + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + throw new Exception("Unsupported media type for the asset"); + } + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + } + resourceUrn = collectionId + "-" + assetId; + } + + //TODO: Check if assetId is null, and instead a jsonSelector and Value is passed! + //else if (parameters.get("jsonSelector", String.class) != null) { + // String selector = parameters.get("jsonSelector", String.class); + // String val = parameters.get("jsonValue", String.class); + // assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, null, STACPathExpression.STACAssetPredicate.fromKongJsonObject(selector, val)); + //} + + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + + Builder builder = buildResource(parameters, project, monitor, resourceUrn); + if (builder != null) { + ret.add(builder); + } else { + monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); + } + + // Think of a way to do the REGEX matching. Possibly not worth it to support + // this for something as diverse as STAC + + // String regex = null; + if (parameters.contains("regex")) { + // regex = parameters.get(Resources.REGEX_ENTRY, String.class); + parameters.remove(Resources.REGEX_ENTRY); + } + + /* Set assetIds = STACAssetMapParser.readAssetNames(assets); for(String assetId : assetIds) { if (regex != null && !assetId.matches(regex)) { Logging.INSTANCE.info("Asset " + assetId + " doesn't match REGEX, skipped"); continue; } - + JSONObject assetData = STACAssetMapParser.getAsset(assets, assetId); - if (!STACAssetParser.isSupportedMediaType(assetData)) { - Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); - continue; + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); + continue; + } } + parameters.put("asset", assetId); String resourceUrn = collectionId + "-" + assetId; String href = assetData.getString("href"); @@ -89,7 +152,7 @@ private void importCollection(List ret, IParameters parameters, String s3Region = "unknown"; // TODO resolve the region parameters.put("awsRegion", s3Region); } - + Builder builder = buildResource(parameters, project, monitor, resourceUrn); if (builder != null) { ret.add(builder); @@ -97,12 +160,14 @@ private void importCollection(List ret, IParameters parameters, monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); } } + + */ } - private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException { - Builder builder = validator.validate( - Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), new URL(parameters.get("collection", String.class)), - parameters, monitor); + private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) + throws MalformedURLException { + Builder builder = validator.validate(Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), + new URL(parameters.get("collection", String.class)), parameters, monitor); if (builder == null) { return null; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java new file mode 100644 index 0000000000..bc4f73bd67 --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -0,0 +1,414 @@ +package org.integratedmodelling.klab.stac; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.TextNode; + +import kong.unirest.json.JSONObject; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; + +public final class STACPathExpression { + + private final List path; + + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands.name"; + public static final String MEDIA_TYPE_BANDS_NAME = "type"; + + private STACPathExpression(List path) { + this.path = path; + } + + public static STACPathExpression parse(String jsonPath) { + if (jsonPath == null || jsonPath.isBlank()) { + throw new IllegalArgumentException("JSON path cannot be empty"); + } + return new STACPathExpression(parsePath(jsonPath)); + } + + public boolean matches(JsonNode root, String expectedValue) { + List resolvedNodes = resolve(root); + + for(JsonNode node : resolvedNodes) { + if (jsonValueEquals(node, expectedValue)) { + return true; + } + } + + return false; + } + + public List resolve(JsonNode root) { + List currentNodes = new ArrayList<>(); + currentNodes.add(root); + + for(PathPart part : path) { + List nextNodes = new ArrayList<>(); + + for(JsonNode current : currentNodes) { + if (current == null || current.isNull() || current.isMissingNode()) { + continue; + } + + resolvePart(current, part, nextNodes); + } + + currentNodes = nextNodes; + + if (currentNodes.isEmpty()) { + break; + } + } + + return currentNodes; + } + + private static void resolvePart(JsonNode current, PathPart part, List nextNodes) { + if (current.isArray()) { + resolveFromArray(current, part, nextNodes); + } else if (current.isObject()) { + resolveFromObject(current, part, nextNodes); + } + } + + private static void resolveFromArray(JsonNode arrayNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + JsonNode indexedNode = arrayNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + resolvePart(indexedNode, new PathPart(part.fieldName(), ArrayMode.NONE, null), nextNodes); + } + + } else { + for(JsonNode element : arrayNode) { + if (isUsable(element)) { + resolvePart(element, part, nextNodes); + } + } + } + } + + private static void resolveFromObject(JsonNode objectNode, PathPart part, List nextNodes) { + JsonNode directField = objectNode.get(part.fieldName()); + + if (isUsable(directField)) { + addResolvedField(directField, part, nextNodes); + } else { + objectNode.fields().forEachRemaining(entry -> { + String mapKey = entry.getKey(); + JsonNode mapValue = entry.getValue(); + + if (!isUsable(mapValue) || !mapValue.isObject()) { + return; + } + + if ("id".equalsIgnoreCase(part.fieldName())) { + nextNodes.add(TextNode.valueOf(mapKey)); + return; + } + + JsonNode nestedField = mapValue.get(part.fieldName()); + + if (isUsable(nestedField)) { + addResolvedField(nestedField, part, nextNodes); + } + }); + + } + } + + private static void addResolvedField(JsonNode fieldNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + if (fieldNode.isArray()) { + JsonNode indexedNode = fieldNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + nextNodes.add(indexedNode); + } + } + } else if (fieldNode.isArray()) { + for(JsonNode element : fieldNode) { + if (isUsable(element)) { + nextNodes.add(element); + } + } + } else { + nextNodes.add(fieldNode); + } + } + + private static boolean isUsable(JsonNode node) { + return node != null && !node.isNull() && !node.isMissingNode(); + } + + private static List parsePath(String jsonPath) { + String[] tokens = jsonPath.split("\\."); + + List parts = new ArrayList<>(); + + for(String token : tokens) { + String trimmed = token.trim(); + + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("Invalid empty path element"); + } + + parts.add(PathPart.parse(trimmed)); + } + + return parts; + } + + private static boolean valueEquals(Object actualValue, String expectedValue) { + if (actualValue == null) { + return expectedValue == null; + } + + if (expectedValue == null) { + return false; + } + + if (actualValue instanceof Number number) { + return numberEquals(number, expectedValue); + } + + if (actualValue instanceof Boolean bool) { + return Boolean.toString(bool).equalsIgnoreCase(expectedValue); + } + + return Objects.equals(String.valueOf(actualValue), expectedValue); + } + + private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { + if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { + return expectedValue == null; + } + + if (expectedValue == null || !actualValue.isValueNode()) { + return false; + } + + if (actualValue.isNumber()) { + return numberEquals(actualValue.decimalValue(), expectedValue); + } + + if (actualValue.isBoolean()) { + return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); + } + + if (actualValue.isTextual()) { + return Objects.equals(actualValue.asText(), expectedValue); + } + + return false; + } + + private static boolean numberEquals(Number actualValue, String expectedValue) { + try { + BigDecimal actual = new BigDecimal(actualValue.toString()); + BigDecimal expected = new BigDecimal(expectedValue); + + return actual.compareTo(expected) == 0; + } catch (NumberFormatException e) { + return false; + } + } + + public record PathPart(String fieldName, ArrayMode arrayMode, Integer arrayIndex) { + + public static PathPart parse(String token) { + int bracketStart = token.indexOf('['); + + if (bracketStart < 0) { + return new PathPart(token, ArrayMode.NONE, null); + } + + int bracketEnd = token.indexOf(']', bracketStart); + + if (bracketEnd < 0) { + throw new IllegalArgumentException("Invalid array syntax in path element: " + token); + } + + if (bracketEnd != token.length() - 1) { + throw new IllegalArgumentException("Unexpected characters after array syntax in path element: " + token); + } + + String fieldName = token.substring(0, bracketStart).trim(); + String indexText = token.substring(bracketStart + 1, bracketEnd).trim(); + + if (fieldName.isEmpty()) { + throw new IllegalArgumentException("Field name cannot be empty in path element: " + token); + } + + int index; + try { + index = Integer.parseInt(indexText); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid array index in path element: " + token, e); + } + if (index < 0) { + throw new IllegalArgumentException("Array index cannot be negative in path element: " + token); + } + return new PathPart(fieldName, ArrayMode.INDEX, index); + } + } + + public enum ArrayMode { + NONE, INDEX + } + + public static final class STACAssetPredicate { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private STACAssetPredicate() { + } + + public static Predicate fromJsonPath(String jsonPath, String expectedValue, + Function jsonNodeExtractor) { + STACPathExpression expression = STACPathExpression.parse(jsonPath); + + return object -> { + if (object == null) { + return false; + } + + JsonNode node = jsonNodeExtractor.apply(object); + + if (node == null || node.isNull() || node.isMissingNode()) { + return false; + } + + /* + * Check if {"a.b": expectedValue} exists and also {"a": { "b": expectedValue }} + */ + + JsonNode directValue = node.get(jsonPath); + if (directValue != null && !directValue.isNull()) { + return expectedValue.equals(directValue.asText()); + } + + return expression.matches(node, expectedValue); + }; + } + + public static Predicate fromHMStacAsset(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, asset -> asset == null ? null : asset.getAssetNode()); + } + + public static Predicate fromJsonNode(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, node -> node); + } + + public static Predicate fromKongJsonObject(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, STACAssetPredicate::toJsonNode); + } + + private static JsonNode toJsonNode(JSONObject jsonObject) { + if (jsonObject == null) { + return null; + } + + try { + return OBJECT_MAPPER.readTree(jsonObject.toString()); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Cannot convert JSONObject to JsonNode", e); + } + } + + public static Predicate fromHMStacAssetAttribute(String attributeName, String expectedValue) { + AssetAttribute attribute = AssetAttribute.fromName(attributeName); + + return asset -> { + if (asset == null) { + return false; + } + + Object actualValue = attribute.read(asset); + + return valueEquals(actualValue, expectedValue); + }; + } + + public static Predicate fromHMStacAssetId(String expectedValue) { + return fromHMStacAssetAttribute("id", expectedValue); + } + } + + public enum AssetAttribute { + + ID("id") { + @Override + Object read(HMStacAsset asset) { + return asset.getId(); + } + }, + + TITLE("title") { + @Override + Object read(HMStacAsset asset) { + return asset.getTitle(); + } + }, + + TYPE("type") { + @Override + Object read(HMStacAsset asset) { + return asset.getType(); + } + }, + + VALID("valid") { + @Override + Object read(HMStacAsset asset) { + return asset.isValid(); + } + }, + + EPSG("epsg") { + @Override + Object read(HMStacAsset asset) { + return asset.getEpsg(); + } + }, + + NON_VALID_REASON("nonValidReason") { + @Override + Object read(HMStacAsset asset) { + return asset.getNonValidReason(); + } + }; + + private final String name; + + AssetAttribute(String name) { + this.name = name; + } + + abstract Object read(HMStacAsset asset); + + public static AssetAttribute fromName(String name) { + if (name == null || name.isBlank()) { + throw new IllegalArgumentException("Asset attribute name cannot be empty"); + } + + for(AssetAttribute attribute : values()) { + if (attribute.name.equalsIgnoreCase(name.trim())) { + return attribute; + } + } + + throw new IllegalArgumentException("Unsupported HMStacAsset attribute: " + name + + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason"); + } + } + +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java index f0856fa005..0239e6f73f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java @@ -11,7 +11,7 @@ import org.integratedmodelling.klab.api.provenance.IArtifact.Type; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; import org.integratedmodelling.klab.utils.DOIReader; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeature; import com.fasterxml.jackson.core.JsonParseException; @@ -32,13 +32,14 @@ public static String readKeywords(JSONObject json) { return null; } List keywords = json.getJSONArray("keywords").toList(); - return keywords.isEmpty() ? null : - keywords.stream().collect(Collectors.joining(",")); + return keywords.isEmpty() ? null : keywords.stream().collect(Collectors.joining(",")); } - final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", "properties.sci:doi", "item_assets.sci:doi"); + final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", + "properties.sci:doi", "item_assets.sci:doi"); public static String readDOI(JSONObject json) { - Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)).findFirst(); + Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)) + .findFirst(); return doi.isPresent() ? doi.get() : null; } @@ -61,13 +62,13 @@ public static String readDOIAuthors(String doi) { */ public static boolean containsLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .anyMatch(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)); + .anyMatch(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)); } public static Optional getLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)) - .map(link -> ((JSONObject)link).getString("href")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)) + .map(link -> ((JSONObject) link).getString("href")).findFirst(); } public static JSONObject requestMetadata(String collectionUrl, String type) { @@ -87,7 +88,7 @@ public static String readLicense(JSONObject collection) { return null; } JSONArray links = collection.getJSONArray("links"); - for (int i = 0; i < links.length(); i++) { + for(int i = 0; i < links.length(); i++) { JSONObject link = links.getJSONObject(i); if (!link.has("rel") || !link.getString("rel").equals("license")) { continue; @@ -121,11 +122,12 @@ public static Type inferValueType(String key) { public static String getCatalogUrl(String collectionUrl, String collectionId, JSONObject collectionData) { // The URL of the catalog is the root if (!collectionData.has("links")) { - throw new KlabResourceAccessException("STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); + throw new KlabResourceAccessException( + "STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); } JSONArray links = collectionData.getJSONArray("links"); Optional rootLink = links.toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("root")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("root")).findFirst(); if (rootLink.isEmpty()) { throw new KlabResourceAccessException("STAC collection is missing a relationship to the root catalog"); } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index a01646ac05..3e3f0d6e66 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import org.integratedmodelling.kim.api.IParameters; import org.integratedmodelling.klab.api.data.IGeometry; @@ -20,11 +21,13 @@ import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor; import org.integratedmodelling.klab.data.resources.Resource; import org.integratedmodelling.klab.data.resources.ResourceBuilder; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnimplementedException; import org.integratedmodelling.klab.rest.CodelistReference; import org.integratedmodelling.klab.rest.MappingReference; import org.integratedmodelling.klab.rest.ResourceCRUDRequest; import org.integratedmodelling.klab.utils.Pair; +import org.integratedmodelling.klab.utils.s3.S3URLUtils; import kong.unirest.json.JSONObject; @@ -43,55 +46,122 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni String collectionUrl = userData.get("collection", String.class); String collectionId = userData.get("collectionId", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); - if (collectionId == null) { + if (collectionId == null) { collectionId = collectionData.getString("id"); userData.put("collectionId", collectionId); } - IGeometry geometry = STACCollectionParser.readGeometry(collectionData); - Builder builder = new ResourceBuilder(urn) - .withParameters(userData) - .withGeometry(geometry) - .withType(Type.OBJECT); + IGeometry geometry = null; + if (!userData.contains("asset") && !userData.contains("jsonSelector")) { + geometry = STACCollectionParser.readGeometry(collectionData, true); + } else { + geometry = STACCollectionParser.readGeometry(collectionData, false); + } + Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); - // The default URL of the resource is the collection endpoint. May be overwritten. + // The default URL of the resource is the collection endpoint. May be overwritten. builder.withMetadata(IMetadata.DC_URL, collectionUrl); + JSONObject assetNode; + if (userData.contains("asset")) { - String assetId = userData.get("asset", String.class); - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); - JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); - } - builder.addCodeList(codelist); + if(userData.contains("cog")) { + throw new KlabIllegalArgumentException("STAC asset and cog URL both shouldn't be provided while importing"); + } + String requestedAssetId = userData.get("asset", String.class); + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); + + } else if (userData.contains("jsonSelector")) { + if (!userData.contains("jsonValue")) { + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); } - if (type != null) { - builder.withType(type); + + if(userData.contains("cog")) { + throw new KlabIllegalArgumentException("jsonSelector and jsonValue shouldn't be provided along with cog URL while importing"); + } + + Predicate predicate = STACPathExpression.STACAssetPredicate + .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); + + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, predicate); + + } else { + // Just import Features + monitor.info("import STAC Collection for Features"); + readMetadata(collectionData, builder); + return builder; + } + + String assetId = assetNode.keys().next(); + JSONObject asset = assetNode.getJSONObject(assetId); + String href = asset.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + if (href.contains("waw3")) { + userData.put("s3EndpointUrl", "https://s3.waw3-1.cloudferro.com"); + } else if (href.contains("waw4")) { + userData.put("s3EndpointUrl", "https://s3.waw4-1.cloudferro.com"); + } else if (collectionUrl.contains("https://stac.dataspace.copernicus.eu/")){ + userData.put("s3EndpointUrl", "https://eodata.dataspace.copernicus.eu/"); + } else { + userData.put("s3EndpointUrl", "unknown"); + } + + } + + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); } - + generateCodeList(builder, assetId, asset); + if (userData.contains("cog")) { - if (userData.get("cog") != null) { - builder.withType(Type.NUMBER); - } + if (userData.get("cog") != null) { + builder.withType(Type.NUMBER); + } } readMetadata(collectionData, builder); return builder; } + private void generateCodeList(Builder builder, String assetId, JSONObject asset) { + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); + } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); + } + } + private Type readRasterDataType(JSONObject asset) { + + if (asset.has("type")) { + if(asset.get("type").toString().contains("image/tiff")) { // Numbers + return Type.NUMBER; + } + } if (!asset.has("raster:bands")) { return null; } - + if (asset.getJSONArray("raster:bands").isEmpty() || !asset.getJSONArray("raster:bands").getJSONObject(0).has("data_type")) { // We assume that most rasters are numeric. When in doubt, we set the default to Number @@ -99,7 +169,8 @@ private Type readRasterDataType(JSONObject asset) { } String type = asset.getJSONArray("raster:bands").getJSONObject(0).getString("data_type"); // https://github.com/stac-extensions/raster?tab=readme-ov-file#data-types - final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", "float16", "float32", "float64"); + final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", + "float16", "float32", "float64"); if (NUMERIC_DATA_TYPES.contains(type)) { return Type.NUMBER; } @@ -121,8 +192,8 @@ private CodelistReference populateCodelist(String assetId, Map v MappingReference direct = new MappingReference(); MappingReference inverse = new MappingReference(); vals.entrySet().forEach(code -> { - direct.getMappings().add(new Pair<>(code.getKey(), (String)code.getValue())); - codelist.getCodeDescriptions().put(code.getKey(), (String)code.getValue()); + direct.getMappings().add(new Pair<>(code.getKey(), (String) code.getValue())); + codelist.getCodeDescriptions().put(code.getKey(), (String) code.getValue()); }); Type type = STACUtils.inferValueType(vals.entrySet().stream().findFirst().get().getKey()); codelist.setType(type); @@ -132,7 +203,8 @@ private CodelistReference populateCodelist(String assetId, Map v } private void readMetadata(final JSONObject json, Builder builder) { - // We could check the doi only if the Scientific Notation extension is provided, but we can try anyway + // We could check the doi only if the Scientific Notation extension is provided, but we can + // try anyway String doi = STACUtils.readDOI(json); if (doi != null && !doi.isBlank()) { builder.withMetadata(IMetadata.DC_URL, doi); @@ -199,7 +271,7 @@ public Collection getAllFilesForResource(File file) { } @Override - public Map describeResource(IResource resource) { + public Map< ? extends String, ? extends Object> describeResource(IResource resource) { // TODO Auto-generated method stub return null; } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java new file mode 100644 index 0000000000..b422e81225 --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -0,0 +1,118 @@ +package org.integratedmodelling.klab.stac.extensions; + +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.geotools.api.data.FeatureSource; +import org.geotools.data.geojson.GeoJSONReader; +import org.geotools.data.memory.MemoryDataStore; +import org.integratedmodelling.klab.stac.STACUtils; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.hortonmachine.gears.io.stac.HMStacItem; + +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; +import org.integratedmodelling.klab.api.observations.scale.IScale; +import org.integratedmodelling.klab.api.observations.scale.space.IEnvelope; +import org.integratedmodelling.klab.api.observations.scale.space.IGrid; +import org.integratedmodelling.klab.api.observations.scale.time.ITimeInstant; +import org.integratedmodelling.klab.stac.STACUtils; +import java.time.format.DateTimeFormatter; +import java.time.*; + +public class STACFeatureExtension { + public static FeatureSource getFeatures(JSONObject catalogData, String collectionId, List bbox, ITimeInstant start, ITimeInstant end) throws Exception { + + String searchEndpoint = STACUtils.getLinkTo(catalogData, "search") + .orElseThrow(() -> new Exception("Search Link not found for the Catalog")); + + List featureList = new ArrayList<>(); + + JSONArray bboxArray = new JSONArray(); + for (Double v : bbox) { + bboxArray.put(v); + } + + + JSONObject searchPayload = new JSONObject() + .put("limit", 1000) + .put("bbox", bboxArray) + .put("collections", new JSONArray().put(collectionId)); + + while (searchEndpoint != null) { + + HttpResponse response = Unirest + .post(searchEndpoint) + .header("Content-Type", "application/json") + .body(searchPayload) + .asJson(); + + JSONObject body = response.getBody().getObject(); + JSONArray features = body.getJSONArray("features"); + + Iterator featureIterator = features.iterator(); + + while (featureIterator.hasNext()) { + try { + JSONObject feature = (JSONObject) featureIterator.next(); + SimpleFeature feat = GeoJSONReader.parseFeature(feature.toString()); + HMStacItem item = HMStacItem.fromSimpleFeature(feat); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + if (item.getStartTimestamp() == null || item.getEndTimestamp() == null) { // Assume best case scenario + featureList.add(feat); + continue; + } + + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { + featureList.add(feat); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + searchEndpoint = null; + if (body.has("links")) { + JSONArray links = body.getJSONArray("links"); + + for (Object obj : links) { + JSONObject link = (JSONObject) obj; + + if ("next".equalsIgnoreCase(link.optString("rel"))) { + String searchEndpointNew = link.getString("href"); + if (searchEndpointNew.equals(searchEndpoint)) { + searchEndpoint = null; + } + break; + } + } + } + } + if (featureList.isEmpty()) { + throw new Exception("No features found for the given parameters"); + } + + SimpleFeatureType type = featureList.get(0).getType(); + MemoryDataStore dataStore = new org.geotools.data.memory.MemoryDataStore(type); + dataStore.addFeatures(featureList); + return dataStore.getFeatureSource(type.getTypeName()); + } +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java index 345f52f22f..8ba0315267 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACIIASAExtension.java @@ -5,12 +5,12 @@ import java.util.Iterator; import java.util.List; -import org.geotools.data.FeatureSource; +import org.geotools.api.data.FeatureSource; import org.geotools.data.geojson.GeoJSONReader; import org.geotools.data.memory.MemoryDataStore; import org.integratedmodelling.klab.stac.STACUtils; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; import kong.unirest.HttpResponse; import kong.unirest.JsonNode; diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index 0404a17d69..899adae92a 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -8,19 +8,26 @@ { final text 'collection' - "The URL pointing to the STAC collection file that contains the resource dataset." + "[REQUIRED] The URL pointing to the STAC collection file that contains the resource dataset." optional text 'asset' - "The asset that is going to be retrieved from the items. Left it blank when the information is stored in the feature." + "The asset that is going to be retrieved from the items. Should be left blank when the information is stored in the feature." optional number 'band' "Relevant only for raster resources. - The band for a multi-band raster. Default is 0." + The band for a multi-band raster. Default it will pick up the first one." default 0 - optional text 'cog' - "Relevant only for Resources served as Cloud Optimized GeoTiff, with Public Access" + optional text 'jsonSelector' + "JSON selector" + default "" + + optional text 'jsonValue' + "JSON value" default "" + + optional text 'cog' + "Relevant only for Resources served as Cloud Optimised GeoTiff, with Public Access" optional enum 'bandmixer' "Relevant only for raster resources.\n diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl index 24e2f0a112..dad0ee4666 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl @@ -35,6 +35,6 @@ optional enum 'axisOrder' "Select the order of the XY axis: 'lat_lon' (default), or 'lon_lat'." - default "lat_lon" + default lat_lon values lat_lon, lon_lat } \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java new file mode 100644 index 0000000000..11c75358ea --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java @@ -0,0 +1,407 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +class STACPathExpressionTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static JsonNode stacItemNode() throws Exception { + String json = """ + { + "assets": { + "CHELSA_bio12_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_bio12_1981-2010_V.2.1.tif", + "title": "Climate Annual Precipitation (CHELSA bio12)", + "description": "Annual precipitation data from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "proj:bbox": [ + -180.0, + -90.0, + 180.0, + 90.0 + ], + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "bio12", + "description": "Annual precipitation" + } + ], + "file:size": 2608605060, + "raster:bands": [ + { + "nodata": -99999, + "data_type": "int32" + } + ] + }, + "CHELSA_gdd5_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gdd5_1981-2010_V.2.1.tif", + "title": "Climate Growing Degree Days > 5°C (CHELSA gdd5)", + "description": "Growing degree days (T > 5°C) from the CHELSA Climate dataset for the period 1981-2010", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gdd5", + "description": "Growing degree days with temperature > 5°C" + } + ], + "file:size": 3520234749, + "raster:bands": [ + { + "nodata": 2147483647, + "data_type": "int32" + } + ] + }, + "CHELSA_gsp_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gsp_1981-2010_V.2.1.tif", + "title": "Climate Precipitation in Growing Season (CHELSA gsp)", + "description": "Precipitation during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:3857", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gsp", + "description": "Precipitation in growing season" + } + ], + "file:size": 2447532198, + "raster:bands": [ + { + "nodata": 2147483648, + "data_type": "float32" + } + ] + }, + "CHELSA_gst_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gst_1981-2010_V.2.1.tif", + "title": "Climate Mean Temperature in Growing Season (CHELSA gst)", + "description": "Mean temperature during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gst", + "description": "Mean temperature in growing season" + } + ], + "file:size": 308275230, + "raster:bands": [ + { + "nodata": 65535, + "data_type": "int32" + } + ] + } + } + } + """; + + return OBJECT_MAPPER.readTree(json); + } + + private static JsonNode assetNode(String assetId) throws Exception { + return stacItemNode().get("assets").get(assetId); + } + + private static HMStacAsset bio12Asset() throws Exception { + return new HMStacAsset( + "CHELSA_bio12_1981-2010_V.2.1", + assetNode("CHELSA_bio12_1981-2010_V.2.1") + ); + } + + private static HMStacAsset gstAsset() throws Exception { + return new HMStacAsset( + "CHELSA_gst_1981-2010_V.2.1", + assetNode("CHELSA_gst_1981-2010_V.2.1") + ); + } + + @Test + void stacPathExpressionMatchesAssetByVirtualIdFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.id"); + + assertTrue(expression.matches(root, "CHELSA_gst_1981-2010_V.2.1")); + assertTrue(expression.matches(root, "CHELSA_bio12_1981-2010_V.2.1")); + assertFalse(expression.matches(root, "CHELSA_unknown")); + } + + @Test + void stacPathExpressionResolvesAllAssetIdsFromAssetsObjectMapKeys() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.id").resolve(root); + + assertEquals(4, resolved.size()); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_bio12_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gdd5_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gsp_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gst_1981-2010_V.2.1"))); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingImplicitArrayExpansion() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands.name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gdd5")); + assertTrue(expression.matches(root, "gsp")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingExplicitArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands[0].name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesAssetMediaTypeFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.type"); + + assertTrue(expression.matches(root, "image/tiff")); + assertFalse(expression.matches(root, "application/json")); + } + + @Test + void stacPathExpressionMatchesRasterBandDataTypeUsingArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.raster:bands[0].data_type"); + + assertTrue(expression.matches(root, "int32")); + assertTrue(expression.matches(root, "float32")); + assertFalse(expression.matches(root, "uint8")); + } + + @Test + void stacPathExpressionMatchesNumericValuesUsingNumericComparison() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.file:size"); + + assertTrue(expression.matches(root, "2608605060")); + assertTrue(expression.matches(root, "308275230.0")); + assertFalse(expression.matches(root, "123")); + } + + @Test + void hmStacAssetPredicateMatchesJsonExpressionAgainstAssetNode() throws Exception { + HMStacAsset asset = bio12Asset(); + + Predicate bandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate indexedBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands[0].name", "bio12"); + + Predicate typePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("type", "image/tiff"); + + Predicate rasterDataTypePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("raster:bands[0].data_type", "int32"); + + assertTrue(bandPredicate.test(asset)); + assertTrue(indexedBandPredicate.test(asset)); + assertTrue(typePredicate.test(asset)); + assertTrue(rasterDataTypePredicate.test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "gst") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("type", "application/json") + .test(asset)); + } + + @Test + void hmStacAssetPredicateMatchesJavaAssetAttributes() throws Exception { + HMStacAsset asset = bio12Asset(); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Climate Annual Precipitation (CHELSA bio12)") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "image/tiff") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326.0") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("valid", Boolean.toString(asset.isValid())) + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Wrong title") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "application/json") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "3857") + .test(asset)); + } + + @Test + void hmStacAssetAttributePredicateDistinguishesDifferentAssets() throws Exception { + HMStacAsset bio12 = bio12Asset(); + HMStacAsset gst = gstAsset(); + + Predicate bio12IdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1"); + + Predicate gstIdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1"); + + Predicate bio12BandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate gstBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "gst"); + + assertTrue(bio12IdPredicate.test(bio12)); + assertFalse(bio12IdPredicate.test(gst)); + + assertTrue(gstIdPredicate.test(gst)); + assertFalse(gstIdPredicate.test(bio12)); + + assertTrue(bio12BandPredicate.test(bio12)); + assertFalse(bio12BandPredicate.test(gst)); + + assertTrue(gstBandPredicate.test(gst)); + assertFalse(gstBandPredicate.test(bio12)); + } + + @Test + void predicatesReturnFalseForNullHMStacAsset() { + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "bio12") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + } + + @Test + void stacPathExpressionReturnsEmptyResultForMissingPath() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.missing.attribute").resolve(root); + + assertTrue(resolved.isEmpty()); + } + + @Test + void parseRejectsOldWildcardArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[*].name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void oldGreaterThanSeparatorDoesNotMatchBecauseDotIsNowTheSeparator() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets>eo:bands>name"); + + assertFalse(expression.matches(root, "bio12")); + } + + @Test + void parseRejectsEmptyPath() { + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(null)); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse("")); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(" ")); + } + + @Test + void parseRejectsNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[-1].name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void parseRejectsUnsupportedHMStacAssetAttribute() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute("unsupported", "value") + ); + + assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); + } +} \ No newline at end of file diff --git a/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java b/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java index 111ed6711a..a629c10cff 100644 --- a/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java +++ b/authorities/klab.authority.caliper/src/main/java/org/integratedmodelling/authorities/caliper/CaliperAuthority.java @@ -61,7 +61,7 @@ public class CaliperAuthority implements IAuthority { + " ?concept skos:prefLabel ?label_en . FILTER(contains(lcase(str(?label_en)), '{QUERY_STRING}')) .\r\n" + " ?concept skos:notation ?code .\r\n" + " ?concept skos:broader ?broader .\r\n" + "} order by ?code"; - private static final String SPARQL_ENDPOINT = "https://stats.fao.org/caliper/AllVocs/"; + private static final String SPARQL_ENDPOINT = "https://caliper.integratedmodelling.org/caliper/sparql"; private static final Map CALIPER_SCHEMES = new HashMap<>(); private static final Map CALIPER_DESCRIPTIONS = new HashMap<>(); private static final Map CALIPER_URLS = new HashMap<>(); @@ -70,26 +70,27 @@ public class CaliperAuthority implements IAuthority { // TODO all this should come from caliper, filtered as needed and cached -// CALIPER_SCHEMES.put("ISIC", "http://unstats.un.org/classifications/ISIC/rev4/scheme"); + CALIPER_SCHEMES.put("ISIC", "https://unstats.un.org/classifications/ISIC/rev4/scheme"); // CALIPER_SCHEMES.put("ICC10", "http://stats-class.fao.uniroma2.it/ICC/v1.0/scheme"); - CALIPER_SCHEMES.put("ICC", "http://stats.fao.org/classifications/ICC/v1.1/scheme"); + CALIPER_SCHEMES.put("ICC", "https://stats.fao.org/classifications/ICC/v1.1/scheme"); // CALIPER_SCHEMES.put("M49", "http://stats.fao.org/classifications/geo/M49"); // CALIPER_SCHEMES.put("SDGEO", "http://stats.fao.org/classifications/geo/M49/SDG-groups"); // CALIPER_SCHEMES.put("FOODEX2", "http://stats.fao.org/classifications/foodex2/all"); // CALIPER_SCHEMES.put("CPC20", "http://stats-class.fao.uniroma2.it/CPC/v2.0/scheme"); - CALIPER_SCHEMES.put("CPC", "http://unstats.un.org/classifications/CPC/v2.1/core"); + CALIPER_SCHEMES.put("CPC", "https://unstats.un.org/classifications/CPC/v2.1/core"); // CALIPER_SCHEMES.put("CPC21FERT", "http://stats-class.fao.uniroma2.it/CPC/v2.1/fert"); // CALIPER_SCHEMES.put("FCL", "http://stats-class.fao.uniroma2.it/FCL/v2019/scheme"); - CALIPER_SCHEMES.put("HS", "http://stats.fao.org/classifications/HS/fao_mapping_targets/scheme"); + // CALIPER_SCHEMES.put("HS", + // "http://stats.fao.org/classifications/HS/fao_mapping_targets/scheme"); - CALIPER_URLS.put("ISIC", "http://unstats.un.org/classifications/ISIC/rev4"); + CALIPER_URLS.put("ISIC", "https://unstats.un.org/classifications/ISIC/rev4"); // CALIPER_SCHEMES.put("ICC10", "http://stats-class.fao.uniroma2.it/ICC/v1.0"); - CALIPER_URLS.put("ICC", "http://stats.fao.org/classifications/ICC/v1.1"); + CALIPER_URLS.put("ICC", "https://stats.fao.org/classifications/ICC/v1.1"); // CALIPER_URLS.put("M49", "http://stats.fao.org/classifications/geo/m49"); // CALIPER_URLS.put("SDGEO", "http://stats.fao.org/classifications/geo/M49/SDG-groups"); // CALIPER_URLS.put("FOODEX2", "http://stats.fao.org/classifications/foodex2"); // CALIPER_SCHEMES.put("CPC20", "http://stats-class.fao.uniroma2.it/CPC/v2.0"); - CALIPER_URLS.put("CPC", "http://stats.fao.org/classifications/CPC/v2.1"); + CALIPER_URLS.put("CPC", "https://stats.fao.org/classifications/CPC/v2.1"); // CALIPER_SCHEMES.put("CPC21FERT", "http://stats-class.fao.uniroma2.it/CPC/v2.1/fert"); // CALIPER_SCHEMES.put("FCL", "http://stats-class.fao.uniroma2.it/FCL/v2019"); // CALIPER_URLS.put("HS", "http://stats.fao.org/classifications/HS/fao_mapping_targets"); @@ -163,7 +164,7 @@ public Identity getIdentity(String identityId, String catalog) { Set parents = new HashSet<>(); source = new AuthorityIdentity(); - for (Statement statement : model) { + for(Statement statement : model) { // System.out.println("CIAPA EL STATEMENT: " + statement); @@ -199,7 +200,7 @@ public Identity getIdentity(String identityId, String catalog) { ((AuthorityIdentity) source).setConceptName(sanitize(catalog, ((AuthorityIdentity) source).getId())); ((AuthorityIdentity) source).setLocator(ID + "." + catalog + ":" + ((AuthorityIdentity) source).getId()); - for (String parent : parents) { + for(String parent : parents) { if (((AuthorityIdentity) source).getParentIds() == null) { ((AuthorityIdentity) source).setParentIds(new ArrayList<>()); } @@ -236,7 +237,7 @@ public Capabilities getCapabilities() { ret.setFuzzy(true); ret.setName(ID); ret.setDescription(DESCRIPTION); - for (String s : CALIPER_DESCRIPTIONS.keySet()) { + for(String s : CALIPER_DESCRIPTIONS.keySet()) { ret.getSubAuthorities().add(new Pair<>(s, CALIPER_DESCRIPTIONS.get(s))); } @@ -259,7 +260,7 @@ public List search(String query, String catalog) { if (response.isSuccess()) { try { JSONObject result = response.getBody().getObject(); - for (Object zoz : result.getJSONObject("results").getJSONArray("bindings")) { + for(Object zoz : result.getJSONObject("results").getJSONArray("bindings")) { JSONObject res = (JSONObject) zoz; String code = res.getJSONObject("code").getString("value"); @@ -299,7 +300,7 @@ public static void main(String[] args) { try (InputStream input = new URL("http://unstats.un.org/classifications/CPC/v2.0/0.ttl").openStream()) { Model model = Rio.parse(input, RDFFormat.TURTLE); - for (Statement statement : model) { + for(Statement statement : model) { System.out.println("CIAPA EL STATEMENT: " + statement); } } catch (Exception e) { @@ -308,15 +309,15 @@ public static void main(String[] args) { } - @Override - public String getName() { - return ID; - } + @Override + public String getName() { + return ID; + } - @Override - public ICodelist getCodelist() { - // TODO this may be less than obvious with Caliper - return null; - } + @Override + public ICodelist getCodelist() { + // TODO this may be less than obvious with Caliper + return null; + } } diff --git a/components/klab.component.hydrology/pom.xml b/components/klab.component.hydrology/pom.xml index 8a42ff8f31..b70b52f0db 100644 --- a/components/klab.component.hydrology/pom.xml +++ b/components/klab.component.hydrology/pom.xml @@ -53,5 +53,6 @@ org.integratedmodelling.klab.api ${klab.version} + \ No newline at end of file diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java index 79e4281eaa..7d41fa927b 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/adapters/GridAdapter.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.Set; -import org.geotools.data.simple.SimpleFeatureSource; +import org.geotools.api.data.SimpleFeatureSource; import org.geotools.feature.FeatureIterator; import org.geotools.grid.Grids; import org.integratedmodelling.klab.Configuration; @@ -44,7 +44,7 @@ import org.mapdb.DB; import org.mapdb.DBMaker; import org.mapdb.Serializer; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeature; @UrnAdapter(type = GridAdapter.NAME, version = Version.CURRENT) public class GridAdapter implements IUrnAdapter { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java index 7717a3d9ee..e2a1bb207d 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/BarnesSurfaceResolver.java @@ -48,12 +48,12 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.geometry.Envelope; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.crs.GeographicCRS; -import org.opengis.referencing.crs.ProjectedCRS; -import org.opengis.referencing.datum.Ellipsoid; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.geometry.Bounds; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.GeographicCRS; +import org.geotools.api.referencing.crs.ProjectedCRS; +import org.geotools.api.referencing.datum.Ellipsoid; public class BarnesSurfaceResolver extends AbstractContextualizer implements IResolver, IExpression { @@ -87,7 +87,7 @@ public IState resolve(IState target, IContextualizationScope context) throws Kla grid.getEast(), grid.getWest(), (int) grid.getXCells(), (int) grid.getYCells(), grid.getProjection().getCoordinateReferenceSystem()); - Envelope env = inInterpolationGrid.getEnvelope(); + Bounds env = inInterpolationGrid.getEnvelope(); CoordinateReferenceSystem Crs = grid.getProjection().getCoordinateReferenceSystem(); diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java index aef6fb5976..6c100f27a2 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/CustomBarnesSurfaceInterpolator.java @@ -22,8 +22,8 @@ import org.geotools.referencing.GeodeticCalculator; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.datum.Ellipsoid; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.datum.Ellipsoid; /** * Interpolates a surface across a regular grid from an irregular set of data points using the diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java index 30ceb2e77e..3484285c14 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/core/KrigingResolver.java @@ -13,7 +13,7 @@ import org.hortonmachine.gears.libs.modules.HMConstants; import org.hortonmachine.gears.utils.coverage.CoverageUtilities; import org.hortonmachine.gears.utils.math.matrixes.MatrixException; -import org.hortonmachine.hmachine.modules.statistics.kriging.old.OmsKriging; +import org.hortonmachine.hmachine.modules.statistics.krigingexp.old.OmsKriging; import org.integratedmodelling.geoprocessing.TaskMonitor; import org.integratedmodelling.kim.api.IKimConcept; import org.integratedmodelling.klab.Concepts; @@ -39,7 +39,7 @@ import org.integratedmodelling.klab.utils.NumberUtils; import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeatureType; +import org.geotools.api.feature.simple.SimpleFeatureType; public class KrigingResolver extends AbstractContextualizer implements IResolver, IExpression { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java index a99c1bb5c2..5919420165 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/MarineFloodResolver.java @@ -3,11 +3,9 @@ import static org.hortonmachine.gears.libs.modules.HMConstants.floatNovalue; import java.awt.image.DataBuffer; - -import javax.media.jai.iterator.RandomIter; +import java.util.ArrayList; import org.geotools.coverage.grid.GridCoverage2D; -import org.geotools.coverage.util.CoverageUtilities; import org.hortonmachine.gears.libs.modules.HMRaster; import org.hortonmachine.gears.utils.RegionMap; import org.integratedmodelling.klab.api.data.general.IExpression; @@ -20,10 +18,6 @@ import org.integratedmodelling.klab.exceptions.KlabException; import org.integratedmodelling.klab.utils.Parameters; -import oms3.gen.doubleAccess; - -import java.util.ArrayList; - public class MarineFloodResolver extends AbstractContextualizer implements IResolver, IExpression { private Double decayFact; diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java index 4229b664b3..7bfae9f4d5 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/hydrology/StreamOutletInstantiator.java @@ -9,9 +9,8 @@ import java.util.Comparator; import java.util.List; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; - +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; import org.hortonmachine.gears.libs.modules.FlowNode; import org.hortonmachine.hmachine.modules.demmanipulation.markoutlets.OmsMarkoutlets; import org.integratedmodelling.geoprocessing.TaskMonitor; diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java index 6a77f0e8c8..84fb4c8db5 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/morphology/MaximaFinderInstantiator.java @@ -31,7 +31,7 @@ import org.integratedmodelling.klab.rest.StateSummary; import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.utils.Parameters; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeature; public class MaximaFinderInstantiator extends AbstractContextualizer implements IInstantiator, IExpression { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java index 9c5b9aac1d..1335ac7188 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/remotesensing/SentinelResolver.java @@ -12,7 +12,7 @@ import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; -import org.hortonmachine.gears.utils.CrsUtilities; +import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.RegionMap; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; import org.integratedmodelling.geoprocessing.TaskMonitor; @@ -34,7 +34,7 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; public class SentinelResolver extends AbstractContextualizer implements IResolver, IExpression { diff --git a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java index 8c6a47e6cc..144f345be1 100644 --- a/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java +++ b/components/klab.component.hydrology/src/main/java/org/integratedmodelling/geoprocessing/viewshed/ViewshedResolver.java @@ -29,7 +29,7 @@ import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; public class ViewshedResolver extends AbstractContextualizer implements IResolver, IExpression { diff --git a/components/klab.component.tables/pom.xml b/components/klab.component.tables/pom.xml index b2066517c0..93196a16c8 100644 --- a/components/klab.component.tables/pom.xml +++ b/components/klab.component.tables/pom.xml @@ -36,9 +36,20 @@ - tech.tablesaw - tablesaw-excel - 0.38.5 + tech.tablesaw + tablesaw-excel + 0.38.5 + + + org.apache.poi + poi-ooxml + + + + + + org.apache.poi + poi-ooxml @@ -76,3 +87,4 @@ + diff --git a/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java b/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java index dfb3debdfb..8b9f36231e 100644 --- a/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java +++ b/components/klab.components.cdm/src/main/java/org/integratedmodelling/cdm/utils/NetCDFUtils.java @@ -25,8 +25,11 @@ import java.util.logging.Level; import java.util.logging.Logger; -import javax.media.jai.RasterFactory; - +import org.eclipse.imagen.RasterFactory; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValueGroup; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.GridCoverageFactory; import org.geotools.coverage.grid.io.AbstractGridFormat; @@ -46,10 +49,6 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnsupportedFeatureException; import org.integratedmodelling.klab.utils.NumberUtils; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.parameter.GeneralParameterValue; -import org.opengis.parameter.ParameterValueGroup; -import org.opengis.referencing.crs.CoordinateReferenceSystem; import com.beust.jcommander.internal.Lists; diff --git a/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java b/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java index 5d64c3f114..7cdf4406f6 100644 --- a/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java +++ b/ide/org.integratedmodelling.klab.ide/src/org/integratedmodelling/klab/ide/views/ResourceEditor.java @@ -560,6 +560,9 @@ public void loadResource(ResourceReference resource) { private File getResourcePath(ResourceReference resource) { String path = resource.getLocalPath(); + if (path == null) { + return null; + } String project = Path.getFirst(path, "/"); IKimProject prj = Kim.INSTANCE.getProject(project); if (project != null) { diff --git a/kactors/org.integratedmodelling.kactors/build.properties b/kactors/org.integratedmodelling.kactors/build.properties index aa338a94e6..c324a919eb 100644 --- a/kactors/org.integratedmodelling.kactors/build.properties +++ b/kactors/org.integratedmodelling.kactors/build.properties @@ -15,6 +15,5 @@ additional.bundles = org.eclipse.xtext.xbase,\ org.eclipse.emf.mwe2.launch,\ org.eclipse.emf.mwe2.lib,\ org.objectweb.asm,\ - org.apache.commons.logging,\ org.apache.log4j,\ com.ibm.icu diff --git a/kdl/org.integratedmodelling.kdl/build.properties b/kdl/org.integratedmodelling.kdl/build.properties index 18d540bf6c..a7b32375b7 100644 --- a/kdl/org.integratedmodelling.kdl/build.properties +++ b/kdl/org.integratedmodelling.kdl/build.properties @@ -15,6 +15,5 @@ additional.bundles = org.eclipse.xtext.xbase,\ org.eclipse.emf.mwe2.launch,\ org.eclipse.emf.mwe2.lib,\ org.objectweb.asm,\ - org.apache.commons.logging,\ org.apache.log4j,\ com.ibm.icu diff --git a/kim/org.integratedmodelling.kim/build.properties b/kim/org.integratedmodelling.kim/build.properties index 2072401d98..05568542e3 100644 --- a/kim/org.integratedmodelling.kim/build.properties +++ b/kim/org.integratedmodelling.kim/build.properties @@ -13,6 +13,5 @@ additional.bundles = org.eclipse.xtext.xbase,\ org.eclipse.emf.mwe2.launch,\ org.eclipse.emf.mwe2.lib,\ org.objectweb.asm,\ - org.apache.commons.logging,\ org.apache.log4j,\ com.ibm.icu \ No newline at end of file diff --git a/klab.authentication/pom.xml b/klab.authentication/pom.xml index 25a62c91b4..c4189aab04 100644 --- a/klab.authentication/pom.xml +++ b/klab.authentication/pom.xml @@ -36,7 +36,6 @@ commons-io commons-io - 2.14.0 diff --git a/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java b/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java index 2d980bbdfc..9c77612615 100644 --- a/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java +++ b/klab.authentication/src/main/java/org/integratedmodelling/klab/Authentication.java @@ -463,7 +463,7 @@ public ExternalAuthenticationCredentials getCredentials(String endpoint) { * @return the credential or null if not present */ private ExternalAuthenticationCredentials getCredentialsByUrl(String hostUrl) { - return externalCredentials.values().stream().filter(credential -> credential.getURL().equals(hostUrl)).sorted() + return externalCredentials.values().stream().filter(credential -> credential.getURL().equals(hostUrl)) .findFirst().orElse(null); } diff --git a/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java b/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java index 512530c3f1..9655a693a1 100644 --- a/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java +++ b/klab.authentication/src/main/java/org/integratedmodelling/klab/communication/client/Client.java @@ -76,23 +76,25 @@ import org.springframework.web.client.ResponseErrorHandler; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.client.UnknownContentTypeException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; /** - * Helper to avoid having to write 10 lines every time I need to do a GET with headers. It can be - * given authorization objects using the with(...) idiom: + * Helper to avoid having to write 10 lines every time I need to do a GET with + * headers. It can be given authorization objects using the with(...) idiom: * * RestTemplateHelper template = new RestTemplateHelper(); ... * template.with(session).post(....) ... * - * with(..) can be called with any {@link IIdentity} according to the type of authentication - * required by the receiver. Automatically sets the necessary tokens and parameters. + * with(..) can be called with any {@link IIdentity} according to the type of + * authentication required by the receiver. Automatically sets the necessary + * tokens and parameters. * - * The template also configures an objectmapper for optimal use in k.LAB, manages errors and inserts - * user agent headers. + * The template also configures an objectmapper for optimal use in k.LAB, + * manages errors and inserts user agent headers. * * * @author ferdinando.villa @@ -100,677 +102,684 @@ */ public class Client extends RestTemplate implements IClient { - public static final String KLAB_VERSION_HEADER = "KlabVersion"; - public static final String KLAB_CONNECTION_TIMEOUT = "klab.connection.timeout"; - - ObjectMapper objectMapper; - String authorizationToken; - String authenticationToken; - MediaType contentType = new MediaType(MediaType.APPLICATION_JSON, StandardCharsets.UTF_8); - RestTemplate basicTemplate; - private Set endpoints = new HashSet<>(); - - private static ClientHttpRequestFactory factory; - - public static Client createCustomTimeoutClient(int timeout) { - HttpComponentsClientHttpRequestFactory custom = new HttpComponentsClientHttpRequestFactory(); - custom.setReadTimeout(timeout); - custom.setConnectTimeout(timeout); - return new Client(custom); - } - - public static synchronized Client create() { - - if (factory == null) { - factory = new HttpComponentsClientHttpRequestFactory(); - if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { - int connectTimeout = 1000 - * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); - ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); - ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); - } - } - - return new Client(factory); - } - - public static Client createJson() { - Client ret = create(); - ret.setup(); - return ret; - } - - /** - * This one will return a client that interprets anything as JSON independent of content type. - * Required for misbehaving services that return JSON with other content types. - * - * @return - */ - public static synchronized Client createUniversalJSON() { - - if (factory == null) { - factory = new HttpComponentsClientHttpRequestFactory(); - if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { - int connectTimeout = 1000 - * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); - ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); - ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); - } - } - - return new UniversalClient(factory); - } - - public static class UniversalClient extends Client { - UniversalClient(ClientHttpRequestFactory factory) { - super(factory); - objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - List> messageConverters = new ArrayList>(); - MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); - /* - * Make this converter process any kind of response, not only application/*json, which - * is the default behaviour - */ - converter.setSupportedMediaTypes(Collections.singletonList(MediaType.ALL)); - messageConverters.add(converter); - this.setMessageConverters(messageConverters); - } - } - - public static class NodeClient extends Client { - - public NodeClient(INodeIdentity node) { - - super(factory); - - objectMapper = new ObjectMapper(); - List> messageConverters = new ArrayList<>(); - MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); - StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); - FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); - ProtobufHttpMessageConverter protobufConverter = new ProtobufHttpMessageConverter(); - // ByteArrayHttpMessageConverter byteConverter = new - // ByteArrayHttpMessageConverter(); - jsonMessageConverter.setObjectMapper(objectMapper); - - setErrorHandler(new JSONResponseErrorHandler()); - messageConverters.add(jsonMessageConverter); - messageConverters.add(utf8); - messageConverters.add(formHttpMessageConverter); - messageConverters.add(protobufConverter); - // messageConverters.add(byteConverter); - setMessageConverters(messageConverters); - this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); - this.authorizationToken = node.getId(); - } - } - - /** - * Send an authentication request to a hub for an engine. - * - * @param url - * @param request - * @return the response. If not authenticated, throw a KlabAuthorizationException. If timeout, - * return null. - */ - public EngineAuthenticationResponse authenticateEngine(String url, EngineAuthenticationRequest request) { - return post(url + API.HUB.AUTHENTICATE_ENGINE, request, EngineAuthenticationResponse.class); - } - - /** - * Send an authentication request to a hub for a node. - * - * @param url - * @param request - * @return the response. If not authenticated, throw a KlabAuthorizationException. If timeout, - * return null. - */ - public NodeAuthenticationResponse authenticateNode(String url, NodeAuthenticationRequest request) { - return post(url + API.HUB.AUTHENTICATE_NODE, request, NodeAuthenticationResponse.class); - } - - /** - * Check an engine's heartbeat. - * - * @param url base engine/node URL - * @return true if alive - */ - public boolean ping(String url, String endpoint) { - try { - ResponseEntity response = basicTemplate.exchange(url + endpoint, HttpMethod.GET, - new HttpEntity(null, null), Object.class); - return response.getStatusCodeValue() == 200; - } catch (Throwable e) { - return false; - } - } - - public boolean ping(String url) { - return ping(url, "/actuator/health"); - } - - public boolean pingService(String url) { - return ping(url, "/public/status"); - } - - private class JSONResponseErrorHandler implements ResponseErrorHandler { - - @Override - public void handleError(ClientHttpResponse response) throws IOException { - } - - @Override - public boolean hasError(ClientHttpResponse response) throws IOException { - HttpStatus status = response.getStatusCode(); - HttpStatus.Series series = status.series(); - return (HttpStatus.Series.CLIENT_ERROR.equals(series) || HttpStatus.Series.SERVER_ERROR.equals(series)); - } - } - - /** - * Add option to set both headers, Authorization and Authentication - * @param headers the headers object to set the new headers - */ - private void setAuthTokens(HttpHeaders headers) { - if (authorizationToken != null) { - headers.set(HttpHeaders.AUTHORIZATION, authorizationToken); - } - if (authenticationToken != null) { - headers.add("Authentication", authenticationToken); - } - } - /** - * Interceptor to add user agents and ensure that the authorization token gets in. - * - * @author ferdinando.villa - * - */ - public class AuthorizationInterceptor implements ClientHttpRequestInterceptor { - - @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) - throws IOException { - - HttpRequestWrapper requestWrapper = new HttpRequestWrapper(request); - HttpHeaders headers = requestWrapper.getHeaders(); - headers.set("Accept", "application/json"); - headers.set("X-User-Agent", "k.LAB " + Version.CURRENT); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - return execution.execute(requestWrapper, body); - } - } - - private void setup() { - - objectMapper = new ObjectMapper(); - List> messageConverters = new ArrayList<>(); - MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); - StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); - FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); - // ByteArrayHttpMessageConverter byteConverter = new - // ByteArrayHttpMessageConverter(); - jsonMessageConverter.setObjectMapper(objectMapper); - - setErrorHandler(new JSONResponseErrorHandler()); - messageConverters.add(jsonMessageConverter); - messageConverters.add(utf8); - messageConverters.add(formHttpMessageConverter); - // messageConverters.add(byteConverter); - setMessageConverters(messageConverters); - this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); - } - - private Client(ClientHttpRequestFactory factory) { - super(factory); - this.basicTemplate = new RestTemplate(); - setup(); - } - - private Client(Client client) { - super(factory); - this.basicTemplate = client.basicTemplate; - this.objectMapper = client.objectMapper; - this.endpoints = client.endpoints; - this.authorizationToken = client.authorizationToken; - this.authenticationToken = client.authenticationToken; - this.contentType = client.contentType; - setup(); - } - - private Client() { - super(factory); - } - - public Client withContentType(MediaType contentType) { - Client ret = new Client(this); - ret.contentType = contentType; - return ret; - } - - /** - * Return a client with authorization set to the passed object. - * - * @param authorizer any identity. - * @return a new - */ - @Override - public Client onBehalfOf(IIdentity authorizer) { - - Client ret = new Client(this); - ret.objectMapper = this.objectMapper; - ret.authorizationToken = authorizer.getId(); - return ret; - } - - public Client withAuthorization(String authorization) { - - Client ret = new Client(this); - ret.objectMapper = this.objectMapper; - ret.authorizationToken = authorization; - return ret; - } - - public Client withAuthentication(String authentication) { - - Client ret = new Client(); - ret.objectMapper = this.objectMapper; - ret.authenticationToken = authentication; - return ret; - } - - @SuppressWarnings("unchecked") - @Override - public T post(String url, Object data, Class< ? extends T> cls) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - if (data != null) { - headers.setContentType(contentType); - } - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - - HttpEntity entity = new HttpEntity<>(data, headers); - - try { - - ResponseEntity response = exchange(url, HttpMethod.POST, entity, Object.class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); - case 500: - throw new KlabResourceAccessException("internal: request " + url + " caused a remote server error"); - } - - if (response.getBody() == null) { - return null; - } - if (response.getBody() instanceof Map && ((Map< ? , ? >) response.getBody()).containsKey("exception") - && ((Map< ? , ? >) response.getBody()).get("exception") != null) { - - Map< ? , ? > map = (Map< ? , ? >) response.getBody(); - Object exception = map.get("exception"); - // Object path = map.get("path"); - Object message = map.get("message"); - // Object error = map.get("error"); - - dumpRequest(url, headers, data); - throw new KlabIOException("remote exception: " + (message == null ? exception : message)); - } - - if (cls.isAssignableFrom(response.getBody().getClass())) { - return (T) response.getBody(); - } - - try { - return objectMapper.convertValue(response.getBody(), cls); - } catch (Throwable t) { - System.out.println("Unrecognized response: " + response.getBody()); - throw t; - } - } catch (RestClientException e) { - - System.out.println("REST exception: " + e.getMessage()); - dumpRequest(url, headers, data); - throw new KlabIOException(e); - } - } - - private void dumpRequest(String url, HttpHeaders headers, Object data) { - - System.out.println("Endpoint: " + url); - System.out.println("Headers:"); - for(String header : headers.keySet()) { - System.out.println(" " + header + " = " + headers.get(header)); - } - System.out.println("Data:\n" + printAsJson(data)); - - } - - public void setUrl(String... url) { - if (url == null || url.length == 0) { - this.endpoints.clear(); - } else { - for(String u : url) { - this.endpoints.add(u); - } - } - } - - private String pickEndpoint() { - // TODO periodically check URLs and choose the first that responds (or the one - // with the smallest load) - return this.endpoints.isEmpty() ? null : this.endpoints.iterator().next(); - } - - @Override - public boolean getDownload(String url, File output) { - - url = checkEndpoint(url); - - RestTemplate restTemplate = new RestTemplate(); - restTemplate.getMessageConverters().add(new ByteArrayHttpMessageConverter()); - - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - - HttpEntity entity = new HttpEntity(headers); - - // HttpHeaders headers = new HttpHeaders(); - // headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); - // HttpEntity entity = new HttpEntity<>(headers); - - ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, entity, byte[].class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); - } - - if (response.getBody() == null) { - return false; - } - if (response.getStatusCode() == HttpStatus.OK) { - try { - Files.write(output.toPath(), response.getBody()); - } catch (IOException e) { - throw new KlabIOException(e); - } - } - - return true; - } - - private String checkEndpoint(String url) { - if (!url.toLowerCase().startsWith("http")) { - String ep = pickEndpoint(); - if (ep != null) { - url = ep + (ep.endsWith("/") || url.startsWith("/") ? "" : "/") + url; - } - } - return url; - } - - /** - * Issue a DELETE request. Called remove to avoid conflict with super. - * - * @param url - * @param parameters - * @return - */ - public Object remove(String url, Object... parameters) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - HttpEntity entity = new HttpEntity<>(headers); - - if (parameters != null) { - String params = ""; - for(int i = 0; i < parameters.length; i++) { - String key = parameters[i].toString(); - String nakedKey = key; - String val = parameters[++i].toString(); - if (!(key.startsWith("{") && key.endsWith("}"))) { - key = "{" + key + "}"; - } else { - nakedKey = key.substring(1, key.length() - 1); - } - if (url.contains(key)) { - url = url.replace(key, val); - } else { - params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + Escape.forURL(val); - } - } - if (!params.isEmpty()) { - url += "?" + params; - } - } - - ResponseEntity< ? > response = exchange(url, HttpMethod.DELETE, entity, Object.class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); - case 503: - throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); - } - - return response.getBody(); - } - - /** - * Instrumented for header communication and error parsing - * - * @param url - * @param cls - * @return the deserialized result - */ - @Override - @SuppressWarnings({"unchecked"}) - public T get(String url, Class< ? extends T> cls, Object... parameters) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", cls.equals(String.class) ? "text/plain" : "application/json"); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - HttpEntity entity = new HttpEntity<>(headers); - - if (parameters != null) { - String params = ""; - for(int i = 0; i < parameters.length; i++) { - String key = parameters[i].toString(); - String nakedKey = key; - Object nextPar = parameters[++i]; - String val = nextPar == null ? null : nextPar.toString(); - - if (val == null) { - continue; - } - - if (!(key.startsWith("{") && key.endsWith("}"))) { - key = "{" + key + "}"; - } else { - nakedKey = key.substring(1, key.length() - 1); - } - if (url.contains(key)) { - url = url.replace(key, val); - } else { - params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + /* Escape.forURL( */val/* ) */; - } - } - if (!params.isEmpty()) { - url += "?" + params; - } - } - - ResponseEntity< ? > response = null; - if (cls.isArray()) { - response = exchange(url, HttpMethod.GET, entity, Object.class); - } else if (String.class.equals(cls)) { - response = basicTemplate.exchange(url, HttpMethod.GET, entity, String.class); - } else /* if (Map.class.isAssignableFrom(cls)) */ { - response = exchange(url, HttpMethod.GET, entity, Map.class); - } - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); - case 406: - case 503: - throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); - } - - if (response.getBody() instanceof Map) { - - Object exception = ((Map< ? , ? >) response.getBody()).get("exception"); - Object error = ((Map< ? , ? >) response.getBody()).get("error"); - - if (exception != null || error != null) { - Object message = ((Map< ? , ? >) response.getBody()).get("message"); - // Object error = response.getBody().get("error"); - throw new KlabIOException( - "remote exception: " + (message == null ? (exception == null ? error : exception) : message)); - } - - return objectMapper.convertValue(response.getBody(), cls); - - } else if (response.getBody() instanceof List && cls.isArray()) { - - List< ? > list = (List< ? >) response.getBody(); - Object ret = Array.newInstance(cls.getComponentType(), (((List< ? >) response.getBody()).size())); - for(int i = 0; i < list.size(); i++) { - Object object = list.get(i); - if (object instanceof Map) { - object = objectMapper.convertValue(object, cls.getComponentType()); - } - Array.set(ret, i, object); - } - - return (T) ret; - - } else if (response.getBody() != null && cls.isAssignableFrom(response.getBody().getClass())) { - return (T) response.getBody(); - } - - return null; - } - - @Override - public T postFile(String url, File file, Class< ? extends T> cls) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("file", new FileSystemResource(file)); - HttpEntity> entity = new HttpEntity<>(body, headers); - - try { - - final RestTemplate restTemplate = new RestTemplate(); - SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); - // without this, large files will eat up the heap - requestFactory.setBufferRequestBody(false); - restTemplate.setRequestFactory(requestFactory); - ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class); - - switch(response.getStatusCodeValue()) { - case 302: - case 401: - case 403: - throw new KlabAuthorizationException("unauthorized request " + url); - case 404: - throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); - } - - if (response.getBody().containsKey("exception") && response.getBody().get("exception") != null) { - Object exception = response.getBody().get("exception"); - // Object path = response.getBody().get("path"); - Object message = response.getBody().get("message"); - // Object error = response.getBody().get("error"); - throw new KlabIOException("remote exception: " + (message == null ? exception : message)); - } - - return objectMapper.convertValue(response.getBody(), cls); - - } catch (RestClientException e) { - throw new KlabIOException(e); - } - - } - - public static String printAsJson(Object object) { - - ObjectMapper om = new ObjectMapper(); - om.enable(SerializationFeature.INDENT_OUTPUT); // pretty print - om.enable(SerializationFeature.WRITE_NULL_MAP_VALUES); // pretty print - om.enable(SerializationFeature.WRITE_SINGLE_ELEM_ARRAYS_UNWRAPPED); // pretty print - om.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); - - try { - return om.writeValueAsString(object); - } catch (Exception e) { - throw new IllegalArgumentException("serialization failed: " + e.getMessage()); - } - } - - @Override - public boolean put(String url, Object data) { - - url = checkEndpoint(url); - - HttpHeaders headers = new HttpHeaders(); - headers.set("Accept", "application/json"); - if (data != null) { - headers.setContentType(MediaType.APPLICATION_JSON); - } - headers.set(KLAB_VERSION_HEADER, Version.CURRENT); - setAuthTokens(headers); - - try { - - super.put(url, data); - return true; - - } catch (RestClientException e) { - System.out.println("REST exception: " + e.getMessage()); - dumpRequest(url, headers, data); - return false; - } - - } + public static final String KLAB_VERSION_HEADER = "KlabVersion"; + public static final String KLAB_USER_AGENT = "k.LAB/0.11.0"; + public static final String KLAB_CONNECTION_TIMEOUT = "klab.connection.timeout"; + + ObjectMapper objectMapper; + String authorizationToken; + String authenticationToken; + MediaType contentType = new MediaType(MediaType.APPLICATION_JSON, StandardCharsets.UTF_8); + RestTemplate basicTemplate; + private Set endpoints = new HashSet<>(); + + private static ClientHttpRequestFactory factory; + + public static Client createCustomTimeoutClient(int timeout) { + HttpComponentsClientHttpRequestFactory custom = new HttpComponentsClientHttpRequestFactory(); + custom.setReadTimeout(timeout); + custom.setConnectTimeout(timeout); + return new Client(custom); + } + + public static synchronized Client create() { + + if (factory == null) { + factory = new HttpComponentsClientHttpRequestFactory(); + if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { + int connectTimeout = 1000 + * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); + ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); + ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); + } + } + + return new Client(factory); + } + + public static Client createJson() { + Client ret = create(); + ret.setup(); + return ret; + } + + /** + * This one will return a client that interprets anything as JSON independent of + * content type. Required for misbehaving services that return JSON with other + * content types. + * + * @return + */ + public static synchronized Client createUniversalJSON() { + + if (factory == null) { + factory = new HttpComponentsClientHttpRequestFactory(); + if (Configuration.INSTANCE.getProperties().containsKey(KLAB_CONNECTION_TIMEOUT)) { + int connectTimeout = 1000 + * Integer.parseInt(Configuration.INSTANCE.getProperties().getProperty(KLAB_CONNECTION_TIMEOUT)); + ((HttpComponentsClientHttpRequestFactory) factory).setReadTimeout(connectTimeout); + ((HttpComponentsClientHttpRequestFactory) factory).setConnectTimeout(connectTimeout); + } + } + + return new UniversalClient(factory); + } + + public static class UniversalClient extends Client { + UniversalClient(ClientHttpRequestFactory factory) { + super(factory); + objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + List> messageConverters = new ArrayList>(); + MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + /* + * Make this converter process any kind of response, not only application/*json, + * which is the default behaviour + */ + converter.setSupportedMediaTypes(Collections.singletonList(MediaType.ALL)); + messageConverters.add(converter); + this.setMessageConverters(messageConverters); + } + } + + public static class NodeClient extends Client { + + public NodeClient(INodeIdentity node) { + + super(factory); + + objectMapper = new ObjectMapper(); + List> messageConverters = new ArrayList<>(); + MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); + StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); + FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); + ProtobufHttpMessageConverter protobufConverter = new ProtobufHttpMessageConverter(); + // ByteArrayHttpMessageConverter byteConverter = new + // ByteArrayHttpMessageConverter(); + jsonMessageConverter.setObjectMapper(objectMapper); + + setErrorHandler(new JSONResponseErrorHandler()); + messageConverters.add(jsonMessageConverter); + messageConverters.add(utf8); + messageConverters.add(formHttpMessageConverter); + messageConverters.add(protobufConverter); + // messageConverters.add(byteConverter); + setMessageConverters(messageConverters); + this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); + this.authorizationToken = node.getId(); + } + } + + /** + * Send an authentication request to a hub for an engine. + * + * @param url + * @param request + * @return the response. If not authenticated, throw a + * KlabAuthorizationException. If timeout, return null. + */ + public EngineAuthenticationResponse authenticateEngine(String url, EngineAuthenticationRequest request) { + return post(url + API.HUB.AUTHENTICATE_ENGINE, request, EngineAuthenticationResponse.class); + } + + /** + * Send an authentication request to a hub for a node. + * + * @param url + * @param request + * @return the response. If not authenticated, throw a + * KlabAuthorizationException. If timeout, return null. + */ + public NodeAuthenticationResponse authenticateNode(String url, NodeAuthenticationRequest request) { + return post(url + API.HUB.AUTHENTICATE_NODE, request, NodeAuthenticationResponse.class); + } + + /** + * Check an engine's heartbeat. + * + * @param url base engine/node URL + * @return true if alive + */ + public boolean ping(String url, String endpoint) { + try { + ResponseEntity response = basicTemplate.exchange(url + endpoint, HttpMethod.GET, + new HttpEntity(null, null), Object.class); + return response.getStatusCodeValue() == 200; + } catch (Throwable e) { + return false; + } + } + + public boolean ping(String url) { + return ping(url, "/actuator/health"); + } + + public boolean pingService(String url) { + return ping(url, "/public/status"); + } + + private class JSONResponseErrorHandler implements ResponseErrorHandler { + + @Override + public void handleError(ClientHttpResponse response) throws IOException { + } + + @Override + public boolean hasError(ClientHttpResponse response) throws IOException { + HttpStatus status = response.getStatusCode(); + HttpStatus.Series series = status.series(); + return (HttpStatus.Series.CLIENT_ERROR.equals(series) || HttpStatus.Series.SERVER_ERROR.equals(series)); + } + } + + /** + * Add option to set both headers, Authorization and Authentication + * + * @param headers the headers object to set the new headers + */ + private void setAuthTokens(HttpHeaders headers) { + if (authorizationToken != null) { + headers.set(HttpHeaders.AUTHORIZATION, authorizationToken); + } + if (authenticationToken != null) { + headers.add("Authentication", authenticationToken); + } + } + + /** + * Interceptor to add user agents and ensure that the authorization token gets + * in. + * + * @author ferdinando.villa + * + */ + public class AuthorizationInterceptor implements ClientHttpRequestInterceptor { + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) + throws IOException { + + HttpRequestWrapper requestWrapper = new HttpRequestWrapper(request); + HttpHeaders headers = requestWrapper.getHeaders(); + headers.set("Accept", "application/json"); + headers.set("X-User-Agent", "k.LAB " + Version.CURRENT); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + return execution.execute(requestWrapper, body); + } + } + + private void setup() { + + objectMapper = new ObjectMapper(); + List> messageConverters = new ArrayList<>(); + MappingJackson2HttpMessageConverter jsonMessageConverter = new MappingJackson2HttpMessageConverter(); + StringHttpMessageConverter utf8 = new StringHttpMessageConverter(Charset.forName("UTF-8")); + FormHttpMessageConverter formHttpMessageConverter = new FormHttpMessageConverter(); + // ByteArrayHttpMessageConverter byteConverter = new + // ByteArrayHttpMessageConverter(); + jsonMessageConverter.setObjectMapper(objectMapper); + + setErrorHandler(new JSONResponseErrorHandler()); + messageConverters.add(jsonMessageConverter); + messageConverters.add(utf8); + messageConverters.add(formHttpMessageConverter); + // messageConverters.add(byteConverter); + setMessageConverters(messageConverters); + this.setInterceptors(Collections.singletonList(new AuthorizationInterceptor())); + } + + private Client(ClientHttpRequestFactory factory) { + super(factory); + this.basicTemplate = new RestTemplate(); + setup(); + } + + private Client(Client client) { + super(factory); + this.basicTemplate = client.basicTemplate; + this.objectMapper = client.objectMapper; + this.endpoints = client.endpoints; + this.authorizationToken = client.authorizationToken; + this.authenticationToken = client.authenticationToken; + this.contentType = client.contentType; + setup(); + } + + private Client() { + super(factory); + } + + public Client withContentType(MediaType contentType) { + Client ret = new Client(this); + ret.contentType = contentType; + return ret; + } + + /** + * Return a client with authorization set to the passed object. + * + * @param authorizer any identity. + * @return a new + */ + @Override + public Client onBehalfOf(IIdentity authorizer) { + + Client ret = new Client(this); + ret.objectMapper = this.objectMapper; + ret.authorizationToken = authorizer.getId(); + return ret; + } + + public Client withAuthorization(String authorization) { + + Client ret = new Client(this); + ret.objectMapper = this.objectMapper; + ret.authorizationToken = authorization; + return ret; + } + + public Client withAuthentication(String authentication) { + + Client ret = new Client(); + ret.objectMapper = this.objectMapper; + ret.authenticationToken = authentication; + return ret; + } + + @SuppressWarnings("unchecked") + @Override + public T post(String url, Object data, Class cls) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + if (data != null) { + headers.setContentType(contentType); + } + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + + HttpEntity entity = new HttpEntity<>(data, headers); + + try { + + ResponseEntity response = exchange(url, HttpMethod.POST, entity, Object.class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); + case 500: + throw new KlabResourceAccessException("internal: request " + url + " caused a remote server error"); + } + + if (response.getBody() == null) { + return null; + } + if (response.getBody() instanceof Map && ((Map) response.getBody()).containsKey("exception") + && ((Map) response.getBody()).get("exception") != null) { + + Map map = (Map) response.getBody(); + Object exception = map.get("exception"); + // Object path = map.get("path"); + Object message = map.get("message"); + // Object error = map.get("error"); + + dumpRequest(url, headers, data); + throw new KlabIOException("remote exception: " + (message == null ? exception : message)); + } + + if (cls.isAssignableFrom(response.getBody().getClass())) { + return (T) response.getBody(); + } + + try { + return objectMapper.convertValue(response.getBody(), cls); + } catch (Throwable t) { + System.out.println("Unrecognized response: " + response.getBody()); + throw t; + } + } catch (RestClientException e) { + + System.out.println("REST exception: " + e.getMessage()); + dumpRequest(url, headers, data); + throw new KlabIOException(e); + } + } + + private void dumpRequest(String url, HttpHeaders headers, Object data) { + + System.out.println("Endpoint: " + url); + System.out.println("Headers:"); + for (String header : headers.keySet()) { + System.out.println(" " + header + " = " + headers.get(header)); + } + System.out.println("Data:\n" + printAsJson(data)); + + } + + public void setUrl(String... url) { + if (url == null || url.length == 0) { + this.endpoints.clear(); + } else { + for (String u : url) { + this.endpoints.add(u); + } + } + } + + private String pickEndpoint() { + // TODO periodically check URLs and choose the first that responds (or the one + // with the smallest load) + return this.endpoints.isEmpty() ? null : this.endpoints.iterator().next(); + } + + @Override + public boolean getDownload(String url, File output) { + + url = checkEndpoint(url); + + RestTemplate restTemplate = new RestTemplate(); + restTemplate.getMessageConverters().add(new ByteArrayHttpMessageConverter()); + + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + + HttpEntity entity = new HttpEntity(headers); + + // HttpHeaders headers = new HttpHeaders(); + // headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM)); + // HttpEntity entity = new HttpEntity<>(headers); + + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, entity, byte[].class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); + } + + if (response.getBody() == null) { + return false; + } + if (response.getStatusCode() == HttpStatus.OK) { + try { + Files.write(output.toPath(), response.getBody()); + } catch (IOException e) { + throw new KlabIOException(e); + } + } + + return true; + } + + private String checkEndpoint(String url) { + if (!url.toLowerCase().startsWith("http")) { + String ep = pickEndpoint(); + if (ep != null) { + url = ep + (ep.endsWith("/") || url.startsWith("/") ? "" : "/") + url; + } + } + return url; + } + + /** + * Issue a DELETE request. Called remove to avoid conflict with super. + * + * @param url + * @param parameters + * @return + */ + public Object remove(String url, Object... parameters) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + HttpEntity entity = new HttpEntity<>(headers); + + if (parameters != null) { + String params = ""; + for (int i = 0; i < parameters.length; i++) { + String key = parameters[i].toString(); + String nakedKey = key; + String val = parameters[++i].toString(); + if (!(key.startsWith("{") && key.endsWith("}"))) { + key = "{" + key + "}"; + } else { + nakedKey = key.substring(1, key.length() - 1); + } + if (url.contains(key)) { + url = url.replace(key, val); + } else { + params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + Escape.forURL(val); + } + } + if (!params.isEmpty()) { + url += "?" + params; + } + } + + ResponseEntity response = exchange(url, HttpMethod.DELETE, entity, Object.class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); + case 503: + throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); + } + + return response.getBody(); + } + + /** + * Instrumented for header communication and error parsing + * + * @param url + * @param cls + * @return the deserialized result + */ + @Override + @SuppressWarnings({ "unchecked" }) + public T get(String url, Class cls, Object... parameters) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", cls.equals(String.class) ? "text/plain" : "application/json"); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + // add user-agent to avoid 403 from Nominatim + headers.set(HttpHeaders.USER_AGENT, KLAB_USER_AGENT); + setAuthTokens(headers); + HttpEntity entity = new HttpEntity<>(headers); + + if (parameters != null) { + String params = ""; + for (int i = 0; i < parameters.length; i++) { + String key = parameters[i].toString(); + String nakedKey = key; + Object nextPar = parameters[++i]; + String val = nextPar == null ? null : nextPar.toString(); + + if (val == null) { + continue; + } + + if (!(key.startsWith("{") && key.endsWith("}"))) { + key = "{" + key + "}"; + } else { + nakedKey = key.substring(1, key.length() - 1); + } + if (url.contains(key)) { + url = url.replace(key, val); + } else { + params += (params.isEmpty() ? "" : "&") + nakedKey + "=" + /* Escape.forURL( */val/* ) */; + } + } + if (!params.isEmpty()) { + url += "?" + params; + } + } + + ResponseEntity response = null; + if (cls.isArray()) { + response = exchange(url, HttpMethod.GET, entity, Object.class); + } else if (String.class.equals(cls)) { + response = basicTemplate.exchange(url, HttpMethod.GET, entity, String.class); + } else /* if (Map.class.isAssignableFrom(cls)) */ { + response = exchange(url, HttpMethod.GET, entity, Map.class); + } + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not recognized"); + case 406: + case 503: + throw new KlabInternalErrorException("internal: request " + url + " caused a server error"); + } + + if (response.getBody() instanceof Map) { + + Object exception = ((Map) response.getBody()).get("exception"); + Object error = ((Map) response.getBody()).get("error"); + + if (exception != null || error != null) { + Object message = ((Map) response.getBody()).get("message"); + // Object error = response.getBody().get("error"); + throw new KlabIOException( + "remote exception: " + (message == null ? (exception == null ? error : exception) : message)); + } + + return objectMapper.convertValue(response.getBody(), cls); + + } else if (response.getBody() instanceof List && cls.isArray()) { + + List list = (List) response.getBody(); + Object ret = Array.newInstance(cls.getComponentType(), (((List) response.getBody()).size())); + for (int i = 0; i < list.size(); i++) { + Object object = list.get(i); + if (object instanceof Map) { + object = objectMapper.convertValue(object, cls.getComponentType()); + } + Array.set(ret, i, object); + } + + return (T) ret; + + } else if (response.getBody() != null && cls.isAssignableFrom(response.getBody().getClass())) { + return (T) response.getBody(); + } + + return null; + } + + @Override + public T postFile(String url, File file, Class cls) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", new FileSystemResource(file)); + HttpEntity> entity = new HttpEntity<>(body, headers); + + try { + + final RestTemplate restTemplate = new RestTemplate(); + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + // without this, large files will eat up the heap + requestFactory.setBufferRequestBody(false); + restTemplate.setRequestFactory(requestFactory); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class); + + switch (response.getStatusCodeValue()) { + case 302: + case 401: + case 403: + throw new KlabAuthorizationException("unauthorized request " + url); + case 404: + throw new KlabInternalErrorException("internal: request " + url + " was not accepted"); + } + + if (response.getBody().containsKey("exception") && response.getBody().get("exception") != null) { + Object exception = response.getBody().get("exception"); + // Object path = response.getBody().get("path"); + Object message = response.getBody().get("message"); + // Object error = response.getBody().get("error"); + throw new KlabIOException("remote exception: " + (message == null ? exception : message)); + } + + return objectMapper.convertValue(response.getBody(), cls); + + } catch (RestClientException e) { + throw new KlabIOException(e); + } + + } + + public static String printAsJson(Object object) { + + ObjectMapper om = new ObjectMapper(); + om.enable(SerializationFeature.INDENT_OUTPUT); // pretty print + om.enable(SerializationFeature.WRITE_NULL_MAP_VALUES); // pretty print + om.enable(SerializationFeature.WRITE_SINGLE_ELEM_ARRAYS_UNWRAPPED); // pretty print + om.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); + + try { + return om.writeValueAsString(object); + } catch (Exception e) { + throw new IllegalArgumentException("serialization failed: " + e.getMessage()); + } + } + + @Override + public boolean put(String url, Object data) { + + url = checkEndpoint(url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Accept", "application/json"); + if (data != null) { + headers.setContentType(MediaType.APPLICATION_JSON); + } + headers.set(KLAB_VERSION_HEADER, Version.CURRENT); + setAuthTokens(headers); + + try { + + super.put(url, data); + return true; + + } catch (RestClientException e) { + System.out.println("REST exception: " + e.getMessage()); + dumpRequest(url, headers, data); + return false; + } + + } } diff --git a/klab.commons/pom.xml b/klab.commons/pom.xml index 38e8bb8d15..092ea4309e 100644 --- a/klab.commons/pom.xml +++ b/klab.commons/pom.xml @@ -1,6 +1,6 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -33,13 +33,11 @@ com.fasterxml.jackson.core jackson-annotations - 2.15.2 com.fasterxml.jackson.core jackson-databind - 2.15.2 diff --git a/klab.engine/pom.xml b/klab.engine/pom.xml index fc3b6f3bd4..24b36110d7 100644 --- a/klab.engine/pom.xml +++ b/klab.engine/pom.xml @@ -135,12 +135,10 @@ com.fasterxml.jackson.core jackson-databind - 2.14.3 com.fasterxml.jackson.core jackson-core - 2.14.3 @@ -148,13 +146,11 @@ org.apache.poi poi - 5.4.1 org.apache.poi poi-ooxml - 5.4.1 @@ -336,6 +332,10 @@ org.locationtech.jts jts-core + @@ -527,7 +527,6 @@ com.fasterxml.jackson.module jackson-module-jsonSchema - ${jackson-version} @@ -572,7 +571,6 @@ com.fasterxml.jackson.datatype jackson-datatype-joda - ${jackson-version} org.integratedmodelling @@ -889,7 +887,6 @@ commons-io commons-io - 2.14.0 org.apache.commons diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java index 1fd8b6d6e6..6257a2d90e 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/events/EventBuilder.java @@ -21,7 +21,7 @@ * find the earliest start among them. Then reconstruct based on that time instead of the time at the call. * * 2. For each layer: - * 2.1 use JAI vectorizer and tag each shape in a layer with a progressive ID and rasterize the ID over a new layer. + * 2.1 use ImageN vectorizer and tag each shape in a layer with a progressive ID and rasterize the ID over a new layer. * 2.2 if there is a previous layer, connect the shapes that overlap. Shapes that don't (or end of observations) causes * event storage. * diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java index 4298d7282f..4ae6a200a1 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Envelope.java @@ -11,8 +11,8 @@ import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.utils.Pair; import org.locationtech.jts.geom.Geometry; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.operation.TransformException; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.operation.TransformException; public class Envelope implements IEnvelope { @@ -59,21 +59,21 @@ public boolean equals(Object obj) { } public static Envelope create(org.locationtech.jts.geom.Envelope envelope, Projection projection) { - Envelope ret = new Envelope(); + Envelope ret = new Envelope(); ret.envelope = new ReferencedEnvelope(envelope, projection.getCoordinateReferenceSystem()); ret.projection = projection; return ret; } - public static Envelope create(org.opengis.geometry.Envelope envelope, Projection projection) { - Envelope ret = new Envelope(); + public static Envelope create(org.geotools.api.geometry.Bounds envelope, Projection projection) { + Envelope ret = new Envelope(); ret.envelope = new ReferencedEnvelope(envelope); ret.projection = projection; return ret; } public static Envelope create(ReferencedEnvelope envelope) { - Envelope ret = new Envelope(); + Envelope ret = new Envelope(); ret.envelope = envelope; ret.projection = Projection.create(envelope.getCoordinateReferenceSystem()); return ret; @@ -89,7 +89,7 @@ public Envelope copy() { } public static Envelope create(ReferencedEnvelope envelope, boolean swapXY) { - Envelope ret = new Envelope(); + Envelope ret = new Envelope(); ret.envelope = swapXY ? new ReferencedEnvelope(envelope.getMinY(), envelope.getMaxY(), envelope.getMinX(), envelope.getMaxX(), envelope.getCoordinateReferenceSystem()) @@ -257,7 +257,7 @@ public Pair getResolutionForZoomLevel(int roundTo, double multi public int getScaleRank() { if (this.scaleRank == null) { - Envelope envelope = transform(Projection.getLatLon(), true); + Envelope envelope = transform(Projection.getLatLon(), true); int zoomLevel; double latDiff = envelope.getHeight(); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java index fd12f09de4..f3d74ed0a6 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Grid.java @@ -46,7 +46,7 @@ import org.integratedmodelling.klab.utils.MultidimensionalCursor; import org.integratedmodelling.klab.utils.Pair; import org.integratedmodelling.klab.utils.Range; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; /** * diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java index 8a3fa6f07b..a52fbff509 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/extents/Projection.java @@ -3,7 +3,7 @@ import javax.measure.Unit; -import org.geotools.geometry.DirectPosition2D; +import org.geotools.geometry.Position2D; import org.geotools.referencing.CRS; import org.geotools.referencing.CRS.AxisOrder; import org.integratedmodelling.klab.api.observations.scale.space.IProjection; @@ -11,10 +11,10 @@ import org.integratedmodelling.klab.components.geospace.utils.WGS84; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabValidationException; -import org.opengis.geometry.DirectPosition; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; -import org.opengis.referencing.operation.MathTransform; +import org.geotools.api.geometry.Position; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.operation.MathTransform; public class Projection implements IProjection { @@ -235,7 +235,7 @@ public double[] transformCoordinate(double[] coordinate, IProjection other) { if (!this.equals(other)) { try { MathTransform transform = CRS.findMathTransform(((Projection) other).crs, this.crs); - DirectPosition position = transform.transform(new DirectPosition2D(ret[0], ret[1]), null); + Position position = transform.transform(new Position2D(ret[0], ret[1]), null); ret = new double[] { position.getCoordinate()[0], position.getCoordinate()[1] }; } catch (Exception e) { throw new KlabInternalErrorException(e); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java index 5e9d6884ca..abe03db129 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/NeighborhoodResolver.java @@ -167,7 +167,7 @@ public IState resolve(IState target, IContextualizationScope context) throws Kla } /* - * build offset mask for quick addressing TODO can use kernels from JAI tools + * build offset mask for quick addressing TODO can use kernels from ImageN tools */ this.maskSize = hCells * 2 + 1; this.offsetMask = new Pair[maskSize][maskSize]; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java index c4fdca9597..a6a2c854d1 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/PolygonInstantiatorJAI.java @@ -55,8 +55,8 @@ import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Point; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.type.Name; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.type.Name; /** * diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java index 8f294978e3..8366a38ba1 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/processing/Rasterizer.java @@ -14,6 +14,7 @@ import org.geotools.geometry.jts.Geometries; import org.geotools.process.vector.VectorToRasterProcess; +import org.integratedmodelling.klab.api.observations.scale.space.Direction; import org.integratedmodelling.klab.api.observations.scale.space.IGrid; import org.integratedmodelling.klab.api.observations.scale.space.IShape; import org.integratedmodelling.klab.components.geospace.extents.Grid; @@ -88,8 +89,6 @@ public class Rasterizer { // this is used only for the getCoordinates() operation private BufferedImage coordinateRaster = null; private Graphics2D coordinateGraphics; - private VectorToRasterProcess gtp; - private GeometryFactory geoFactory = new GeometryFactory(); /** * @@ -309,10 +308,13 @@ private void drawGeometry(Geometry geometry, Color color) { coordGridX = new int[coords.length]; coordGridY = new int[coords.length]; } + double yRes = extent.getCellHeight(); + double xRes = extent.getCellWidth(); + double west = extent.getWest(); + double south = extent.getSouth(); for (int n = 0; n < coords.length; n++) { - coordGridX[n] = (int) (((coords[n].x - extent.getWest()) / extent.getCellWidth())); - coordGridY[n] = this.raster.getHeight() - - (int) (((coords[n].y - extent.getSouth()) / extent.getCellHeight())); + coordGridX[n] = (int) (((coords[n].x - west) / xRes)); + coordGridY[n] = this.raster.getHeight() - (int) (((coords[n].y - south) / yRes)); } if (geometry.getClass().equals(Polygon.class)) { @@ -322,7 +324,12 @@ private void drawGeometry(Geometry geometry, Color color) { } else if (geometry.getClass().equals(LineString.class)) { graphics.drawPolyline(coordGridX, coordGridY, coords.length); } else if (geometry.getClass().equals(Point.class)) { - graphics.drawLine(coordGridX[0], coordGridY[0], coordGridX[0], coordGridY[0]); + double cellWest = extent.snapX(coords[0].x, Direction.LEFT); + double centerX = cellWest + xRes/2.0; + double cellSouth = extent.snapY(coords[0].y, Direction.BOTTOM); + double centerY = cellSouth + yRes/2.0; + long[] gc = extent.getGridCoordinatesAt(centerX, centerY); + graphics.drawLine((int)gc[0], (int)gc[1], (int)gc[0], (int)gc[1]); } } diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java index 76da1f3d99..6f2bcb831d 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/GeotoolsUtils.java @@ -19,34 +19,35 @@ import java.util.Set; import java.util.function.Function; -import javax.media.jai.ImageLayout; -import javax.media.jai.JAI; -import javax.media.jai.ParameterBlockJAI; -import javax.media.jai.RasterFactory; -import javax.media.jai.RenderedOp; -import javax.media.jai.iterator.RandomIter; -import javax.media.jai.iterator.RandomIterFactory; -import javax.media.jai.iterator.RectIterFactory; -import javax.media.jai.iterator.WritableRectIter; - -import org.eclipse.lsp4j.AbstractTextDocumentRegistrationAndWorkDoneProgressOptions; +import org.eclipse.imagen.ImageLayout; +import org.eclipse.imagen.ImageN; +import org.eclipse.imagen.ParameterBlockImageN; +import org.eclipse.imagen.RasterFactory; +import org.eclipse.imagen.RenderedOp; +import org.eclipse.imagen.iterator.RandomIter; +import org.eclipse.imagen.iterator.RandomIterFactory; +import org.geotools.api.filter.expression.Literal; +import org.geotools.api.metadata.Identifier; +import org.geotools.api.metadata.citation.Citation; +import org.geotools.api.parameter.GeneralParameterValue; +import org.geotools.api.parameter.ParameterValueGroup; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.style.ColorMapEntry; +import org.geotools.api.style.RasterSymbolizer; import org.geotools.coverage.Category; import org.geotools.coverage.CoverageFactoryFinder; import org.geotools.coverage.GridSampleDimension; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.GridCoverageFactory; -import org.geotools.coverage.grid.io.AbstractGridCoverageWriter; import org.geotools.coverage.grid.io.AbstractGridFormat; import org.geotools.coverage.grid.io.imageio.GeoToolsWriteParams; import org.geotools.gce.geotiff.GeoTiffFormat; import org.geotools.gce.geotiff.GeoTiffWriteParams; import org.geotools.gce.geotiff.GeoTiffWriter; -import org.geotools.geometry.Envelope2D; import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.metadata.iso.citation.Citations; import org.geotools.referencing.CRS; -import org.geotools.styling.ColorMapEntry; -import org.geotools.styling.RasterSymbolizer; import org.geotools.swing.data.JFileDataStoreChooser; import org.geotools.util.factory.Hints; import org.hortonmachine.gears.utils.files.FileUtilities; @@ -60,7 +61,6 @@ import org.integratedmodelling.klab.api.observations.scale.space.ISpace; import org.integratedmodelling.klab.api.runtime.IContextualizationScope; import org.integratedmodelling.klab.api.services.IConfigurationService; -import org.integratedmodelling.klab.common.Geometry; import org.integratedmodelling.klab.components.geospace.extents.Grid; import org.integratedmodelling.klab.components.geospace.extents.Projection; import org.integratedmodelling.klab.components.geospace.extents.Space; @@ -73,14 +73,6 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.utils.Pair; -import org.jaitools.tiledimage.DiskMemImage; -import org.opengis.filter.expression.Literal; -import org.opengis.metadata.Identifier; -import org.opengis.metadata.citation.Citation; -import org.opengis.parameter.GeneralParameterValue; -import org.opengis.parameter.ParameterValueGroup; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; public enum GeotoolsUtils { @@ -113,7 +105,7 @@ public GridCoverage2D wrapStateInFloatCoverage(IState state, ILocator locator, F double east = grid.getEast(); double north = grid.getNorth(); CoordinateReferenceSystem crs = ((Projection) grid.getProjection()).getCoordinateReferenceSystem(); - Envelope2D writeEnvelope = new Envelope2D(crs, west, south, east - west, north - south); + ReferencedEnvelope writeEnvelope = ReferencedEnvelope.rect(west, south, east - west, north - south, crs); GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); GridCoverage2D coverage = factory.create("stateraster", ri, writeEnvelope); @@ -142,7 +134,7 @@ public GridCoverage2D getTemporaryFloatStorage(IScale scale) { double east = grid.getEast(); double north = grid.getNorth(); CoordinateReferenceSystem crs = ((Projection) grid.getProjection()).getCoordinateReferenceSystem(); - Envelope2D writeEnvelope = new Envelope2D(crs, west, south, east - west, north - south); + ReferencedEnvelope writeEnvelope = ReferencedEnvelope.rect(west, south, east - west, north - south, crs); GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); GridCoverage2D coverage = factory.create("stateraster", ri, writeEnvelope); @@ -173,7 +165,7 @@ public GridCoverage2D getTemporaryIntStorage(IScale scale) { double east = grid.getEast(); double north = grid.getNorth(); CoordinateReferenceSystem crs = ((Projection) grid.getProjection()).getCoordinateReferenceSystem(); - Envelope2D writeEnvelope = new Envelope2D(crs, west, south, east - west, north - south); + ReferencedEnvelope writeEnvelope = ReferencedEnvelope.rect(west, south, east - west, north - south, crs); GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); GridCoverage2D coverage = factory.create("stateraster", ri, writeEnvelope); @@ -680,124 +672,6 @@ public static WritableRaster createWritableRaster(int width, int height, Class= MIN_SIZE_NEED_TILING) { - isTiled = true; - - // This implementation supposes that tileWidth is - // equal to the width - // of the whole image - tileWidth = width; - - // actually (need improvements) tileHeight is given by - // the default tile size divided by the tileWidth - // multiplied by the - // sample size (in byte) - tileHeight = DEFAULT_TILE_SIZE / (tileWidth * sampleSizeByte); - - // if computed tileHeight is zero, it is setted to 1 - // as precaution - if (tileHeight < 1) { - tileHeight = 1; - } - - } else { - // If no Tiling needed, I set the tile sizes equal to the image - // sizes - tileWidth = width; - tileHeight = height; - } - - Envelope2D envelope = new Envelope2D(crs, 0, 0, width, height); - SampleModel sampleModel = new ComponentSampleModel(DataBuffer.TYPE_FLOAT, tileWidth, tileHeight, 1, tileWidth, - new int[]{0}); - DiskMemImage img = new DiskMemImage(width, height, sampleModel); - - WritableRectIter iter = RectIterFactory.createWritable(img, null); - do { - int x = 0; - do { - iter.setSample(x / tileWidth); - } while(!iter.nextPixelDone()); - } while(!iter.nextLineDone()); - - GridCoverageFactory factory = CoverageFactoryFinder.getGridCoverageFactory(null); - - GridCoverage2D gc = factory.create("bigtif", img, envelope); - - GeoTiffFormat fmt = new GeoTiffFormat(); - // getting the write parameters - final GeoTiffWriteParams wp = new GeoTiffWriteParams(); - - // setting compression to Deflate - wp.setCompressionMode(GeoTiffWriteParams.MODE_EXPLICIT); - wp.setCompressionType("Deflate"); - wp.setCompressionQuality(0.75F); - - // setting the tile size to 256X256 - wp.setTilingMode(GeoToolsWriteParams.MODE_EXPLICIT); - wp.setTiling(256, 256); - - // setting the write parameters for this geotiff - final ParameterValueGroup[] params = {fmt.getWriteParameters()}; - - params[0].parameter(AbstractGridFormat.GEOTOOLS_WRITE_PARAMS.getName().toString()).setValue(wp); - - AbstractGridCoverageWriter writer = new GeoTiffWriter(file); - writer.write(gc/* .view(ViewType.GEOPHYSICS) */, params); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - /** * Dump one or more states to rasters in the klab configuration folder. * @@ -1020,7 +894,7 @@ public static void main(String[] args) throws Exception { return; } - ParameterBlockJAI pb = new ParameterBlockJAI("Constant"); + ParameterBlockImageN pb = new ParameterBlockImageN("Constant"); pb.setParameter("width", (float) IMAGE_WIDTH); pb.setParameter("height", (float) IMAGE_HEIGHT); pb.setParameter("bandValues", new Double[]{0.0d}); @@ -1031,9 +905,9 @@ public static void main(String[] args) throws Exception { layout.setTileWidth(tileWidth); layout.setTileHeight(tileWidth); - RenderingHints hints = new RenderingHints(JAI.KEY_IMAGE_LAYOUT, layout); + RenderingHints hints = new RenderingHints(ImageN.KEY_IMAGE_LAYOUT, layout); - RenderedOp image = JAI.create("Constant", pb, hints); + RenderedOp image = ImageN.create("Constant", pb, hints); GeoTiffWriter writer = new GeoTiffWriter(file, null); GridCoverageFactory factory = new GridCoverageFactory(); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java index df6093bd01..d4cae60133 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/utils/SpatialDisplay.java @@ -42,18 +42,18 @@ import org.geotools.map.Layer; import org.geotools.map.MapContent; import org.geotools.map.MapViewport; -import org.geotools.styling.FeatureTypeStyle; -import org.geotools.styling.Fill; -import org.geotools.styling.Graphic; -import org.geotools.styling.LineSymbolizer; -import org.geotools.styling.Mark; -import org.geotools.styling.PointSymbolizer; -import org.geotools.styling.PolygonSymbolizer; -import org.geotools.styling.Rule; +import org.geotools.api.style.FeatureTypeStyle; +import org.geotools.api.style.Fill; +import org.geotools.api.style.Graphic; +import org.geotools.api.style.LineSymbolizer; +import org.geotools.api.style.Mark; +import org.geotools.api.style.PointSymbolizer; +import org.geotools.api.style.PolygonSymbolizer; +import org.geotools.api.style.Rule; import org.geotools.styling.SLD; -import org.geotools.styling.Stroke; -import org.geotools.styling.Style; -import org.geotools.styling.StyleFactory; +import org.geotools.api.style.Stroke; +import org.geotools.api.style.Style; +import org.geotools.api.style.StyleFactory; import org.geotools.swing.JMapFrame; import org.integratedmodelling.klab.Concepts; import org.integratedmodelling.klab.api.data.ILocator; @@ -77,9 +77,9 @@ import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; -import org.opengis.feature.simple.SimpleFeature; -import org.opengis.feature.simple.SimpleFeatureType; -import org.opengis.filter.FilterFactory; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.filter.FilterFactory; /** * Generic spatial visualizer/debugger that is initialized with a spatial extent and you can just diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java index 60608cddac..ac3d3a2334 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/Renderer.java @@ -25,12 +25,12 @@ import org.geotools.factory.CommonFactoryFinder; import org.geotools.renderer.lite.RendererUtilities; import org.geotools.renderer.lite.gridcoverage2d.GridCoverageRenderer; -import org.geotools.styling.ColorMap; -import org.geotools.styling.ContrastEnhancement; -import org.geotools.styling.RasterSymbolizer; -import org.geotools.styling.ShadedRelief; +import org.geotools.api.style.ColorMap; +import org.geotools.api.style.ContrastEnhancement; +import org.geotools.api.style.RasterSymbolizer; +import org.geotools.api.style.ShadedRelief; import org.geotools.styling.StyleBuilder; -import org.geotools.styling.StyleFactory; +import org.geotools.api.style.StyleFactory; import org.integratedmodelling.klab.Concepts; import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Observations; @@ -57,7 +57,7 @@ import org.integratedmodelling.klab.utils.Pair; import org.integratedmodelling.klab.utils.Range; import org.integratedmodelling.klab.utils.Triple; -import org.opengis.style.ContrastMethod; +import org.geotools.api.style.ContrastMethod; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java index bad0f56e25..3ae4cc5252 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/geospace/visualization/raster/FloatRasterWrapper.java @@ -4,7 +4,7 @@ import java.awt.image.Raster; -import com.sun.media.jai.iterator.WrapperRI; +import org.eclipse.imagen.media.iterator.WrapperRI; /** * RenderedImage wrapper for {@link Raster}s. diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java index 2c4b039b13..403fd50118 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/network/algorithms/Project.java @@ -82,13 +82,13 @@ import org.geotools.coverage.grid.GridCoverageFactory; import org.geotools.coverage.grid.GridEnvelope2D; import org.geotools.feature.SchemaException; -import org.geotools.geometry.Envelope2D; +import org.geotools.geometry.jts.ReferencedEnvelope; //import org.geotools.graph.structure.Graph; //import org.geotools.graph.structure.Node; import org.geotools.referencing.CRS; import org.geotools.referencing.crs.DefaultEngineeringCRS; -import org.opengis.referencing.FactoryException; -import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.geotools.api.referencing.FactoryException; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; //import org.thema.common.Config; //import org.thema.common.JTS; //import org.thema.common.ProgressBar; @@ -297,7 +297,7 @@ public final class Project { // codes.remove((int)noData); // } // -// Envelope2D gZone = cov.getEnvelope2D(); +// ReferencedEnvelope gZone = cov.getEnvelope2D(); // zone = gZone.getBounds2D(); // CoordinateReferenceSystem crs = cov.getCoordinateReferenceSystem2D(); // if(crs instanceof DefaultEngineeringCRS) { @@ -320,7 +320,7 @@ public final class Project { // } // resolution = grid2space.getMatrixEntries()[0]; // -// Envelope2D extZone = new Envelope2D(crs, +// ReferencedEnvelope extZone = new ReferencedEnvelope(crs, // gZone.x-resolution, gZone.y-resolution, gZone.width+2*resolution, gZone.height+2*resolution); // // TreeMap envMap = new TreeMap<>(); @@ -1565,7 +1565,7 @@ public List getPatches() { // } // // GridCoverage2D gridCov = new GridCoverageFactory().create("rasterpatch", -// newRaster, new Envelope2D(getCRS(), zone)); +// newRaster, new ReferencedEnvelope(getCRS(), zone)); // IOImage.saveTiffCoverage(new File(dir, PATCH_RASTER), gridCov); // // TreeSet newCodes = new TreeSet<>(codes); @@ -1711,7 +1711,7 @@ public List getPatches() { // } // // GridCoverage2D gridCov = new GridCoverageFactory().create("rasterpatch", -// newRaster, new Envelope2D(getCRS(), zone)); +// newRaster, new ReferencedEnvelope(getCRS(), zone)); // IOImage.saveTiffCoverage(new File(dir, PATCH_RASTER), gridCov); // // TreeSet newCodes = new TreeSet<>(codes); @@ -1970,7 +1970,7 @@ public List getPatches() { // } // // GridEnvelope2D grid = cov.getGridGeometry().getGridRange2D(); -// Envelope2D env = cov.getEnvelope2D(); +// ReferencedEnvelope env = cov.getEnvelope2D(); // double res = env.getWidth() / grid.getWidth(); // if(res != resolution) { // throw new IllegalArgumentException(java.util.ResourceBundle.getBundle("org/thema/graphab/Bundle").getString("Resolution_does_not_match.")); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java b/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java index ab4c5d7e14..6017ac5ec2 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/components/processing/openbuildings/OpenBuildingsInstantiator.java @@ -41,8 +41,8 @@ import org.integratedmodelling.klab.utils.CamelCase; import org.integratedmodelling.klab.utils.Parameters; import org.locationtech.jts.geom.Geometry; -import org.opengis.feature.Property; -import org.opengis.feature.simple.SimpleFeature; +import org.geotools.api.feature.Property; +import org.geotools.api.feature.simple.SimpleFeature; public class OpenBuildingsInstantiator extends AbstractContextualizer implements IInstantiator, IExpression { private IDirectObservation contextSubject = null; diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java b/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java index 6a5348401f..ee585ce846 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/data/storage/ResourceCatalog.java @@ -172,7 +172,7 @@ public IResource put(String key, IResource value) { try { File resFile = new File(resourcePath + File.separator + "resource.json"); if (!resFile.exists() || resFile.lastModified() < ref.getResourceTimestamp()) { - FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8.name()); } } catch (IOException e) { throw new KlabIOException(e); @@ -329,7 +329,7 @@ public IResource move(IResource resource, IProject destinationProject) { } } FileUtils.writeStringToFile(new File(newData.getFirst() + File.separator + "resource.json"), - JsonUtils.printAsJson(newData.getSecond()), StandardCharsets.UTF_8); + JsonUtils.printAsJson(newData.getSecond()), StandardCharsets.UTF_8.name()); FileUtils.deleteDirectory(previousDir); resources.remove(resource.getUrn()); resources.put(resource.getUrn(), newData.getSecond()); @@ -371,7 +371,7 @@ public IResource update(IResource resource, String message) { try { File resFile = new File(resourcePath + File.separator + "resource.json"); if (!resFile.exists() || resFile.lastModified() < ref.getResourceTimestamp()) { - FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(resFile, JsonUtils.printAsJson(ref), StandardCharsets.UTF_8.name()); } } catch (IOException e) { throw new KlabIOException(e); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java index f8dbc13835..1a595663fe 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/APIObservationTask.java @@ -204,7 +204,9 @@ private long observe() { * make every observation */ for (IObservable observable : observables) { - + if (observable == null) { + continue; + } IObservation observation = null; ITask observationTask = ((ISubject) context).observe(observable); observationTask.addScenarios(scenarios); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java index f9288190a7..f3ca258371 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/engine/runtime/SessionState.java @@ -350,7 +350,7 @@ public Future submit(String urn, BiConsumer, IArtifact> obse */ IGeometry geom = getGeometry(); activity.setStart(System.currentTimeMillis()); - if (this.currentActivity.getGeometrySet() == null && geom != null) { + if (this.currentActivity != null && this.currentActivity.getGeometrySet() == null && geom != null) { this.currentActivity.setGeometrySet(geom.encode()); } activity.setActivityId(task.getId()); diff --git a/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java b/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java index 9cfc519de2..543ed36bc6 100644 --- a/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java +++ b/klab.engine/src/main/java/org/integratedmodelling/klab/scale/Scale.java @@ -455,19 +455,27 @@ public List getKimSpecification() { private class ScaleIterator implements Iterator { long offset = 0; + + private void moveToCovered() { + while (this.offset < size() && !isCovered(offset)) { + this.offset++; + } + } @Override public boolean hasNext() { + moveToCovered(); return offset < size(); } @Override public IScale next() { + moveToCovered(); IScale ret = new Scale(Scale.this, offset); this.offset++; - while (this.offset < size() && !isCovered(offset)) { - this.offset++; - } +// while (this.offset < size() && !isCovered(offset)) { +// this.offset++; +// } return ret; } } diff --git a/klab.engine/src/main/resources/knowledge/observation.owl b/klab.engine/src/main/resources/knowledge/observation.owl index 2b177c5675..2779e27bf1 100644 --- a/klab.engine/src/main/resources/knowledge/observation.owl +++ b/klab.engine/src/main/resources/knowledge/observation.owl @@ -1549,6 +1549,14 @@ Importantly, a deliberative agent can access history, although the full meaning + + + + + + + + @@ -1813,12 +1821,6 @@ Importantly, a deliberative agent can access history, although the full meaning - - - - - - + diff --git a/klab.hub/pom.xml b/klab.hub/pom.xml index ea463c3460..a7e249bca6 100644 --- a/klab.hub/pom.xml +++ b/klab.hub/pom.xml @@ -235,22 +235,19 @@ com.fasterxml.jackson.core jackson-databind - 2.12.4 com.fasterxml.jackson.core jackson-core - 2.12.4 + com.fasterxml.jackson.core jackson-annotations - 2.12.4 com.fasterxml.jackson.datatype jackson-datatype-jsr310 - 2.12.4 com.kjetland diff --git a/klab.node/pom.xml b/klab.node/pom.xml index 36321591a0..2d967d4b8e 100644 --- a/klab.node/pom.xml +++ b/klab.node/pom.xml @@ -85,7 +85,6 @@ com.fasterxml.jackson.datatype jackson-datatype-joda - ${jackson-version} - 28.0 - 0.10.13 + 34.4 + 0.11.3-SNAPSHOT 1.20.0 1.5.0 - 11 - 11 + 17 + 17 4.0.9 2.36.0 2.19.0 @@ -131,7 +131,7 @@ 5.5.1 2020.0.3 5.6.3 - 2.12.4 + 2.17.2 0.13.4 @@ -145,7 +145,10 @@ 2.2 2.0.9 0.1.23 + 2.29.14 24.0.4 + 5.4.1 + 2.14.0 @@ -293,4 +296,32 @@ + + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + commons-io + commons-io + ${commons-io.version} + + + + com.fasterxml.jackson + jackson-bom + ${jackson-version} + pom + import + + + + diff --git a/products/cli/pom.xml b/products/cli/pom.xml index 37531dd150..1928942edb 100644 --- a/products/cli/pom.xml +++ b/products/cli/pom.xml @@ -28,6 +28,47 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + strip-commons-io-from-oms + prepare-package + + + + + + + + + + + + + + + + + + + + run + + + + com.google.cloud.tools jib-maven-plugin diff --git a/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java b/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java index 832434e3aa..6de5b1e696 100644 --- a/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java +++ b/products/cli/src/test/java/org/integratedmodelling/klab/test/cli/WFSWithPreemptiveAuth.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; -import org.geotools.data.DataStore; +import org.geotools.api.data.DataStore; import org.geotools.data.wfs.WFSDataStore; import org.geotools.data.wfs.WFSDataStoreFactory; import org.geotools.http.SimpleHttpClient; diff --git a/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java b/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java index 609c42f993..7994029acc 100644 --- a/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java +++ b/products/semantic/src/main/java/org/integratedmodelling/klab/semantic/SemanticServerApplication.java @@ -3,7 +3,6 @@ import java.util.Arrays; import javax.annotation.PreDestroy; -import javax.inject.Singleton; import org.apache.catalina.startup.Tomcat; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -18,7 +17,6 @@ import org.springframework.web.client.RestTemplate; @Component -@Singleton @EnableAutoConfiguration @ComponentScan(basePackages = { /* * "org.integratedmodelling.klab.node.security",