From 110b6f1fc5be2b68bbafa51e72663a8873f8abb0 Mon Sep 17 00:00:00 2001 From: jayancv Date: Sat, 25 Mar 2023 00:48:58 +0530 Subject: [PATCH 1/8] Adding data replica catalog --- pom.xml | 4 + replica-catalog-api/client/pom.xml | 29 +++ .../airavata/ReplicaCatalogAPIClient.java | 177 +++++++++++++ .../airavata/ReplicaCatalogAPIClientTest.java | 129 ++++++++++ replica-catalog-api/pom.xml | 20 ++ replica-catalog-api/server/pom.xml | 83 +++++++ .../ReplicaCatalogApiServiceApplication.java | 13 + .../catalogapi/mapper/DataProductMapper.java | 47 ++++ .../catalogapi/mapper/DataReplicaMapper.java | 53 ++++ .../catalogapi/model/DataProductEntity.java | 163 ++++++++++++ .../model/DataProductMetadataEntity.java | 71 ++++++ .../model/DataProductMetadataPK.java | 76 ++++++ .../model/DataReplicaLocationEntity.java | 186 ++++++++++++++ .../model/DataReplicaMetadataEntity.java | 71 ++++++ .../model/DataReplicaMetadataPK.java | 74 ++++++ .../repository/DataProductRepository.java | 15 ++ .../DataReplicaLocationRepository.java | 36 +++ .../service/IReplicaCatalogService.java | 28 +++ .../service/ReplicaCatalogAPIService.java | 116 +++++++++ .../impl/ReplicaCatalogServiceImp.java | 121 +++++++++ .../mapper/ResourceStorageMapper.java | 54 ++++ .../resource/model/GenericResourceEntity.java | 96 ++++++++ .../resource/model/ResolveStorageEntity.java | 65 +++++ .../resource/model/S3StorageEntity.java | 98 ++++++++ .../resource/model/StorageSecretEntity.java | 74 ++++++ .../repository/GenericResourceRepository.java | 32 +++ .../repository/ResolveStorageRepository.java | 32 +++ .../repository/S3StorageRepository.java | 31 +++ .../repository/StorageSecretRepository.java | 31 +++ .../resource/service/IResourceService.java | 53 ++++ .../resource/service/ResourceAPIService.java | 56 +++++ .../resource/service/SQLIResourceService.java | 233 ++++++++++++++++++ .../sceret/mapper/ResourceSecretMapper.java | 33 +++ .../sceret/model/S3SecretEntity.java | 77 ++++++ .../sceret/repository/S3SecretRepository.java | 29 +++ .../sceret/service/ISecretService.java | 41 +++ .../sceret/service/SQLISecretService.java | 109 ++++++++ .../service/StorageSecretAPIService.java | 41 +++ .../src/main/resources/application.properties | 5 + .../service/ReplicaCatalogAPIServiceTest.java | 37 +++ replica-catalog-api/stubs/pom.xml | 90 +++++++ .../proto/catalogapi/ReplicaCatalogAPI.proto | 195 +++++++++++++++ .../catalogapi/ReplicaCatalogAPIService.proto | 79 ++++++ .../proto/resource/common/StorageCommon.proto | 185 ++++++++++++++ .../main/proto/resource/s3/S3Storage.proto | 76 ++++++ .../proto/secret/common/SecretCommon.proto | 112 +++++++++ .../main/proto/secret/s3/S3Credential.proto | 58 +++++ 47 files changed, 3534 insertions(+) create mode 100644 replica-catalog-api/client/pom.xml create mode 100644 replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java create mode 100644 replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java create mode 100644 replica-catalog-api/pom.xml create mode 100644 replica-catalog-api/server/pom.xml create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java create mode 100644 replica-catalog-api/server/src/main/resources/application.properties create mode 100644 replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java create mode 100644 replica-catalog-api/stubs/pom.xml create mode 100644 replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/resource/s3/S3Storage.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/secret/s3/S3Credential.proto diff --git a/pom.xml b/pom.xml index 2fd997b..d217819 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ under the License. data-catalog-api + replica-catalog-api @@ -47,6 +48,9 @@ under the License. 1.52.1 ${grpc.version} 6.0.53 + 3.1.1 + 2.14.1 + 5.5.1 diff --git a/replica-catalog-api/client/pom.xml b/replica-catalog-api/client/pom.xml new file mode 100644 index 0000000..bd56f10 --- /dev/null +++ b/replica-catalog-api/client/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + org.apache.airavata + replica-catalog-api + 0.1-SNAPSHOT + + + replica-catalog-api-client + + + + org.apache.airavata + replica-catalog-api-stubs + 0.1-SNAPSHOT + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + \ No newline at end of file diff --git a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java new file mode 100644 index 0000000..130aa65 --- /dev/null +++ b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java @@ -0,0 +1,177 @@ +package org.apache.airavata; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder;; +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +import org.apache.airavata.replicacatalog.catalog.service.ReplicaCatalogAPIServiceGrpc; +import org.apache.airavata.replicacatalog.catalog.stubs.*; +import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommon; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; +import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretWrapper; +import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReplicaCatalogAPIClient { + private static final Logger logger = LoggerFactory.getLogger(ReplicaCatalogAPIClient.class); + + private final ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub blockingApiStub; + + public ReplicaCatalogAPIClient(Channel channel) { + blockingApiStub = ReplicaCatalogAPIServiceGrpc.newBlockingStub(channel); + } + + public ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub getBlockingApiStub() { + return blockingApiStub; + } + + public StorageCommonServiceGrpc.StorageCommonServiceBlockingStub getBlockingStorageStub(Channel channel) { + return StorageCommonServiceGrpc.newBlockingStub(channel); + } + + // public GenericResourceServiceGrpc.GenericResourceServiceBlockingStub getBlockingResourceStub( Channel channel ) + // { + // return GenericResourceServiceGrpc.newBlockingStub( channel ); + // + // } + + public SecretCommonServiceGrpc.SecretCommonServiceBlockingStub getBlockingSecretStub(Channel channel) { + return SecretCommonServiceGrpc.newBlockingStub(channel); + } + + public static void main(String[] args) throws InterruptedException { + String target = "localhost:6565"; + + /* + --- Sample scenario 1--- + Airavata MFT copied a 100GB astrological data file to Amazon S3 bucket + User can access that S3 bucket data file via Replica catalog details + + 1) Register Data Product + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + ManagedChannel channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build(); + try { + ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient(channel); + + DataProduct dataProduct = DataProduct.newBuilder() + .setDataProductType(DataProductType.FILE) + .setProductName("ASTRO") + .setProductUri("613") + .setProductDescription("Astro Data").build(); + DataProduct productResult = client.createDataProduct(dataProduct); + + DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder() + .setReplicaName("ASTRO S3") + .setReplicaDescription("S3 replica") + .setDataProductId(productResult.getProductUri()) + .setCreationTime(System.currentTimeMillis()).build(); + DataReplicaLocation replicaResult = client.createReplicaLocation(replicaLocation); + + S3Storage storage = S3Storage.newBuilder() + .setName("ASTRO S3") + .setBucketName("arn:aws:s3:::mftjayan") + .setRegion("us-east-1") + .setEndpoint("https://s3.us-east-1.amazonaws.com").build(); + + GenericResource resource = GenericResource.newBuilder() + .setStorage(StorageWrapper.newBuilder().setS3Storage(storage).build()) + .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); + GenericResource resourceResult = client.createStorage(channel, resource); + + S3Secret secret = S3Secret.newBuilder() + .setAccessKey("access") + .setSecretKey("secKey") + .setSessionToken("token").build(); + StorageSecret storageSecret = StorageSecret.newBuilder() + .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) + .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3).build(); + + StorageSecret secretResult = client.createSecret(channel, storageSecret); + + SecretForStorage secretForStorage = SecretForStorage.newBuilder() + .setStorageId(resourceResult.getStorage().getS3Storage().getStorageId()) + .setSecretId(secretResult.getSecretId()).setStorageType(StorageType.S3).build(); + + client.createCommonStorage(channel, secretForStorage); + + + System.out.println( + MessageFormat.format("Created data product with id [{0}]", replicaResult.getDataReplicaId())); + + /* + --- Sample scenario 2--- + Airavata MFT copied a 1000GB astrological data Folder which contains 100 data files to Amazon S3 bucket + User can access that S3 bucket data folder via Replica catalog details + + 1) Register Data Product + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + + } finally { + channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + } + } + + + public DataProduct createDataProduct(DataProduct dataProduct) { + DataProductCreateRequest request = DataProductCreateRequest.newBuilder().setDataProduct(dataProduct).build(); + DataProductCreateResponse response = blockingApiStub.registerDataProduct(request); + return response.getDataProduct(); + } + + public DataReplicaLocation createReplicaLocation(DataReplicaLocation replicaLocation) { + DataReplicaCreateRequest request = + DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); + DataReplicaCreateResponse response = blockingApiStub.registerReplicaLocation(request); + return response.getDataReplica(); + } + + + public SecretForStorage createCommonStorage(Channel channel, SecretForStorage secretForStorage) { + + SecretForStorage response = getBlockingStorageStub(channel).registerSecretForStorage(secretForStorage); + return response; + } + + public GenericResource createStorage(Channel channel, GenericResource resource) { + StorageType storageType = null; + switch (resource.getStorage().getStorageCase().getNumber()) { + case StorageWrapper.S3STORAGE_FIELD_NUMBER: + storageType = StorageType.S3; + break; + + default: + break; + } + GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder(). + setResource(resource).setStorageType(storageType).build(); + GenericResource response = getBlockingStorageStub(channel).createGenericResource(request); + return response; + } + + public StorageSecret createSecret(Channel channel, StorageSecret resource) { + StorageSecret response = getBlockingSecretStub(channel).registerSecret(resource); + return response; + } + + +} \ No newline at end of file diff --git a/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java new file mode 100644 index 0000000..5b2a5b9 --- /dev/null +++ b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java @@ -0,0 +1,129 @@ +//package org.apache.airavata; +// +// +//import java.text.MessageFormat; +//import java.util.concurrent.TimeUnit; +// +//import io.grpc.Channel; +//import io.grpc.ManagedChannel; +//import io.grpc.ManagedChannelBuilder; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateRequest; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateResponse; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataProductType; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; +//import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +//import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; +//import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +//import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +//import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +//import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommon; +//import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; +//import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; +//import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +//import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; +// +//class ReplicaCatalogAPIClientTest +//{ +// public void testCase1() +// { +// String target = "localhost:6565"; +// +// /* +// --- Sample scenario --- +// Airavata MFT copied a 1000GB astrological data Folder which contains 100 data files to Amazon S3 bucket +// User can access that S3 bucket data folder via Replica catalog details +// +// 1) Register Data Product +// 2) Register Replica Location +// 3) Register Location storage details +// 4) Register location storage credentials +// +// */ +// ManagedChannel channel = ManagedChannelBuilder.forTarget( target ).usePlaintext().build(); +// try +// { +// ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient( channel ); +// +// DataProduct dataProduct = DataProduct.newBuilder().setDataProductType( DataProductType.FILE ) +// .setProductName( "ASTRO" ).setProductUri( "613" ).setProductDescription( "Astro Data" ).build(); +// DataProduct productResult = client.createDataProduct( dataProduct ); +// +// DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder().setReplicaName( "ASTRO S3" ) +// .setReplicaDescription( "S3 replica" ).setDataProductId( productResult.getProductUri() ).build(); +// +// DataReplicaLocation replicaResult = client.createReplicaLocation( replicaLocation ); +// +// GenericResource resource = client.createStorage( channel, null ); +// +// StorageSecret secret = client.createSecret( channel, null ); +// +// client.createCommonStorage( channel, null ); +// +// +// DataReplicaLocation result = client.createReplicaLocation( replicaLocation ); +// System.out.println( +// MessageFormat.format( "Created data product with id [{0}]", result.getDataReplicaId() ) ); +// +// } +// finally +// { +// channel.shutdownNow().awaitTermination( 5, TimeUnit.SECONDS ); +// } +// } +// +// public DataReplicaLocation createReplicaLocation( DataReplicaLocation replicaLocation ) +// { +// DataReplicaCreateRequest request = +// DataReplicaCreateRequest.newBuilder().setDataReplica( replicaLocation ).build(); +// DataReplicaCreateResponse response = blockingApiStub.registerReplicaLocation( request ); +// return response.getDataReplica(); +// } +// +// public DataProduct createDataProduct( DataProduct dataProduct ) +// { +// DataProductCreateRequest request = DataProductCreateRequest.newBuilder().setDataProduct( dataProduct ).build(); +// DataProductCreateResponse response = blockingApiStub.registerDataProduct( request ); +// return response.getDataProduct(); +// } +// +// public SecretForStorage createCommonStorage( Channel channel, StorageCommon storageCommon ) +// { +// SecretForStorage request = SecretForStorage.newBuilder() +// .setStorageId( "-1" ).setSecretId( "-1" ).setStorageType( StorageType.S3 ).build(); +// SecretForStorage response = getBlockingStorageStub( channel ).registerSecretForStorage( request ); +// return response; +// } +// +// public GenericResource createStorage( Channel channel, GenericResource resource ) +// { +// S3Storage storage = S3Storage.newBuilder().setStorageId( "-1" ).setBucketName( "bucket" ).setName( "Name" ) +// .setEndpoint( "end" ).setRegion( "us-west" ).build(); +// +// GenericResource resource1 = GenericResource.newBuilder().setResourceId( "-1" ).setS3Storage( storage ) +// .setFile( FileResource.newBuilder().setResourcePath( "path" ).build() ).build(); +// +// GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder().setStorageId( "-1" ). +// setResource( resource1 ).setStorageType( GenericResourceCreateRequest.StorageType.S3 ).build(); +// GenericResource response = getBlockingResourceStub( channel ).createGenericResource( request ); +// return response; +// } +// +// public StorageSecret createSecret( Channel channel, GenericResource resource ) +// { +// S3Secret secret = S3Secret.newBuilder().setSecretId( "-1" ).setAccessKey( "access" ).setSecretKey( "secKey" ) +// .setSessionToken( "token" ).build(); +// StorageSecret +// storageSecret = StorageSecret.newBuilder().setSecretId( "-1" ).setS3Secret( secret ) +// .setStorageType( org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3 ).build(); +// +// +// StorageSecret request = StorageSecret.newBuilder().setSecretId( "-1" ).setS3Secret( secret ). +// setStorageType( org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3 ).build(); +// StorageSecret response = getBlockingSecretStub( channel ).registerSecret( storageSecret ); +// return storageSecret; +// } +// +// +//} \ No newline at end of file diff --git a/replica-catalog-api/pom.xml b/replica-catalog-api/pom.xml new file mode 100644 index 0000000..32f2816 --- /dev/null +++ b/replica-catalog-api/pom.xml @@ -0,0 +1,20 @@ + + + 4.0.0 + + org.apache.airavata + airavata-data-catalog + 0.1-SNAPSHOT + + + replica-catalog-api + pom + + stubs + server + client + + + \ No newline at end of file diff --git a/replica-catalog-api/server/pom.xml b/replica-catalog-api/server/pom.xml new file mode 100644 index 0000000..7f7120a --- /dev/null +++ b/replica-catalog-api/server/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + + org.apache.airavata + replica-catalog-api + 0.1-SNAPSHOT + + + replica-catalog-api-server + + + + + org.springframework.boot + spring-boot-starter-parent + ${spring.boot.version} + pom + import + + + + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + io.github.lognet + grpc-spring-boot-starter + ${grpc.spring.boot} + + + org.springframework.boot + spring-boot-starter + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.postgresql + postgresql + + + org.apache.airavata + replica-catalog-api-stubs + 0.1-SNAPSHOT + + + io.hypersistence + hypersistence-utils-hibernate-60 + ${hypersistence.60.version} + + + com.fasterxml.jackson.module + jackson-module-jakarta-xmlbind-annotations + ${jackson.jakarta.version} + + + net.sf.dozer + dozer + ${dozer.version} + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + + + + \ No newline at end of file diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java new file mode 100644 index 0000000..f66e4c2 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java @@ -0,0 +1,13 @@ +package org.apache.airavata.replicacatalog; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ReplicaCatalogApiServiceApplication { + + public static void main(String[] args) { + SpringApplication.run(ReplicaCatalogApiServiceApplication.class, args); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java new file mode 100644 index 0000000..3a17cb5 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java @@ -0,0 +1,47 @@ +package org.apache.airavata.replicacatalog.catalogapi.mapper; + +import java.sql.Timestamp; +import java.util.UUID; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; +import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation} + */ +@Component +public class DataProductMapper { + + @Autowired + DataProductRepository dataProductRepository; + + public void mapModelToEntity(DataProduct dataProduct, DataProductEntity dataProductEntity) { + + dataProductEntity.setProductUri(dataProduct.getProductUri()); + dataProductEntity.setProductName(dataProduct.getProductName()); + dataProductEntity.setProductDescription(dataProduct.getProductDescription()); + dataProductEntity.setDataProductType(dataProduct.getDataProductType()); + dataProductEntity.setParentProductUri(dataProduct.getParentProductUri()); + dataProductEntity.setCreationTime(new Timestamp(System.currentTimeMillis())); + + } + + public void mapEntityToModel(DataProductEntity dataProductEntity, DataProduct.Builder dataProductBuilder) { + + dataProductBuilder + .setProductUri(dataProductEntity.getProductUri()) + .setProductName(dataProductEntity.getProductName()) + .setDataProductType(dataProductEntity.getDataProductType()) + .setParentProductUri(dataProductEntity.getParentProductUri()) + .setCreationTime(dataProductEntity.getCreationTime().getTime()).build(); + + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java new file mode 100644 index 0000000..5fa3961 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java @@ -0,0 +1,53 @@ +package org.apache.airavata.replicacatalog.catalogapi.mapper; + +import java.sql.Timestamp; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link DataReplicaLocation} + */ +@Component +public class DataReplicaMapper { + + @Autowired + DataReplicaLocationRepository replicaLocationRepository; + + @Autowired + DataProductRepository dataProductRepository; + + public void mapModelToEntity(DataReplicaLocation model, DataReplicaLocationEntity entity) { + + entity.setReplicaName(model.getReplicaName()); + entity.setReplicaId(model.getDataReplicaId()); + entity.setProductUri(model.getDataProductId()); + entity.setReplicaDescription(model.getReplicaDescription()); + entity.setCreationTime(new Timestamp(System.currentTimeMillis())); + // TODO: handle parent data product not found + DataProductEntity parentDataEntity = dataProductRepository.findByProductUri(model.getDataProductId()); + entity.setDataProduct(parentDataEntity); + + } + + public void mapEntityToModel(DataReplicaLocationEntity dataReplicaLocationEntity, DataReplicaLocation.Builder dataProductBuilder) { + + dataProductBuilder + .setDataReplicaId( dataReplicaLocationEntity.getReplicaId() ) + .setReplicaName( dataReplicaLocationEntity.getReplicaName() ) + .setDataProductId( dataReplicaLocationEntity.getProductUri() ) + .setCreationTime( dataReplicaLocationEntity.getCreationTime().getTime() ); + + if ( dataReplicaLocationEntity.getDataProduct() != null ) + { + dataProductBuilder.setDataProductId( dataReplicaLocationEntity.getDataProduct().getProductUri() ); + } + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java new file mode 100644 index 0000000..02e2e19 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java @@ -0,0 +1,163 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.replicacatalog.catalogapi.model; + + +import jakarta.persistence.*; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductType; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; + +/** + * The persistent class for the data_product database table. + */ +@Entity +@Table(name = "DATA_PRODUCT") +public class DataProductEntity implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "PRODUCT_URI") + private String productUri; + + @Column(name = "PRODUCT_NAME") + private String productName; + + @Column(name = "PRODUCT_DESCRIPTION") + private String productDescription; + + @Column(name = "OWNER_NAME") + private String ownerName; + + @Column(name = "PARENT_PRODUCT_URI") + private String parentProductUri; + + @Column(name = "PRODUCT_SIZE") + private int productSize; + + @Column(name = "CREATION_TIME") + private Timestamp creationTime; + + @Column(name = "LAST_MODIFIED_TIME") + private Timestamp lastModifiedTime; + + @Column(name = "PRODUCT_TYPE") + @Enumerated(EnumType.STRING) + private DataProductType dataProductType; + + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name="DATA_PRODUCT_METADATA", joinColumns = @JoinColumn(name="PRODUCT_URI")) + @MapKeyColumn(name = "METADATA_KEY") + @Column(name = "METADATA_VALUE") + private Map productMetadata; + + @OneToMany(targetEntity = DataReplicaLocationEntity.class, cascade = CascadeType.ALL, + mappedBy = "dataProduct", fetch = FetchType.EAGER) + private List replicaLocations; + + public String getProductUri() { + return productUri; + } + + public void setProductUri(String productUri) { + this.productUri = productUri; + } + + public String getProductName() { + return productName; + } + + public void setProductName(String productName) { + this.productName = productName; + } + + public String getProductDescription() { + return productDescription; + } + + public void setProductDescription(String productDescription) { + this.productDescription = productDescription; + } + + public String getOwnerName() { + return ownerName; + } + + public void setOwnerName(String ownerName) { + this.ownerName = ownerName; + } + + public String getParentProductUri() { + return parentProductUri; + } + + public void setParentProductUri(String parentProductUri) { + this.parentProductUri = parentProductUri; + } + + public int getProductSize() { + return productSize; + } + + public void setProductSize(int productSize) { + this.productSize = productSize; + } + + public Timestamp getCreationTime() { + return creationTime; + } + + public void setCreationTime(Timestamp creationTime) { + this.creationTime = creationTime; + } + + public Timestamp getLastModifiedTime() { + return lastModifiedTime; + } + + public void setLastModifiedTime(Timestamp lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + public DataProductType getDataProductType() { + return dataProductType; + } + + public void setDataProductType(DataProductType dataProductType) { + this.dataProductType = dataProductType; + } + + public Map getProductMetadata() { return productMetadata; } + + public void setProductMetadata(Map productMetadata) { this.productMetadata = productMetadata; } + + public List getReplicaLocations() { + return replicaLocations; + } + + public void setReplicaLocations(List replicaLocations) { + this.replicaLocations = replicaLocations; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java new file mode 100644 index 0000000..cb4da3d --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java @@ -0,0 +1,71 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import jakarta.persistence.*; + +import java.io.Serializable; + +/** + * The persistent class for the data_product_metadata database table. + */ +@Entity +@Table(name = "DATA_PRODUCT_METADATA") +@IdClass(DataProductMetadataPK.class) +public class DataProductMetadataEntity implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "PRODUCT_URI") + private String productUri; + + @Id + @Column(name = "METADATA_KEY") + private String metadataKey; + + @Column(name = "METADATA_VALUE") + private String metadataValue; + + public String getProductUri() { + return productUri; + } + + public void setProductUri(String productUri) { + this.productUri = productUri; + } + + public String getMetadataKey() { + return metadataKey; + } + + public void setMetadataKey(String metadataKey) { + this.metadataKey = metadataKey; + } + + public String getMetadataValue() { + return metadataValue; + } + + public void setMetadataValue(String metadataValue) { + this.metadataValue = metadataValue; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java new file mode 100644 index 0000000..07ac45b --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java @@ -0,0 +1,76 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import java.io.Serializable; + +/** + * The primary key class for the data_product_metadata database table. + */ +public class DataProductMetadataPK implements Serializable { + //default serial version id, required for serializable classes. + private static final long serialVersionUID = 1L; + + private String productUri; + private String metadataKey; + + public DataProductMetadataPK() { + } + + public String getProductUri() { + return productUri; + } + + public void setProductUri(String productUri) { + this.productUri = productUri; + } + + public String getMetadataKey() { + return metadataKey; + } + + public void setMetadataKey(String metadataKey) { + this.metadataKey = metadataKey; + } + + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof DataProductMetadataPK)) { + return false; + } + DataProductMetadataPK castOther = (DataProductMetadataPK) other; + return + this.productUri.equals(castOther.productUri) + && this.metadataKey.equals(castOther.metadataKey); + } + + public int hashCode() { + final int prime = 31; + int hash = 17; + hash = hash * prime + this.productUri.hashCode(); + hash = hash * prime + this.metadataKey.hashCode(); + + return hash; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java new file mode 100644 index 0000000..f68bf3f --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java @@ -0,0 +1,186 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import jakarta.persistence.*; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Map; + +/** + * The persistent class for the data_replica_location database table. + */ +@Entity +@Table(name = "DATA_REPLICA_LOCATION") +public class DataReplicaLocationEntity implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "REPLICA_ID") + private String replicaId; + + @Column(name = "PRODUCT_URI") + private String productUri; + + @Column(name = "REPLICA_NAME") + private String replicaName; + + @Column(name = "REPLICA_DESCRIPTION") + private String replicaDescription; + + @Column(name = "STORAGE_RESOURCE_ID") + private String storageResourceId; + + @Column(name = "FILE_PATH") + private String filePath; + + @Column(name = "CREATION_TIME") + private Timestamp creationTime; + + @Column(name = "LAST_MODIFIED_TIME") + private Timestamp lastModifiedTime; + + @Column(name = "VALID_UNTIL_TIME") + private Timestamp validUntilTime; + +// @Column(name = "REPLICA_LOCATION_CATEGORY") +// @Enumerated(EnumType.STRING) +// private ReplicaLocationCategory replicaLocationCategory; +// +// @Column(name = "REPLICA_PERSISTENT_TYPE") +// @Enumerated(EnumType.STRING) +// private ReplicaPersistentType replicaPersistentType; + + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "DATA_REPLICA_METADATA", joinColumns = @JoinColumn(name = "REPLICA_ID")) + @MapKeyColumn(name = "METADATA_KEY") + @Column(name = "METADATA_VALUE") + private Map replicaMetadata; + + @ManyToOne(targetEntity = DataProductEntity.class) + @JoinColumn(name = "PRODUCT_URI", nullable = false, updatable = false, insertable = false) + private DataProductEntity dataProduct; + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getProductUri() { + return productUri; + } + + public void setProductUri(String productUri) { + this.productUri = productUri; + } + + public String getReplicaName() { + return replicaName; + } + + public void setReplicaName(String replicaName) { + this.replicaName = replicaName; + } + + public String getReplicaDescription() { + return replicaDescription; + } + + public void setReplicaDescription(String replicaDescription) { + this.replicaDescription = replicaDescription; + } + + public String getStorageResourceId() { + return storageResourceId; + } + + public void setStorageResourceId(String storageResourceId) { + this.storageResourceId = storageResourceId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public Timestamp getCreationTime() { + return creationTime; + } + + public void setCreationTime(Timestamp creationTime) { + this.creationTime = creationTime; + } + + public Timestamp getLastModifiedTime() { + return lastModifiedTime; + } + + public void setLastModifiedTime(Timestamp lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + public Timestamp getValidUntilTime() { + return validUntilTime; + } + + public void setValidUntilTime(Timestamp validUntilTime) { + this.validUntilTime = validUntilTime; + } + +// public ReplicaLocationCategory getReplicaLocationCategory() { +// return replicaLocationCategory; +// } +// +// public void setReplicaLocationCategory(ReplicaLocationCategory replicaLocationCategory) { +// this.replicaLocationCategory = replicaLocationCategory; +// } +// +// public ReplicaPersistentType getReplicaPersistentType() { +// return replicaPersistentType; +// } +// +// public void setReplicaPersistentType(ReplicaPersistentType replicaPersistentType) { +// this.replicaPersistentType = replicaPersistentType; +// } + + public Map getReplicaMetadata() { + return replicaMetadata; + } + + public void setReplicaMetadata(Map replicaMetadata) { + this.replicaMetadata = replicaMetadata; + } + + public DataProductEntity getDataProduct() { + return dataProduct; + } + + public void setDataProduct(DataProductEntity dataProduct) { + this.dataProduct = dataProduct; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java new file mode 100644 index 0000000..cef6df4 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java @@ -0,0 +1,71 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import jakarta.persistence.*; + +import java.io.Serializable; + +/** + * The persistent class for the data_replica_metadata database table. + */ +@Entity +@Table(name = "DATA_REPLICA_METADATA") +@IdClass(DataReplicaMetadataPK.class) +public class DataReplicaMetadataEntity implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "REPLICA_ID") + private String replicaId; + + @Id + @Column(name = "METADATA_KEY") + private String metadataKey; + + @Column(name = "METADATA_VALUE") + private String metadataValue; + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getMetadataKey() { + return metadataKey; + } + + public void setMetadataKey(String metadataKey) { + this.metadataKey = metadataKey; + } + + public String getMetadataValue() { + return metadataValue; + } + + public void setMetadataValue(String metadataValue) { + this.metadataValue = metadataValue; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java new file mode 100644 index 0000000..0fa59c4 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java @@ -0,0 +1,74 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import java.io.Serializable; + +/** + * The primary key class for the data_replica_metadata database table. + */ +public class DataReplicaMetadataPK implements Serializable { + private static final long serialVersionUID = 1L; + + private String replicaId; + private String metadataKey; + + public DataReplicaMetadataPK() { + } + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getMetadataKey() { + return metadataKey; + } + + public void setMetadataKey(String metadataKey) { + this.metadataKey = metadataKey; + } + + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof DataReplicaMetadataPK)) { + return false; + } + DataReplicaMetadataPK castOther = (DataReplicaMetadataPK) other; + return + this.replicaId.equals(castOther.replicaId) + && this.metadataKey.equals(castOther.metadataKey); + } + + public int hashCode() { + final int prime = 31; + int hash = 17; + hash = hash * prime + this.replicaId.hashCode(); + hash = hash * prime + this.metadataKey.hashCode(); + + return hash; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java new file mode 100644 index 0000000..af99525 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java @@ -0,0 +1,15 @@ +package org.apache.airavata.replicacatalog.catalogapi.repository; + +import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.transaction.annotation.Transactional; + +@Transactional(readOnly = true) +public interface DataProductRepository extends JpaRepository { + + DataProductEntity findByProductUri(String productUri); + + @Transactional + void deleteByProductUri(String productUri); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java new file mode 100644 index 0000000..9fa5cb7 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java @@ -0,0 +1,36 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.repository; + +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.transaction.annotation.Transactional; + +@Transactional(readOnly = true) +public interface DataReplicaLocationRepository extends JpaRepository { + + DataReplicaLocationEntity findByReplicaId(String replicaId); + + @Transactional + void deleteByReplicaId(String replicaId); + + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java new file mode 100644 index 0000000..d2fa9ed --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java @@ -0,0 +1,28 @@ +package org.apache.airavata.replicacatalog.catalogapi.service; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; + +import java.util.List; + +public interface IReplicaCatalogService { + + DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica); + + DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica); + + DataReplicaLocation getDataReplica(String replicaId); + + void deleteDataReplica(String replicaId); + + + DataProduct createDataProduct(DataProduct dataProduct); + + DataProduct updateDataProduct(DataProduct dataProduct); + + DataProduct getDataProduct(String productUri); + + void deleteDataProduct(String productUri); + + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java new file mode 100644 index 0000000..789939e --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java @@ -0,0 +1,116 @@ +package org.apache.airavata.replicacatalog.catalogapi.service; + +import io.grpc.stub.StreamObserver; + +import org.apache.airavata.replicacatalog.catalog.service.ReplicaCatalogAPIServiceGrpc; +import org.apache.airavata.replicacatalog.catalog.stubs.*; +import org.apache.airavata.replicacatalog.catalogapi.mapper.DataProductMapper; +import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * API level services + */ +@GRpcService +public class ReplicaCatalogAPIService extends ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceImplBase { + private static final Logger logger = LoggerFactory.getLogger(ReplicaCatalogAPIService.class); + + + @Autowired + DataProductMapper dataProductMapper = new DataProductMapper(); + + @Autowired + DataReplicaMapper replicaMapper = new DataReplicaMapper(); + + @Autowired + IReplicaCatalogService dataCatalogService; + + + @Override + public void registerDataProduct(DataProductCreateRequest request, + StreamObserver responseObserver) { + + DataProduct result = dataCatalogService.createDataProduct(request.getDataProduct()); + DataProductCreateResponse.Builder responseBuilder = DataProductCreateResponse.newBuilder(); + responseBuilder.setDataProduct(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + + } + + @Override + public void updateDataProduct(DataProductUpdateRequest request, + StreamObserver responseObserver) { + super.updateDataProduct(request, responseObserver); + } + + @Override + public void getDataProduct(DataProductGetRequest request, StreamObserver responseObserver) { + super.getDataProduct(request, responseObserver); + } + + @Override + public void removeDataProduct(DataProductDeleteRequest request, + StreamObserver responseObserver) { + super.removeDataProduct(request, responseObserver); + } + + @Override + public void registerReplicaLocation(DataReplicaCreateRequest request, + StreamObserver responseObserver) { + + logger.info("Creating Replica for a data product {}", request.getDataReplica()); + if (request.getDataReplica() == null) { + logger.debug("No Data Replica Location"); + } + DataReplicaLocation result = dataCatalogService.createDataReplica(request.getDataReplica()); + + DataReplicaCreateResponse.Builder responseBuilder = DataReplicaCreateResponse.newBuilder(); + responseBuilder.setDataReplica(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void updateReplicaLocation(DataReplicaUpdateRequest request, + StreamObserver responseObserver) { + + DataReplicaLocation result = dataCatalogService.updateDataReplica(request.getDataReplica()); + + DataReplicaUpdateResponse.Builder responseBuilder = DataReplicaUpdateResponse.newBuilder(); + responseBuilder.setDataReplica(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getReplicaLocation(DataReplicaGetRequest request, + StreamObserver responseObserver) { + super.getReplicaLocation(request, responseObserver); + } + + @Override + public void removeReplicaLocation(DataReplicaDeleteRequest request, + StreamObserver responseObserver) { + super.removeReplicaLocation(request, responseObserver); + } + + @Override + public void getAllReplicaLocation(AllDataReplicaGetRequest request, + StreamObserver responseObserver) { + super.getAllReplicaLocation(request, responseObserver); + } + + @Override + public void removeAllReplicaLocation(AllDataReplicaDeleteRequest request, + StreamObserver responseObserver) { + super.removeAllReplicaLocation(request, responseObserver); + } + + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java new file mode 100644 index 0000000..12585b7 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java @@ -0,0 +1,121 @@ +package org.apache.airavata.replicacatalog.catalogapi.service.impl; + +import jakarta.transaction.Transactional; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalogapi.mapper.DataProductMapper; +import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; +import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; +import org.apache.airavata.replicacatalog.catalogapi.service.IReplicaCatalogService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.UUID; + +@Service +@Transactional +public class ReplicaCatalogServiceImp implements IReplicaCatalogService { + private final static Logger logger = LoggerFactory.getLogger(ReplicaCatalogServiceImp.class); + + @Autowired + DataProductRepository dataProductRepository; + + @Autowired + DataReplicaLocationRepository dataReplicaLocationRepository; + + @Autowired + DataProductMapper dataProductMapper = new DataProductMapper(); + + @Autowired + DataReplicaMapper replicaMapper = new DataReplicaMapper(); + + + @Override + public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation) { + + DataReplicaLocationEntity dataReplicaLocationEntity = new DataReplicaLocationEntity(); + + if (replicaLocation != null && replicaLocation.getDataReplicaId() != null) { + dataReplicaLocationEntity.setReplicaId(replicaLocation.getDataReplicaId()); + } + replicaMapper.mapModelToEntity(replicaLocation, dataReplicaLocationEntity); + if (dataReplicaLocationEntity.getReplicaId() == null || dataReplicaLocationEntity.getReplicaId().isEmpty()) { + dataReplicaLocationEntity.setReplicaId(UUID.randomUUID().toString()); + } + DataReplicaLocationEntity savedDataReplicaLocationEntity = dataReplicaLocationRepository.save(dataReplicaLocationEntity); + return toDataReplicaLocation(savedDataReplicaLocationEntity); + + } + + @Override + public DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplicaLocation) { + + DataReplicaLocationEntity dataReplicaLocationEntity = dataReplicaLocationRepository.findByReplicaId(dataReplicaLocation.getDataReplicaId()); + if (dataReplicaLocationEntity == null) { + logger.debug("Data Replica Location not exists"); + } + replicaMapper.mapModelToEntity(dataReplicaLocation, dataReplicaLocationEntity); + DataReplicaLocationEntity savedDataReplicaLocationEntity = dataReplicaLocationRepository.save(dataReplicaLocationEntity); + return toDataReplicaLocation(savedDataReplicaLocationEntity); + } + + @Override + public DataReplicaLocation getDataReplica(String replicaId) { + return null; + } + + @Override + public void deleteDataReplica(String replicaId) { + + } + + @Override + public DataProduct createDataProduct(DataProduct dataProduct) { + DataProductEntity dataProductEntity = new DataProductEntity(); + + if (dataProduct != null && dataProduct.getProductUri() != null) { + dataProductEntity.setProductUri(dataProduct.getProductUri()); + } + dataProductMapper.mapModelToEntity(dataProduct, dataProductEntity); + + if (dataProductEntity.getProductUri() == null) { + dataProductEntity.setProductUri(UUID.randomUUID().toString()); + } + DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); + return toDataProduct(savedDataProductEntity); + } + + @Override + public DataProduct updateDataProduct(DataProduct dataProduct) { + return null; + } + + @Override + public DataProduct getDataProduct(String productUri) { + return null; + } + + @Override + public void deleteDataProduct(String productUri) { + + } + + + private DataReplicaLocation toDataReplicaLocation(DataReplicaLocationEntity savedDataLocationEntity) { + DataReplicaLocation.Builder builder = DataReplicaLocation.newBuilder(); + replicaMapper.mapEntityToModel(savedDataLocationEntity, builder); + return builder.build(); + } + + private DataProduct toDataProduct(DataProductEntity savedDataProductEntity) { + DataProduct.Builder builder = DataProduct.newBuilder(); + dataProductMapper.mapEntityToModel(savedDataProductEntity, builder); + return builder.build(); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java new file mode 100644 index 0000000..6832b90 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java @@ -0,0 +1,54 @@ +package org.apache.airavata.replicacatalog.resource.mapper; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link DataReplicaLocation} + */ +@Component +public class ResourceStorageMapper { + + @Autowired + GenericResourceRepository genericResourceRepository; + + public void mapGenericStorageModelToEntity( GenericResource storage, GenericResourceEntity resourceEntity) { + + resourceEntity.setStorageId(storage.getResourceId()); + resourceEntity.setStorageType( GenericResourceEntity.StorageType.S3); + resourceEntity.setResourcePath( storage.getFile().getResourcePath() ); + resourceEntity.setResourceType( GenericResourceEntity.ResourceType.FILE ); + // TODO + } + + public void mapGenericStorageEntityToModel(GenericResourceEntity resourceEntity, StorageWrapper wrapper, GenericResource.Builder builder) { + + builder.setResourceId( resourceEntity.getResourceId() ); + builder.setStorage(wrapper); + } + + public void mapStorageSecretModelToEntity(SecretForStorage storage, StorageSecretEntity resourceEntity) { + resourceEntity.setStorageId(storage.getStorageId()); + resourceEntity.setSecretId(storage.getSecretId()); + resourceEntity.setType( storage.getStorageType().name()); + // TODO + } + + public void mapStorageSecretEntityToModel(StorageSecretEntity resourceEntity, SecretForStorage.Builder dataProductBuilder) { + + dataProductBuilder.setStorageType( StorageType.valueOf( resourceEntity.getType() ) ).setSecretId( resourceEntity.getSecretId() ) + .setStorageId( resourceEntity.getStorageId() ); + + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java new file mode 100644 index 0000000..82b811d --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import org.hibernate.annotations.GenericGenerator; + + + +@Entity +public class GenericResourceEntity { + + public enum ResourceType { + FILE, DIRECTORY; + } + + public enum StorageType { + S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT, ODATA; + } + + @Id + @Column(name = "RESOURCE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String resourceId; + + @Column(name = "RESOURCE_PATH") + private String resourcePath; + + @Column(name = "RESOURCE_TYPE") + private ResourceType resourceType; + + @Column(name = "STORAGE_ID") + private String storageId; + + @Column(name = "STORAGE_TYPE") + private StorageType storageType; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getResourcePath() { + return resourcePath; + } + + public void setResourcePath(String resourcePath) { + this.resourcePath = resourcePath; + } + + public ResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(ResourceType resourceType) { + this.resourceType = resourceType; + } + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java new file mode 100644 index 0000000..12c48bd --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; + + +@Entity +public class ResolveStorageEntity { + + public enum StorageType { + S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT, ODATA; + } + + @Id + @Column(name = "STORAGE_ID") + private String storageId; + + @Column(name = "STORAGE_TYPE") + private StorageType storageType; + + @Column(name = "STORAGE_NAME") + private String storageName; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + public String getStorageName() { + return storageName; + } + + public void setStorageName(String storageName) { + this.storageName = storageName; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java new file mode 100644 index 0000000..56e3c5f --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +public class S3StorageEntity { + + @Id + @Column(name = "S3_STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "BUCKET_NAME") + private String bucketName; + + @Column(name = "REGION") + private String region; + + @Column(name = "ENDPOINT") + private String endpoint; + + @Column(name = "USE_TLS") + private boolean useTLS; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public boolean isUseTLS() { + return useTLS; + } + + public void setUseTLS(boolean useTLS) { + this.useTLS = useTLS; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java new file mode 100644 index 0000000..820a2c1 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import org.hibernate.annotations.GenericGenerator; + +@Entity +public class StorageSecretEntity { + @Id + @Column(name = "RESOURCE_SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String id; + + @Column(name = "STORAGE_ID") + private String storageId; + + @Column(name = "SECRET_ID") + private String secretId; + + @Column(name = "STORAGE_TYPE") + private String type; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java new file mode 100644 index 0000000..8c65512 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface GenericResourceRepository extends CrudRepository { + Optional findByResourceId(String resourceId); + List findByStorageId(String storageId); + void deleteByStorageIdAndStorageType(String storageId, GenericResourceEntity.StorageType storageType); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java new file mode 100644 index 0000000..c04e019 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.ResolveStorageEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface ResolveStorageRepository extends CrudRepository { + Optional getByStorageId(String storageID); + List getByStorageName(String storageName); + List getByStorageType(ResolveStorageEntity.StorageType storageType); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java new file mode 100644 index 0000000..4590b0b --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.S3StorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface S3StorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java new file mode 100644 index 0000000..f87e761 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface StorageSecretRepository extends CrudRepository { + + Optional findByStorageId(String storageId); + void deleteByStorageId(String resourceId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java new file mode 100644 index 0000000..5215113 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.service; + + + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.resource.stubs.common.*; +import org.apache.airavata.replicacatalog.resource.stubs.s3.*; + +import java.util.Optional; + +public interface IResourceService { + + public void init(); + public void destroy(); + public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception; + + public SecretForStorage registerSecretForStorage(SecretForStorage request) throws Exception; + + public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) throws Exception; + + public StorageListResponse searchStorages(StorageSearchRequest request) throws Exception; + + public StorageListResponse listStorage(StorageListRequest request) throws Exception; + + public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception; + + +// public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception; +// public Optional getS3Storage(S3StorageGetRequest request) throws Exception; +// public S3Storage createS3Storage(S3StorageCreateRequest request) throws Exception; +// public boolean updateS3Storage(S3StorageUpdateRequest request) throws Exception; +// public boolean deleteS3Storage(S3StorageDeleteRequest request) throws Exception; +// + + StorageTypeResolveResponse resolveStorageType(StorageTypeResolveRequest request) throws Exception; +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java new file mode 100644 index 0000000..a11e08f --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java @@ -0,0 +1,56 @@ +package org.apache.airavata.replicacatalog.resource.service; + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateResponse; +import org.apache.airavata.replicacatalog.catalogapi.service.IReplicaCatalogService; +import org.apache.airavata.replicacatalog.resource.mapper.ResourceStorageMapper; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.repository.StorageSecretRepository; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.UUID; + +@GRpcService +public class ResourceAPIService extends StorageCommonServiceGrpc.StorageCommonServiceImplBase { + private static final Logger logger = LoggerFactory.getLogger(ResourceAPIService.class); + + + @Autowired + IResourceService resourceService; + + @Override + public void createGenericResource(GenericResourceCreateRequest request, StreamObserver responseObserver) { + logger.info("Creating Storage {}", request.getStorageId()); + GenericResource resource = null; + try { + resource = resourceService.createGenericResource(request); + } catch (Exception e) { + logger.error("Error {0}", e); + } + + responseObserver.onNext(resource); + responseObserver.onCompleted(); + } + + @Override + public void registerSecretForStorage(SecretForStorage request, StreamObserver responseObserver) { + SecretForStorage secretForStorage = null; + try { + secretForStorage = resourceService.registerSecretForStorage(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(secretForStorage); + responseObserver.onCompleted(); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java new file mode 100644 index 0000000..00cb977 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.service; + + +import org.apache.airavata.replicacatalog.resource.mapper.ResourceStorageMapper; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.model.ResolveStorageEntity; +import org.apache.airavata.replicacatalog.resource.model.S3StorageEntity; +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.repository.ResolveStorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.S3StorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.StorageSecretRepository; + +import org.apache.airavata.replicacatalog.resource.stubs.common.*; +import org.apache.airavata.replicacatalog.resource.stubs.common.Error; +import org.apache.airavata.replicacatalog.resource.stubs.s3.*; +import org.dozer.DozerBeanMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Component("SQLResourceBackend") +public class SQLIResourceService implements IResourceService { + + private static final Logger logger = LoggerFactory.getLogger(SQLIResourceService.class); + private DozerBeanMapper mapper = new DozerBeanMapper(); + @Autowired + private GenericResourceRepository resourceRepository; + + @Autowired + private S3StorageRepository s3StorageRepository; + + @Autowired + private StorageSecretRepository resourceSecretRepository; + + @Autowired + private ResolveStorageRepository resolveStorageRepository; + + @Autowired + ResourceStorageMapper resourceStorageMapper = new ResourceStorageMapper(); + + @Override + public void init() { + logger.info("Initializing database resource backend"); + } + + @Override + public void destroy() { + logger.info("Destroying database resource backend"); + } + + @Override + public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception { + GenericResourceEntity resourceEntity = new GenericResourceEntity(); + resourceEntity.setResourceId(UUID.randomUUID().toString()); + resourceStorageMapper.mapGenericStorageModelToEntity(request.getResource(), resourceEntity); + GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); + S3Storage storage = null; + if (request.getStorageType() == StorageType.S3) { + storage = createS3Storage(request.getResource().getStorage().getS3Storage()); + } + StorageWrapper wrapper = StorageWrapper.newBuilder().setS3Storage(storage).build(); + + GenericResource.Builder responseBuilder = GenericResource.newBuilder(); + resourceStorageMapper.mapGenericStorageEntityToModel(savedDataProductEntity, wrapper, responseBuilder); + return responseBuilder.build(); + } + + + @Override + public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception { + Optional resourceSecEtyOp = resourceSecretRepository.findByStorageId(request.getStorageId()); + SecretForStorage.Builder resultBuilder = SecretForStorage.newBuilder(); + if (resourceSecEtyOp.isPresent()) { + StorageSecretEntity storageSecretEntity = resourceSecEtyOp.get(); + resultBuilder.setSecretId(storageSecretEntity.getSecretId()); + resultBuilder.setStorageId(storageSecretEntity.getStorageId()); + } else { + resultBuilder.setError(Error.NOT_FOUND); + } + return resultBuilder.build(); + } + + @Override + public SecretForStorage registerSecretForStorage(SecretForStorage request) throws Exception { + + StorageSecretEntity resourceEntity = new StorageSecretEntity(); + resourceStorageMapper.mapStorageSecretModelToEntity(request, resourceEntity); + StorageSecretEntity savedDataProductEntity = resourceSecretRepository.save(resourceEntity); + SecretForStorage.Builder responseBuilder = SecretForStorage.newBuilder(); + resourceStorageMapper.mapStorageSecretEntityToModel(savedDataProductEntity, responseBuilder); + + return request; + } + + @Override + public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) throws Exception { + resourceSecretRepository.deleteByStorageId(request.getStorageId()); + return true; + } + + @Override + public StorageListResponse searchStorages(StorageSearchRequest request) throws Exception { + StorageListResponse.Builder resp = StorageListResponse.newBuilder(); + switch (request.getSearchQueryCase()) { + case STORAGE_ID: + Optional storageOp = resolveStorageRepository.getByStorageId(request.getStorageId()); + if (storageOp.isPresent()) { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(storageOp.get().getStorageId()); + entry.setStorageName(storageOp.get().getStorageName()); + entry.setStorageType(StorageType.valueOf(storageOp.get().getStorageType().name())); + resp.addStorageList(entry); + } + break; + case STORAGE_NAME: + List storages = resolveStorageRepository.getByStorageName(request.getStorageName()); + storages.forEach(st -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(st.getStorageId()); + entry.setStorageName(st.getStorageName()); + entry.setStorageType(StorageType.valueOf(st.getStorageType().name())); + resp.addStorageList(entry); + }); + break; + case STORAGE_TYPE: + storages = resolveStorageRepository.getByStorageType(ResolveStorageEntity.StorageType.valueOf(request.getStorageType().name())); + storages.forEach(st -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(st.getStorageId()); + entry.setStorageName(st.getStorageName()); + entry.setStorageType(StorageType.valueOf(st.getStorageType().name())); + resp.addStorageList(entry); + }); + break; + } + return resp.build(); + } + + @Override + public StorageListResponse listStorage(StorageListRequest request) throws Exception { + Iterable all = resolveStorageRepository.findAll(); + StorageListResponse.Builder builder = StorageListResponse.newBuilder(); + all.forEach(r -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(r.getStorageId()); + entry.setStorageType(StorageType.valueOf(r.getStorageType().name())); + entry.setStorageName(r.getStorageName()); + builder.addStorageList(entry); + }); + return builder.build(); + } + + + public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception { + S3StorageListResponse.Builder respBuilder = S3StorageListResponse.newBuilder(); + List all = s3StorageRepository.findAll(PageRequest.of(request.getOffset(), request.getLimit())); + all.forEach(ety -> respBuilder.addStorages(mapper.map(ety, S3Storage.newBuilder().getClass()))); + return respBuilder.build(); + } + + + public Optional getS3Storage(S3StorageGetRequest request) throws Exception { + Optional entity = s3StorageRepository.findById(request.getStorageId()); + return entity.map(e -> mapper.map(e, S3Storage.newBuilder().getClass()).build()); + } + + + public S3Storage createS3Storage(S3Storage request) throws Exception { + S3StorageEntity savedEntity = s3StorageRepository.save(mapper.map(request, S3StorageEntity.class)); + + ResolveStorageEntity storageTypeEty = new ResolveStorageEntity(); + storageTypeEty.setStorageId(savedEntity.getStorageId()); + storageTypeEty.setStorageType(ResolveStorageEntity.StorageType.S3); + storageTypeEty.setStorageName(savedEntity.getName()); + resolveStorageRepository.save(storageTypeEty); + + return mapper.map(savedEntity, S3Storage.newBuilder().getClass()).build(); + } + + + public boolean updateS3Storage(S3StorageUpdateRequest request) throws Exception { + s3StorageRepository.save(mapper.map(request, S3StorageEntity.class)); + return true; + } + + + public boolean deleteS3Storage(S3StorageDeleteRequest request) throws Exception { + s3StorageRepository.deleteById(request.getStorageId()); + resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.S3); + return true; + } + + + @Override + public StorageTypeResolveResponse resolveStorageType(StorageTypeResolveRequest request) throws Exception { + Optional resolveStorageOp = resolveStorageRepository.getByStorageId(request.getStorageId()); + StorageTypeResolveResponse.Builder responseBuilder = StorageTypeResolveResponse.newBuilder(); + + if (resolveStorageOp.isPresent()) { + ResolveStorageEntity resolveStorageEntity = resolveStorageOp.get(); + responseBuilder.setStorageId(resolveStorageEntity.getStorageId()); + responseBuilder.setStorageType(StorageType.valueOf(resolveStorageEntity.getStorageType().name())); + responseBuilder.setStorageName(resolveStorageEntity.getStorageName()); + } else { + responseBuilder.setError(Error.NOT_FOUND); + } + return responseBuilder.build(); + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java new file mode 100644 index 0000000..893b8a4 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java @@ -0,0 +1,33 @@ +package org.apache.airavata.replicacatalog.sceret.mapper; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link DataReplicaLocation} + */ +@Component +public class ResourceSecretMapper +{ + + @Autowired + GenericResourceRepository genericResourceRepository; + + public void mapModelToEntity(SecretForStorage storage, GenericResourceEntity resourceEntity) { + + resourceEntity.setResourceId(storage.getStorageId()); + // TODO + } + + public void mapEntityToModel(GenericResourceEntity resourceEntity, SecretForStorage.Builder dataProductBuilder) { + + //TODO + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java new file mode 100644 index 0000000..a17cf29 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import org.hibernate.annotations.GenericGenerator; + + + +@Entity +public class S3SecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "ACCESS_KEY") + private String accessKey; + + @Column(name = "SECRET_KEY") + private String secretKey; + + @Column(name = "SESSION_TOKEN", length = 512) + private String sessionToken; + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public String getSessionToken() { + return sessionToken; + } + + public void setSessionToken(String sessionToken) { + this.sessionToken = sessionToken; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java new file mode 100644 index 0000000..8f60741 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.repository; + +import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface S3SecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java new file mode 100644 index 0000000..5c5be78 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.service; + +import org.apache.airavata.replicacatalog.secret.stubs.common.*; + +import java.util.Optional; + +public interface ISecretService { + + public void init(); + public void destroy(); + + public StorageSecret getSecretForStorage(SecretGetRequest request) throws Exception; + public StorageSecret registerSecretForStorage(StorageSecret request) throws Exception; + public boolean deleteSecretForStorage(SecretDeleteRequest request) throws Exception; + public SecretListResponse searchStorages(SecretSearchRequest request) throws Exception; + public SecretListResponse listStorage(SecretListRequest request) throws Exception; + +// public Optional getS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretGetRequest request) throws Exception; +// public org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret createS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretCreateRequest request) throws Exception; +// public boolean updateS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretUpdateRequest request) throws Exception; +// public boolean deleteS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretDeleteRequest request) throws Exception; +// + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java new file mode 100644 index 0000000..5c4c188 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.service; + +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; +import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; +import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; +import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; +import org.apache.airavata.replicacatalog.secret.stubs.common.*; +import org.apache.airavata.replicacatalog.secret.stubs.s3.*; +import org.dozer.DozerBeanMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +@Component("SQLSecretBackend") +public class SQLISecretService implements ISecretService { + + private static final Logger logger = LoggerFactory.getLogger(SQLISecretService.class); + + + @Autowired + ResourceSecretMapper secretMapper = new ResourceSecretMapper(); + + @Autowired + private S3SecretRepository s3SecretRepository; + + private DozerBeanMapper mapper = new DozerBeanMapper(); + + @Override + public void init() { + logger.info("Initializing database secret backend"); + } + + @Override + public void destroy() { + logger.info("Destroying database secret backend"); + } + + @Override + public StorageSecret getSecretForStorage(SecretGetRequest request) throws Exception { + return null; + } + + @Override + public StorageSecret registerSecretForStorage(StorageSecret request) throws Exception { + StorageSecret.Builder storageSecret = StorageSecret.newBuilder(); + if (request.getStorageType().name().equals(StorageType.S3.name())) { + return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret(createS3Secret(request.getSecret().getS3Secret()))).build(); + } + return null; + } + + @Override + public boolean deleteSecretForStorage(SecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public SecretListResponse searchStorages(SecretSearchRequest request) throws Exception { + return null; + } + + @Override + public SecretListResponse listStorage(SecretListRequest request) throws Exception { + return null; + } + + + public Optional getS3Secret(S3SecretGetRequest request) throws Exception { + Optional secretEty = s3SecretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(s3SecretEntity -> mapper.map(s3SecretEntity, S3Secret.newBuilder().getClass()).build()); + } + + public S3Secret createS3Secret(S3Secret request) throws Exception { + S3SecretEntity savedEntity = s3SecretRepository.save(mapper.map(request, S3SecretEntity.class)); + return mapper.map(savedEntity, S3Secret.newBuilder().getClass()).build(); + } + + public boolean updateS3Secret(S3SecretUpdateRequest request) throws Exception { + s3SecretRepository.save(mapper.map(request, S3SecretEntity.class)); + return true; + } + + public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { + s3SecretRepository.deleteById(request.getSecretId()); + return true; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java new file mode 100644 index 0000000..c806988 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java @@ -0,0 +1,41 @@ +package org.apache.airavata.replicacatalog.sceret.service; + +import java.util.UUID; + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; +import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; +import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +@GRpcService +public class StorageSecretAPIService extends SecretCommonServiceGrpc.SecretCommonServiceImplBase { + + + private static final Logger logger = LoggerFactory.getLogger(StorageSecretAPIService.class); + + + @Autowired + ISecretService secretService; + + @Autowired + ResourceSecretMapper secretMapper = new ResourceSecretMapper(); + + @Override + public void registerSecret(StorageSecret request, StreamObserver responseObserver) { + StorageSecret secretResult = null; + try { + secretResult = secretService.registerSecretForStorage(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + + responseObserver.onNext(secretResult); + responseObserver.onCompleted(); + } +} diff --git a/replica-catalog-api/server/src/main/resources/application.properties b/replica-catalog-api/server/src/main/resources/application.properties new file mode 100644 index 0000000..3948d56 --- /dev/null +++ b/replica-catalog-api/server/src/main/resources/application.properties @@ -0,0 +1,5 @@ +spring.datasource.url=jdbc:postgresql://localhost:5432/replica_catalog +spring.datasource.username=postgres +spring.datasource.password=example +spring.jpa.hibernate.ddl-auto=create-drop +spring.jpa.show-sql=true \ No newline at end of file diff --git a/replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java b/replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java new file mode 100644 index 0000000..e77dd6e --- /dev/null +++ b/replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java @@ -0,0 +1,37 @@ +package org.apache.airavata.replicacatalog.catalogapi.service; + +import org.junit.jupiter.api.Test; + +class ReplicaCatalogAPIServiceTest { + + @Test + void registerReplicaLocation() { + + ReplicaCatalogAPIService apiService = new ReplicaCatalogAPIService(); +// apiService.registerReplicaLocation(); + } + + @Test + void updateReplicaLocation() { + } + + @Test + void getReplicaLocation() { + } + + @Test + void removeReplicaLocation() { + } + + @Test + void getAllReplicaLocation() { + } + + @Test + void removeAllReplicaLocation() { + } + + @Test + void testRegisterReplicaLocation() { + } +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/pom.xml b/replica-catalog-api/stubs/pom.xml new file mode 100644 index 0000000..895127c --- /dev/null +++ b/replica-catalog-api/stubs/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + org.apache.airavata + replica-catalog-api + 0.1-SNAPSHOT + + + replica-catalog-api-stubs + + + io.grpc + grpc-netty-shaded + ${grpc.version} + runtime + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + org.apache.tomcat + annotations-api + ${tomcat.annotations-api.version} + provided + + + + + + kr.motd.maven + os-maven-plugin + ${os.maven.plugin} + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf.maven.plugin} + + com.google.protobuf:protoc:${protobuf.protoc.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + + + + + compile + compile-custom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto new file mode 100644 index 0000000..9909885 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.apache.airavata.replicacatalog.catalog.stubs"; + +message UserInfo { + /* + * This is an external identifier for the user, which identifies the user in + * whatever user management scheme data catalog is integrated with. + */ + string user_id = 1; + /* + * This is an (optional) external identifier for the tenant, which + * identifies the tenant in whatever tenant management scheme data catalog + * is integrated with. + */ + string tenant_id = 2; +} + +message GroupInfo { + /* + * This is an external identifier for the group, which identifies the group in + * whatever group management scheme data catalog is integrated with. + */ + string group_id = 1; + /* + * This is an (optional) external identifier for the tenant, which + * identifies the tenant in whatever tenant management scheme data catalog + * is integrated with. + */ + string tenant_id = 2; +} + +enum Permission { + OWNER = 0; + READ = 1; + READ_METADATA = 2; + WRITE = 3; + WRITE_METADATA = 4; + MANAGE_SHARING = 5; +} + +enum StorageType { + LOCAL = 0; + S3 = 1; + GCS = 2; + AZURE = 3; + +} + +enum DataProductType { + FILE = 0; + COLLECTION = 1; +} + +// Entity which used to model a replica entry +message DataReplicaLocation { + string data_replica_id = 1; // Unique replica id + string data_product_id = 2; + string storage_resource_id = 3; // Replica storage id (TO link the storage entry) + string replica_name = 4; // Replica user friendly name + string replica_description = 5; // Replica description + int64 creation_time = 6; // Replica creation date and time + int64 last_modified_time = 7; // Last modified time of replica + int64 valid_until_time = 8; // Validity time of the replica + StorageType storage_type = 9; // Storage type of the replica + map metadata = 10; // Replica's metadata information +} + +message DataReplicaCreateRequest { + DataReplicaLocation data_replica = 1; +} +message DataReplicaCreateResponse{ + DataReplicaLocation data_replica = 1; +} + +message DataReplicaGroupEntryCreateRequest { + ReplicaGroupEntry data_replica_group = 1; +} +message DataReplicaGroupEntryCreateResponse{ + ReplicaGroupEntry data_replica_group = 1; +} + + +message DataReplicaUpdateRequest { + DataReplicaLocation data_replica = 1; +} +message DataReplicaUpdateResponse{ + DataReplicaLocation data_replica = 1; +} +message DataReplicaGetRequest { + string data_replica_id = 1; +} +message DataReplicaGetResponse { + DataReplicaLocation data_replica = 1; +} +message DataReplicaDeleteRequest { + string data_replica_id = 1; +} +message DataReplicaDeleteResponse { +} + +message AllDataReplicaGetRequest { + string data_product_id = 1; + int32 page_number = 2; + int32 page_size = 3; +} + +message AllDataReplicaGetResponse { + string data_product_id = 1; + repeated ReplicaGroupEntry replica_list = 2; +} + +message ReplicaGroupEntry { + string replica_group_id = 1; + string data_replica_id = 2; + repeated ReplicaGroupEntry directories = 3; + repeated DataReplicaLocation files = 4; +} + +message ReplicaListEntry { + string data_replica_id = 1; + string replica_name = 2; + StorageType storage_type = 3; +} + +message AllDataReplicaDeleteRequest { + string data_product_id = 1; +} + +message AllDataReplicaDeleteResponse{ + int32 replica_location_count = 1; +} + + +// Entity which used to model a DataProduct entry +message DataProduct{ + string product_uri = 1; // Unique id + string parent_product_uri = 2; // + string product_name = 3; // Product user friendly name + string product_description = 4; // Product description + int64 creation_time = 5; // Product creation date and time + int64 last_modified_time = 6; // Last modified time of replica + int64 valid_until_time = 7; // Validity time of the replica + DataProductType data_product_type = 8; // Product type of the DataProduct + map metadata = 9; // Product metadata information + repeated DataReplicaLocation replica_locations = 10; //Link to replicas + +} + +message DataProductCreateRequest { + DataProduct data_product = 1; +} +message DataProductCreateResponse{ + DataProduct data_product = 1; +} + +message DataProductUpdateRequest { + DataProduct data_product = 1; +} +message DataProductUpdateResponse{ + DataProduct data_product = 1; +} + +message DataProductGetRequest { + string data_product_uri = 1; +} +message DataProductGetResponse { + DataProduct data_product = 1; +} + +message DataProductDeleteRequest { + string data_product_uri = 1; +} +message DataProductDeleteResponse { +} + + diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto new file mode 100644 index 0000000..d607fd4 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +syntax = "proto3"; + +option java_package = "org.apache.airavata.replicacatalog.catalog.service"; + +import "catalogapi/ReplicaCatalogAPI.proto"; + +service ReplicaCatalogAPIService { + + /* + * Register a new replica to the Replica Catalog + */ + rpc registerReplicaLocation(DataReplicaCreateRequest) returns (DataReplicaCreateResponse){} + + /* + * Update existing replica details in the ReplicaCatalog + */ + rpc updateReplicaLocation(DataReplicaUpdateRequest) returns (DataReplicaUpdateResponse){} + + /* + * Retrieving existing replica details in the ReplicaCatalog + */ + rpc getReplicaLocation(DataReplicaGetRequest) returns (DataReplicaGetResponse){} + + /* + * Removing a replica in the ReplicaCatalog + */ + rpc removeReplicaLocation(DataReplicaDeleteRequest) returns (DataReplicaDeleteResponse){} + + /* + * Retrieving all the existing replica details for a selected product data in the ReplicaCatalog + */ + rpc getAllReplicaLocation(AllDataReplicaGetRequest) returns (AllDataReplicaGetResponse){} + + /* + * Remove all the existing replicas for a selected product data in the ReplicaCatalog + */ + rpc removeAllReplicaLocation(AllDataReplicaDeleteRequest) returns (AllDataReplicaDeleteResponse){} + + + /* + * Register a new data product to the Replica Catalog + */ + rpc registerDataProduct(DataProductCreateRequest) returns (DataProductCreateResponse){} + + /* + * Update existing Data Product details in the ReplicaCatalog + */ + rpc updateDataProduct(DataProductUpdateRequest) returns (DataProductUpdateResponse){} + + /* + * Retrieving existing Data Product details in the ReplicaCatalog + */ + rpc getDataProduct(DataProductGetRequest) returns (DataProductGetResponse){} + + /* + * Removing a Data Product in the ReplicaCatalog + */ + rpc removeDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){} + + +} diff --git a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto new file mode 100644 index 0000000..5cac081 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.common; + +import "resource/s3/S3Storage.proto"; + + +enum StorageType { + S3 = 0; + SCP = 1; + FTP = 2; + LOCAL = 3; + BOX = 4; + DROPBOX = 5; + GCS = 6; + AZURE = 7; + SWIFT = 8; + ODATA = 9; +} + +enum Error { + NOT_FOUND = 0; + NO_PERMISSION = 1; + LIMIT_OVERFLOWED = 2; +} + +message StorageTypeResolveRequest { + string storage_id = 1; +} + +message StorageTypeResolveResponse { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; + Error error = 4; +} + +message SecretForStorage { + string storage_id = 1; + string secret_id = 2; + string replica_id = 3; + StorageType storage_type = 4; + Error error = 5; +} + +message SecretForStorageGetRequest { + string storage_id = 1; +} + +message SecretForStorageDeleteRequest { + string storage_id = 1; +} + +message SecretForStorageDeleteResponse { + bool status = 1; +} + +message StorageListEntry { + string storage_id = 1; + string storage_name = 2; + string replica_id = 3; + StorageType storage_type = 4; +} + +message StorageListResponse { + repeated StorageListEntry storage_list = 1; +} + +message StorageListRequest { + int32 page_number = 1; + int32 page_size = 2; +} + +message StorageSearchRequest { + oneof searchQuery { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; + } +} + + +message FileResource { + string resourcePath = 1; +} + +message DirectoryResource { + string resourcePath = 1; +} + +message GenericResource { + + string resourceId = 1; + + oneof resource { + org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 2; + org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 3; + } + + StorageWrapper storage =4; + +} + +message StorageWrapper { + oneof storage { + org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage s3Storage = 1; + } +} + +message GenericResourceGetRequest { + string resourceId = 1; +} + +message GenericResourceCreateRequest { + string storageId = 1; + GenericResource resource = 2; + StorageType storageType = 3; +} + +message GenericResourceUpdateRequest { + string resourceId = 1; + string storageId = 2; + oneof resource { + org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; + org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; + } +} + +message GenericResourceUpdateResponse { + string resourceId = 1; +} + +message GenericResourceDeleteRequest { + string resourceId = 1; +} + +message GenericResourceDeleteResponse { + bool status = 1; +} + +/* + * Replica storage entry related services + */ +service StorageCommonService { + rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); + + rpc registerSecretForStorage(SecretForStorage) returns (SecretForStorage); + + rpc getSecretForStorage(SecretForStorageGetRequest) returns (SecretForStorage); + + rpc deleteSecretsForStorage(SecretForStorageDeleteRequest) returns (SecretForStorageDeleteResponse); + + rpc searchStorages(StorageSearchRequest) returns (StorageListResponse); + + rpc listStorages(StorageListRequest) returns (StorageListResponse); + +/*****************************/ + rpc getGenericResource (GenericResourceGetRequest) returns + (org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource); + + rpc createGenericResource (GenericResourceCreateRequest) returns + (org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource); + + rpc updateGenericResource (GenericResourceUpdateRequest) returns (GenericResourceUpdateResponse); + + rpc deleteGenericResource (GenericResourceDeleteRequest) returns (GenericResourceDeleteResponse); +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/resource/s3/S3Storage.proto b/replica-catalog-api/stubs/src/main/proto/resource/s3/S3Storage.proto new file mode 100644 index 0000000..4b3a4d2 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/s3/S3Storage.proto @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.s3; + +message S3Storage { + string storage_id = 1; + string bucket_name = 2; + string region = 3; + string endpoint = 4; + bool useTLS = 5; + string name = 6; +} + +message S3StorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message S3StorageListResponse { + repeated S3Storage storages = 1; +} + +message S3StorageGetRequest { + string storageId = 1; +} + +message S3StorageCreateRequest { + string bucket_name = 1; + string region = 2; + string storage_id = 3; + string endpoint = 4; + bool useTLS = 5; + string name = 6; +} + +message S3StorageUpdateRequest { + string storage_id = 1; + string bucket_name = 2; + string region = 3; + string endpoint = 4; + bool useTLS = 5; + string name = 6; +} + +message S3StorageUpdateResponse { + string storage_id = 1; +} + + +message S3StorageDeleteRequest { + string storage_id = 1; +} + +message S3StorageDeleteResponse { + bool status = 1; +} + + diff --git a/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto b/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto new file mode 100644 index 0000000..ae6910c --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.secret.stubs.common; + +import "secret/s3/S3Credential.proto"; + +enum StorageType { + S3 = 0; + SCP = 1; + FTP = 2; + LOCAL = 3; + BOX = 4; + DROPBOX = 5; + GCS = 6; + AZURE = 7; + SWIFT = 8; + ODATA = 9; +} + +enum Error { + NOT_FOUND = 0; + NO_PERMISSION = 1; + LIMIT_OVERFLOWED = 2; +} + +message StorageTypeResolveRequest { + string secret_id = 1; +} + +message StorageTypeResolveResponse { + string secret_id = 1; + StorageType storage_type = 2; + Error error = 3; +} + +message StorageSecret { + string secret_id = 1; + StorageType storage_type = 2; + SecretWrapper secret = 3; + Error error = 4; +} + +message SecretWrapper { + oneof secret { + org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret s3Secret = 1; + + } +} + +message SecretGetRequest { + string secret_id = 1; +} + +message SecretDeleteRequest { + string secret_id = 1; +} + +message SecretDeleteResponse { + bool status = 1; +} + +message SecretListEntry { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; +} + +message SecretListResponse { + repeated SecretListEntry secret_list = 1; +} + +message SecretListRequest { + int32 page_number = 1; + int32 page_size = 2; +} + +message SecretSearchRequest { + oneof searchQuery { + string secret_id = 1; + StorageType storage_type = 2; + } +} + +/* + * Replica storage secret related services + */ +service SecretCommonService { + rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); + rpc registerSecret(StorageSecret) returns (StorageSecret); + rpc getSecret(SecretGetRequest) returns (StorageSecret); + rpc deleteSecrets(SecretDeleteRequest) returns (SecretDeleteResponse); + rpc searchSecrets(SecretSearchRequest) returns (SecretListResponse); + rpc listSecrets(SecretListRequest) returns (SecretListResponse); +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/s3/S3Credential.proto b/replica-catalog-api/stubs/src/main/proto/secret/s3/S3Credential.proto new file mode 100644 index 0000000..b59f2fe --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/s3/S3Credential.proto @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; + +package org.apache.airavata.replicacatalog.secret.stubs.s3; + +message S3Secret { + string secret_id = 1; + string access_key = 2; + string secret_key = 3; + string session_token = 4; +} + +message S3SecretGetRequest { + string secret_id = 1; +} + +message S3SecretCreateRequest { + string access_key = 1; + string secret_key = 2; + string session_token = 3; +} + +message S3SecretUpdateRequest { + string secret_id = 1; + string access_key = 2; + string secret_key = 3; + string session_token = 4; +} + +message S3SecretUpdateResponse { + string secret_id = 1; +} + +message S3SecretDeleteRequest { + string secret_id = 1; +} + +message S3SecretDeleteResponse { + bool status = 1; +} \ No newline at end of file From 25e04ec02c82dc22f50e150a654e92573a6753ef Mon Sep 17 00:00:00 2001 From: jayancv Date: Sun, 26 Mar 2023 16:35:29 +0530 Subject: [PATCH 2/8] Adding data replica catalog data retrival basic flow --- .../airavata/ReplicaCatalogAPIClient.java | 53 +++-- .../service/ReplicaCatalogAPIService.java | 11 +- .../impl/ReplicaCatalogServiceImp.java | 7 +- .../mapper/ResourceStorageMapper.java | 25 ++- .../resource/model/GenericResourceEntity.java | 11 + .../resource/model/S3StorageEntity.java | 2 +- .../repository/GenericResourceRepository.java | 1 + .../resource/service/IResourceService.java | 3 +- .../resource/service/ResourceAPIService.java | 28 ++- ...ceService.java => SQLResourceService.java} | 40 +++- .../sceret/service/ISecretService.java | 2 +- ...cretService.java => SQLSecretService.java} | 13 +- .../service/StorageSecretAPIService.java | 3 +- .../proto/resource/common/StorageCommon.proto | 206 +++++++++--------- .../proto/secret/common/SecretCommon.proto | 36 +-- 15 files changed, 274 insertions(+), 167 deletions(-) rename replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/{SQLIResourceService.java => SQLResourceService.java} (88%) rename replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/{SQLISecretService.java => SQLSecretService.java} (88%) diff --git a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java index 130aa65..9f25a68 100644 --- a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java +++ b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java @@ -11,13 +11,17 @@ import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommon; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretWrapper; import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; @@ -41,12 +45,6 @@ public StorageCommonServiceGrpc.StorageCommonServiceBlockingStub getBlockingStor return StorageCommonServiceGrpc.newBlockingStub(channel); } - // public GenericResourceServiceGrpc.GenericResourceServiceBlockingStub getBlockingResourceStub( Channel channel ) - // { - // return GenericResourceServiceGrpc.newBlockingStub( channel ); - // - // } - public SecretCommonServiceGrpc.SecretCommonServiceBlockingStub getBlockingSecretStub(Channel channel) { return SecretCommonServiceGrpc.newBlockingStub(channel); } @@ -90,6 +88,7 @@ public static void main(String[] args) throws InterruptedException { .setEndpoint("https://s3.us-east-1.amazonaws.com").build(); GenericResource resource = GenericResource.newBuilder() + .setReplicaId(replicaResult.getDataReplicaId()) .setStorage(StorageWrapper.newBuilder().setS3Storage(storage).build()) .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); GenericResource resourceResult = client.createStorage(channel, resource); @@ -101,18 +100,30 @@ public static void main(String[] args) throws InterruptedException { StorageSecret storageSecret = StorageSecret.newBuilder() .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3).build(); - - StorageSecret secretResult = client.createSecret(channel, storageSecret); + SecretCreateRequest secretCreateRequest = SecretCreateRequest.newBuilder().setSecret(storageSecret).build(); + StorageSecret secretResult = client.createSecret(channel, secretCreateRequest); SecretForStorage secretForStorage = SecretForStorage.newBuilder() .setStorageId(resourceResult.getStorage().getS3Storage().getStorageId()) - .setSecretId(secretResult.getSecretId()).setStorageType(StorageType.S3).build(); + .setSecretId(secretResult.getSecret().getS3Secret().getSecretId()).setStorageType(StorageType.S3).build(); + SecretForStorageCreateRequest createRequest = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage).build(); + SecretForStorage secretForStorageResult = client.createCommonStorage(channel, createRequest); - client.createCommonStorage(channel, secretForStorage); + System.out.println( + MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}]", + (Object[]) new String[]{replicaResult.getDataReplicaId(),secretForStorageResult.getStorageId(),secretForStorageResult.getSecretId()})); + DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(replicaResult.getDataReplicaId()).build(); + DataReplicaGetResponse replicaResponse = client.blockingApiStub.getReplicaLocation(replicaGetRequest); + if (replicaResponse != null && replicaResponse.getDataReplica() != null) { + SecretForStorage secretForStorage1= client.getStorageSecretIds(channel,replicaResponse.getDataReplica().getDataReplicaId()); + if(secretForStorage1!= null){ + System.out.println( + MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}]", + (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(),secretForStorage1.getStorageId(),secretForStorage1.getSecretId()})); - System.out.println( - MessageFormat.format("Created data product with id [{0}]", replicaResult.getDataReplicaId())); + } + } /* --- Sample scenario 2--- @@ -127,7 +138,7 @@ public static void main(String[] args) throws InterruptedException { */ } finally { - channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + channel.shutdownNow().awaitTermination(180, TimeUnit.SECONDS); } } @@ -146,9 +157,9 @@ public DataReplicaLocation createReplicaLocation(DataReplicaLocation replicaLoca } - public SecretForStorage createCommonStorage(Channel channel, SecretForStorage secretForStorage) { + public SecretForStorage createCommonStorage(Channel channel, SecretForStorageCreateRequest createRequest) { - SecretForStorage response = getBlockingStorageStub(channel).registerSecretForStorage(secretForStorage); + SecretForStorage response = getBlockingStorageStub(channel).registerSecretForStorage(createRequest); return response; } @@ -168,10 +179,18 @@ public GenericResource createStorage(Channel channel, GenericResource resource) return response; } - public StorageSecret createSecret(Channel channel, StorageSecret resource) { - StorageSecret response = getBlockingSecretStub(channel).registerSecret(resource); + public StorageSecret createSecret(Channel channel, SecretCreateRequest createRequest) { + StorageSecret response = getBlockingSecretStub(channel).registerSecret(createRequest); return response; } + public SecretForStorage getStorageSecretIds(Channel channel, String replicaId){ + GenericResourceGetRequest resourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaId).build(); + GenericResource resource = getBlockingStorageStub(channel).getGenericResource(resourceGetRequest); + SecretForStorageGetRequest request = SecretForStorageGetRequest.newBuilder().setStorageId(resource.getStorage().getS3Storage().getStorageId()).build(); + SecretForStorage secretForStorage = getBlockingStorageStub(channel).getSecretForStorage(request); + return secretForStorage; + } + } \ No newline at end of file diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java index 789939e..9969110 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java @@ -91,8 +91,15 @@ public void updateReplicaLocation(DataReplicaUpdateRequest request, @Override public void getReplicaLocation(DataReplicaGetRequest request, StreamObserver responseObserver) { - super.getReplicaLocation(request, responseObserver); - } + + logger.info("Loading Replica for a Replica ID : {}", request.getDataReplicaId()); + + DataReplicaLocation result = dataCatalogService.getDataReplica(request.getDataReplicaId()); + + DataReplicaGetResponse.Builder responseBuilder = DataReplicaGetResponse.newBuilder(); + responseBuilder.setDataReplica(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); } @Override public void removeReplicaLocation(DataReplicaDeleteRequest request, diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java index 12585b7..4e49ede 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java @@ -66,7 +66,12 @@ public DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplicaLoca @Override public DataReplicaLocation getDataReplica(String replicaId) { - return null; + + DataReplicaLocationEntity dataReplicaLocationEntity = dataReplicaLocationRepository.findByReplicaId(replicaId); + if (dataReplicaLocationEntity == null) { + logger.debug("Data Replica Location not exists"); + } + return toDataReplicaLocation(dataReplicaLocationEntity); } @Override diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java index 6832b90..d50422f 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java @@ -23,32 +23,37 @@ public class ResourceStorageMapper { @Autowired GenericResourceRepository genericResourceRepository; - public void mapGenericStorageModelToEntity( GenericResource storage, GenericResourceEntity resourceEntity) { + public void mapGenericStorageModelToEntity(GenericResource storage, GenericResourceEntity resourceEntity) { + resourceEntity.setReplicaId(storage.getReplicaId()); resourceEntity.setStorageId(storage.getResourceId()); - resourceEntity.setStorageType( GenericResourceEntity.StorageType.S3); - resourceEntity.setResourcePath( storage.getFile().getResourcePath() ); - resourceEntity.setResourceType( GenericResourceEntity.ResourceType.FILE ); - // TODO + resourceEntity.setStorageType(GenericResourceEntity.StorageType.S3); + resourceEntity.setResourcePath(storage.getFile().getResourcePath()); + resourceEntity.setResourceType(GenericResourceEntity.ResourceType.FILE); + // TODO } public void mapGenericStorageEntityToModel(GenericResourceEntity resourceEntity, StorageWrapper wrapper, GenericResource.Builder builder) { - builder.setResourceId( resourceEntity.getResourceId() ); - builder.setStorage(wrapper); + builder.setResourceId(resourceEntity.getResourceId()); + builder.setReplicaId(resourceEntity.getReplicaId()); + if (wrapper != null) { + builder.setStorage(wrapper); + } } public void mapStorageSecretModelToEntity(SecretForStorage storage, StorageSecretEntity resourceEntity) { resourceEntity.setStorageId(storage.getStorageId()); resourceEntity.setSecretId(storage.getSecretId()); - resourceEntity.setType( storage.getStorageType().name()); + resourceEntity.setType(storage.getStorageType().name()); // TODO } public void mapStorageSecretEntityToModel(StorageSecretEntity resourceEntity, SecretForStorage.Builder dataProductBuilder) { - dataProductBuilder.setStorageType( StorageType.valueOf( resourceEntity.getType() ) ).setSecretId( resourceEntity.getSecretId() ) - .setStorageId( resourceEntity.getStorageId() ); + dataProductBuilder.setStorageType(StorageType.valueOf(resourceEntity.getType())).setSecretId(resourceEntity.getSecretId()) + .setStorageId(resourceEntity.getStorageId()); } + } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java index 82b811d..bcda805 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java @@ -48,6 +48,9 @@ public enum StorageType { @Column(name = "RESOURCE_TYPE") private ResourceType resourceType; + @Column(name = "REPLICA_ID") + private String replicaId; + @Column(name = "STORAGE_ID") private String storageId; @@ -78,6 +81,14 @@ public void setResourceType(ResourceType resourceType) { this.resourceType = resourceType; } + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + public String getStorageId() { return storageId; } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java index 56e3c5f..93da9ef 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java @@ -28,7 +28,7 @@ public class S3StorageEntity { @Id - @Column(name = "S3_STORAGE_ID") + @Column(name = "STORAGE_ID") @GeneratedValue(generator = "uuid") @GenericGenerator(name = "uuid", strategy = "uuid2") private String storageId; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java index 8c65512..189cb1b 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java @@ -28,5 +28,6 @@ public interface GenericResourceRepository extends CrudRepository { Optional findByResourceId(String resourceId); List findByStorageId(String storageId); + List findByReplicaId(String replicaId); void deleteByStorageIdAndStorageType(String storageId, GenericResourceEntity.StorageType storageType); } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java index 5215113..2e6d407 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java @@ -31,7 +31,7 @@ public interface IResourceService { public void destroy(); public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception; - public SecretForStorage registerSecretForStorage(SecretForStorage request) throws Exception; + public SecretForStorage registerSecretForStorage(SecretForStorageCreateRequest request) throws Exception; public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) throws Exception; @@ -41,6 +41,7 @@ public interface IResourceService { public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception; + public GenericResource getGenericResource(GenericResourceGetRequest request) throws Exception; // public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception; // public Optional getS3Storage(S3StorageGetRequest request) throws Exception; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java index a11e08f..9f2ad79 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java @@ -10,7 +10,10 @@ import org.apache.airavata.replicacatalog.resource.repository.StorageSecretRepository; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; @@ -42,7 +45,19 @@ public void createGenericResource(GenericResourceCreateRequest request, StreamOb } @Override - public void registerSecretForStorage(SecretForStorage request, StreamObserver responseObserver) { + public void getGenericResource(GenericResourceGetRequest request, StreamObserver responseObserver) { + GenericResource genericResource = null; + try { + genericResource = resourceService.getGenericResource(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(genericResource); + responseObserver.onCompleted(); + } + + @Override + public void registerSecretForStorage(SecretForStorageCreateRequest request, StreamObserver responseObserver) { SecretForStorage secretForStorage = null; try { secretForStorage = resourceService.registerSecretForStorage(request); @@ -53,4 +68,15 @@ public void registerSecretForStorage(SecretForStorage request, StreamObserver responseObserver) { + SecretForStorage secretForStorage = null; + try { + secretForStorage = resourceService.getSecretForStorage(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(secretForStorage); + responseObserver.onCompleted(); + } } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java similarity index 88% rename from replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java rename to replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java index 00cb977..b1cced0 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLIResourceService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java @@ -43,9 +43,9 @@ import java.util.UUID; @Component("SQLResourceBackend") -public class SQLIResourceService implements IResourceService { +public class SQLResourceService implements IResourceService { - private static final Logger logger = LoggerFactory.getLogger(SQLIResourceService.class); + private static final Logger logger = LoggerFactory.getLogger(SQLResourceService.class); private DozerBeanMapper mapper = new DozerBeanMapper(); @Autowired private GenericResourceRepository resourceRepository; @@ -75,20 +75,44 @@ public void destroy() { @Override public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception { GenericResourceEntity resourceEntity = new GenericResourceEntity(); - resourceEntity.setResourceId(UUID.randomUUID().toString()); - resourceStorageMapper.mapGenericStorageModelToEntity(request.getResource(), resourceEntity); - GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); + S3Storage storage = null; if (request.getStorageType() == StorageType.S3) { storage = createS3Storage(request.getResource().getStorage().getS3Storage()); } StorageWrapper wrapper = StorageWrapper.newBuilder().setS3Storage(storage).build(); + resourceEntity.setResourceId(UUID.randomUUID().toString()); + resourceStorageMapper.mapGenericStorageModelToEntity(request.getResource(), resourceEntity); + resourceEntity.setStorageId(storage.getStorageId()); + GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); GenericResource.Builder responseBuilder = GenericResource.newBuilder(); resourceStorageMapper.mapGenericStorageEntityToModel(savedDataProductEntity, wrapper, responseBuilder); return responseBuilder.build(); } + @Override + public GenericResource getGenericResource(GenericResourceGetRequest request) throws Exception { + + List savedGenericResourceEntityList = resourceRepository.findByReplicaId(request.getReplicaId()); + if(savedGenericResourceEntityList.isEmpty()){ + return null; + } + GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(0); + Optional storage = null; + StorageWrapper wrapper = null; + if (StorageType.S3.name().equals(resourceEntity.getStorageType().name() )) { + storage = s3StorageRepository.findById(resourceEntity.getStorageId()); + if(storage.isPresent()){ + S3Storage s3 =mapper.map(storage.get(),S3Storage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); + } + } + + GenericResource.Builder responseBuilder = GenericResource.newBuilder(); + resourceStorageMapper.mapGenericStorageEntityToModel(resourceEntity, wrapper, responseBuilder); + return responseBuilder.build(); + } @Override public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception { @@ -105,15 +129,15 @@ public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) } @Override - public SecretForStorage registerSecretForStorage(SecretForStorage request) throws Exception { + public SecretForStorage registerSecretForStorage(SecretForStorageCreateRequest request) throws Exception { StorageSecretEntity resourceEntity = new StorageSecretEntity(); - resourceStorageMapper.mapStorageSecretModelToEntity(request, resourceEntity); + resourceStorageMapper.mapStorageSecretModelToEntity(request.getSecretForStorage(), resourceEntity); StorageSecretEntity savedDataProductEntity = resourceSecretRepository.save(resourceEntity); SecretForStorage.Builder responseBuilder = SecretForStorage.newBuilder(); resourceStorageMapper.mapStorageSecretEntityToModel(savedDataProductEntity, responseBuilder); - return request; + return responseBuilder.build(); } @Override diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java index 5c5be78..d637329 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java @@ -27,7 +27,7 @@ public interface ISecretService { public void destroy(); public StorageSecret getSecretForStorage(SecretGetRequest request) throws Exception; - public StorageSecret registerSecretForStorage(StorageSecret request) throws Exception; + public StorageSecret registerSecretForStorage(SecretCreateRequest request) throws Exception; public boolean deleteSecretForStorage(SecretDeleteRequest request) throws Exception; public SecretListResponse searchStorages(SecretSearchRequest request) throws Exception; public SecretListResponse listStorage(SecretListRequest request) throws Exception; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java similarity index 88% rename from replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java rename to replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java index 5c4c188..59db567 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLISecretService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java @@ -17,7 +17,6 @@ package org.apache.airavata.replicacatalog.sceret.service; -import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; @@ -33,9 +32,9 @@ import java.util.Optional; @Component("SQLSecretBackend") -public class SQLISecretService implements ISecretService { +public class SQLSecretService implements ISecretService { - private static final Logger logger = LoggerFactory.getLogger(SQLISecretService.class); + private static final Logger logger = LoggerFactory.getLogger(SQLSecretService.class); @Autowired @@ -62,10 +61,12 @@ public StorageSecret getSecretForStorage(SecretGetRequest request) throws Except } @Override - public StorageSecret registerSecretForStorage(StorageSecret request) throws Exception { + public StorageSecret registerSecretForStorage(SecretCreateRequest request) throws Exception { + StorageSecret stoReq = request.getSecret(); StorageSecret.Builder storageSecret = StorageSecret.newBuilder(); - if (request.getStorageType().name().equals(StorageType.S3.name())) { - return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret(createS3Secret(request.getSecret().getS3Secret()))).build(); + if (stoReq.getStorageType().name().equals(StorageType.S3.name())) { + S3Secret secret= createS3Secret(stoReq.getSecret().getS3Secret()); + return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret( secret )).setSecretId(secret.getSecretId()).build(); } return null; } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java index c806988..295591b 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java @@ -7,6 +7,7 @@ import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; @@ -27,7 +28,7 @@ public class StorageSecretAPIService extends SecretCommonServiceGrpc.SecretCommo ResourceSecretMapper secretMapper = new ResourceSecretMapper(); @Override - public void registerSecret(StorageSecret request, StreamObserver responseObserver) { + public void registerSecret(SecretCreateRequest request, StreamObserver responseObserver) { StorageSecret secretResult = null; try { secretResult = secretService.registerSecretForStorage(request); diff --git a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto index 5cac081..2269564 100644 --- a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto @@ -24,162 +24,164 @@ import "resource/s3/S3Storage.proto"; enum StorageType { - S3 = 0; - SCP = 1; - FTP = 2; - LOCAL = 3; - BOX = 4; - DROPBOX = 5; - GCS = 6; - AZURE = 7; - SWIFT = 8; - ODATA = 9; + S3 = 0; + SCP = 1; + FTP = 2; + LOCAL = 3; + BOX = 4; + DROPBOX = 5; + GCS = 6; + AZURE = 7; + SWIFT = 8; + ODATA = 9; } enum Error { - NOT_FOUND = 0; - NO_PERMISSION = 1; - LIMIT_OVERFLOWED = 2; + NOT_FOUND = 0; + NO_PERMISSION = 1; + LIMIT_OVERFLOWED = 2; } message StorageTypeResolveRequest { - string storage_id = 1; + string storage_id = 1; } message StorageTypeResolveResponse { - string storage_id = 1; - string storage_name = 2; - StorageType storage_type = 3; - Error error = 4; + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; + Error error = 4; } -message SecretForStorage { - string storage_id = 1; - string secret_id = 2; - string replica_id = 3; - StorageType storage_type = 4; - Error error = 5; -} -message SecretForStorageGetRequest { - string storage_id = 1; +message FileResource { + string resource_path = 1; } -message SecretForStorageDeleteRequest { - string storage_id = 1; +message DirectoryResource { + string resource_path = 1; } -message SecretForStorageDeleteResponse { - bool status = 1; -} +message GenericResource { + + string resource_id = 1; + string replica_id = 2; + oneof resource { + org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; + org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; + } + StorageWrapper storage = 5; -message StorageListEntry { - string storage_id = 1; - string storage_name = 2; - string replica_id = 3; - StorageType storage_type = 4; } -message StorageListResponse { - repeated StorageListEntry storage_list = 1; +// Storage wrapper which holds the different storages like S3, GCS, Azura ... +message StorageWrapper { + oneof storage { + org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage s3Storage = 1; + } } -message StorageListRequest { - int32 page_number = 1; - int32 page_size = 2; +message GenericResourceGetRequest { + string replica_id = 1; + string resource_id = 2; } -message StorageSearchRequest { - oneof searchQuery { - string storage_id = 1; - string storage_name = 2; - StorageType storage_type = 3; - } +message GenericResourceCreateRequest { + string storage_id = 1; + GenericResource resource = 2; + StorageType storage_type = 3; } +message GenericResourceUpdateRequest { + string resource_id = 1; + string storage_id = 2; + oneof resource { + org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; + org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; + } + GenericResource storage = 5; -message FileResource { - string resourcePath = 1; } -message DirectoryResource { - string resourcePath = 1; +message GenericResourceUpdateResponse { + string resource_id = 1; } -message GenericResource { - - string resourceId = 1; +message GenericResourceDeleteRequest { + string resource_id = 1; +} - oneof resource { - org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 2; - org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 3; - } +message GenericResourceDeleteResponse { + bool status = 1; +} - StorageWrapper storage =4; +// Link storage and secret +message SecretForStorage { + string storage_id = 1; + string secret_id = 2; + string replica_id = 3; + StorageType storage_type = 4; + Error error = 5; +} +message SecretForStorageCreateRequest { + SecretForStorage secret_for_storage = 1; } -message StorageWrapper { - oneof storage { - org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage s3Storage = 1; - } +message SecretForStorageGetRequest { + string storage_id = 1; } -message GenericResourceGetRequest { - string resourceId = 1; +message SecretForStorageDeleteRequest { + string storage_id = 1; } -message GenericResourceCreateRequest { - string storageId = 1; - GenericResource resource = 2; - StorageType storageType = 3; +message SecretForStorageDeleteResponse { + bool status = 1; } -message GenericResourceUpdateRequest { - string resourceId = 1; - string storageId = 2; - oneof resource { - org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; - org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; - } +message StorageListEntry { + string storage_id = 1; + string storage_name = 2; + string replica_id = 3; + StorageType storage_type = 4; } -message GenericResourceUpdateResponse { - string resourceId = 1; +message StorageListResponse { + repeated StorageListEntry storage_list = 1; } -message GenericResourceDeleteRequest { - string resourceId = 1; +message StorageListRequest { + int32 page_number = 1; + int32 page_size = 2; } -message GenericResourceDeleteResponse { - bool status = 1; +message StorageSearchRequest { + oneof searchQuery { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; + } } + /* * Replica storage entry related services */ service StorageCommonService { - rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); - - rpc registerSecretForStorage(SecretForStorage) returns (SecretForStorage); - - rpc getSecretForStorage(SecretForStorageGetRequest) returns (SecretForStorage); - - rpc deleteSecretsForStorage(SecretForStorageDeleteRequest) returns (SecretForStorageDeleteResponse); - - rpc searchStorages(StorageSearchRequest) returns (StorageListResponse); - - rpc listStorages(StorageListRequest) returns (StorageListResponse); - -/*****************************/ - rpc getGenericResource (GenericResourceGetRequest) returns - (org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource); - rpc createGenericResource (GenericResourceCreateRequest) returns - (org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource); + /* Replica catalog resource storage operations */ + rpc createGenericResource (GenericResourceCreateRequest) returns (GenericResource); // Create Generic Resource and Create storage + rpc getGenericResource (GenericResourceGetRequest) returns (GenericResource); + rpc updateGenericResource (GenericResourceUpdateRequest) returns (GenericResourceUpdateResponse); + rpc deleteGenericResource (GenericResourceDeleteRequest) returns (GenericResourceDeleteResponse); - rpc updateGenericResource (GenericResourceUpdateRequest) returns (GenericResourceUpdateResponse); + /* Replica catalog resource storage and secret related operations */ + rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); // Get resource type + rpc registerSecretForStorage (SecretForStorageCreateRequest) returns (SecretForStorage); // Add new resource to replica catalog + rpc getSecretForStorage (SecretForStorageGetRequest) returns (SecretForStorage); // Get storage secret using storage ID + rpc deleteSecretsForStorage (SecretForStorageDeleteRequest) returns (SecretForStorageDeleteResponse); // Delete storage secret mapping + rpc searchStorages (StorageSearchRequest) returns (StorageListResponse); // Search storage secrets using storage id, name or the type + rpc listStorages (StorageListRequest) returns (StorageListResponse); // List storages secrets - rpc deleteGenericResource (GenericResourceDeleteRequest) returns (GenericResourceDeleteResponse); } \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto b/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto index ae6910c..1e502aa 100644 --- a/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto @@ -40,16 +40,16 @@ enum Error { NO_PERMISSION = 1; LIMIT_OVERFLOWED = 2; } - -message StorageTypeResolveRequest { - string secret_id = 1; -} - -message StorageTypeResolveResponse { - string secret_id = 1; - StorageType storage_type = 2; - Error error = 3; -} +// +//message StorageTypeResolveRequest { +// string secret_id = 1; +//} +// +//message StorageTypeResolveResponse { +// string secret_id = 1; +// StorageType storage_type = 2; +// Error error = 3; +//} message StorageSecret { string secret_id = 1; @@ -65,6 +65,10 @@ message SecretWrapper { } } +message SecretCreateRequest { + StorageSecret secret = 1; +} + message SecretGetRequest { string secret_id = 1; } @@ -103,10 +107,10 @@ message SecretSearchRequest { * Replica storage secret related services */ service SecretCommonService { - rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); - rpc registerSecret(StorageSecret) returns (StorageSecret); - rpc getSecret(SecretGetRequest) returns (StorageSecret); - rpc deleteSecrets(SecretDeleteRequest) returns (SecretDeleteResponse); - rpc searchSecrets(SecretSearchRequest) returns (SecretListResponse); - rpc listSecrets(SecretListRequest) returns (SecretListResponse); +// rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); // Get secret type + rpc registerSecret (SecretCreateRequest) returns (StorageSecret); // Add new secret to replica catalog + rpc getSecret (SecretGetRequest) returns (StorageSecret); // Get secret using secret Id (Secret Id can retrieve from the replica ID using replica storage id-secret id linking (StorageSecretEntity)) + rpc deleteSecrets (SecretDeleteRequest) returns (SecretDeleteResponse); // Delete secret suing secret ID + rpc searchSecrets (SecretSearchRequest) returns (SecretListResponse); // Search secrets using secret Id or the secret type + rpc listSecrets (SecretListRequest) returns (SecretListResponse); // List all the secrets } \ No newline at end of file From af733b4bd2d2761d31a84b432e5647621184220d Mon Sep 17 00:00:00 2001 From: jayancv Date: Sat, 1 Apr 2023 01:43:19 +0530 Subject: [PATCH 3/8] Update entity table names --- .../resource/model/GenericResourceEntity.java | 2 ++ .../replicacatalog/resource/model/ResolveStorageEntity.java | 2 ++ .../replicacatalog/resource/model/S3StorageEntity.java | 2 ++ .../replicacatalog/resource/model/StorageSecretEntity.java | 6 +++++- .../replicacatalog/sceret/model/S3SecretEntity.java | 2 ++ 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java index bcda805..091bd71 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java @@ -21,11 +21,13 @@ import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; +import jakarta.persistence.Table; import org.hibernate.annotations.GenericGenerator; @Entity +@Table(name = "REPLICA_RESOURCE") public class GenericResourceEntity { public enum ResourceType { diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java index 12c48bd..b0a1574 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java @@ -20,9 +20,11 @@ import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.Id; +import jakarta.persistence.Table; @Entity +@Table(name = "REPLICA_STORAGE") public class ResolveStorageEntity { public enum StorageType { diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java index 93da9ef..81f04f3 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java @@ -21,10 +21,12 @@ import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; +import jakarta.persistence.Table; import org.hibernate.annotations.GenericGenerator; @Entity +@Table(name = "S3_STORAGE") public class S3StorageEntity { @Id diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java index 820a2c1..dbe23c0 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java @@ -21,16 +21,20 @@ import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; +import jakarta.persistence.Table; import org.hibernate.annotations.GenericGenerator; @Entity +@Table(name = "STORAGE_SECRET") public class StorageSecretEntity { @Id - @Column(name = "RESOURCE_SECRET_ID") + @Column(name = "STORAGE_SECRET_ID") @GeneratedValue(generator = "uuid") @GenericGenerator(name = "uuid", strategy = "uuid2") private String id; + // Identifier can add here + @Column(name = "STORAGE_ID") private String storageId; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java index a17cf29..363467c 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java @@ -21,11 +21,13 @@ import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; +import jakarta.persistence.Table; import org.hibernate.annotations.GenericGenerator; @Entity +@Table(name = "S3_SECRET") public class S3SecretEntity { @Id From 104220901d952bf0b7e53034e00672783fe9d451 Mon Sep 17 00:00:00 2001 From: jayancv Date: Tue, 4 Apr 2023 23:23:00 +0530 Subject: [PATCH 4/8] Update replica catalog service implementations --- .../catalogapi/mapper/DataProductMapper.java | 22 ++++- .../service/IReplicaCatalogService.java | 16 ++-- .../service/ReplicaCatalogAPIService.java | 88 ++++++++++++------- .../impl/ReplicaCatalogServiceImp.java | 78 +++++++++------- .../proto/catalogapi/ReplicaCatalogAPI.proto | 19 ++-- 5 files changed, 137 insertions(+), 86 deletions(-) diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java index 3a17cb5..c33e0a5 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java @@ -1,6 +1,8 @@ package org.apache.airavata.replicacatalog.catalogapi.mapper; import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,13 +27,19 @@ public class DataProductMapper { public void mapModelToEntity(DataProduct dataProduct, DataProductEntity dataProductEntity) { - dataProductEntity.setProductUri(dataProduct.getProductUri()); + // For new data product entities + if (dataProductEntity.getProductUri() == null) { + dataProductEntity.setProductUri(dataProduct.getProductUri()); + dataProductEntity.setCreationTime(new Timestamp(System.currentTimeMillis())); + } dataProductEntity.setProductName(dataProduct.getProductName()); dataProductEntity.setProductDescription(dataProduct.getProductDescription()); dataProductEntity.setDataProductType(dataProduct.getDataProductType()); dataProductEntity.setParentProductUri(dataProduct.getParentProductUri()); - dataProductEntity.setCreationTime(new Timestamp(System.currentTimeMillis())); + dataProductEntity.setLastModifiedTime(new Timestamp(System.currentTimeMillis())); + //TODO Map metadata + mapMetaData(dataProduct, dataProductEntity); } public void mapEntityToModel(DataProductEntity dataProductEntity, DataProduct.Builder dataProductBuilder) { @@ -44,4 +52,14 @@ public void mapEntityToModel(DataProductEntity dataProductEntity, DataProduct.Bu .setCreationTime(dataProductEntity.getCreationTime().getTime()).build(); } + + private void mapMetaData(DataProduct dataProduct, DataProductEntity dataProductEntity) { + Map newMetaData = new HashMap<>(); + dataProduct.getMetadataMap().keySet().forEach(m -> { + newMetaData.put(m, dataProduct.getMetadataMap().get(m)); + }); + dataProductEntity.setProductMetadata(newMetaData); + } + + } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java index d2fa9ed..bb27bf7 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java @@ -7,15 +7,6 @@ public interface IReplicaCatalogService { - DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica); - - DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica); - - DataReplicaLocation getDataReplica(String replicaId); - - void deleteDataReplica(String replicaId); - - DataProduct createDataProduct(DataProduct dataProduct); DataProduct updateDataProduct(DataProduct dataProduct); @@ -24,5 +15,12 @@ public interface IReplicaCatalogService { void deleteDataProduct(String productUri); + DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica); + + DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica); + + DataReplicaLocation getDataReplica(String replicaId); + + void deleteDataReplica(String replicaId); } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java index 9969110..9e1d7d6 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java @@ -1,13 +1,31 @@ package org.apache.airavata.replicacatalog.catalogapi.service; import io.grpc.stub.StreamObserver; - import org.apache.airavata.replicacatalog.catalog.service.ReplicaCatalogAPIServiceGrpc; -import org.apache.airavata.replicacatalog.catalog.stubs.*; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaDeleteRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaDeleteResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductDeleteRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductDeleteResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductUpdateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataProductUpdateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaDeleteRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaDeleteResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateResponse; import org.apache.airavata.replicacatalog.catalogapi.mapper.DataProductMapper; import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; -import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; -import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,47 +46,51 @@ public class ReplicaCatalogAPIService extends ReplicaCatalogAPIServiceGrpc.Repli DataReplicaMapper replicaMapper = new DataReplicaMapper(); @Autowired - IReplicaCatalogService dataCatalogService; + IReplicaCatalogService catalogService; @Override - public void registerDataProduct(DataProductCreateRequest request, - StreamObserver responseObserver) { - - DataProduct result = dataCatalogService.createDataProduct(request.getDataProduct()); + public void registerDataProduct(DataProductCreateRequest request, StreamObserver responseObserver) { + DataProduct result = catalogService.createDataProduct(request.getDataProduct()); DataProductCreateResponse.Builder responseBuilder = DataProductCreateResponse.newBuilder(); responseBuilder.setDataProduct(result); responseObserver.onNext(responseBuilder.build()); responseObserver.onCompleted(); - } @Override - public void updateDataProduct(DataProductUpdateRequest request, - StreamObserver responseObserver) { - super.updateDataProduct(request, responseObserver); + public void updateDataProduct(DataProductUpdateRequest request, StreamObserver responseObserver) { + DataProduct result = catalogService.updateDataProduct(request.getDataProduct()); + DataProductUpdateResponse.Builder responseBuilder = DataProductUpdateResponse.newBuilder(); + responseBuilder.setDataProduct(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); } @Override public void getDataProduct(DataProductGetRequest request, StreamObserver responseObserver) { - super.getDataProduct(request, responseObserver); + DataProduct result = catalogService.getDataProduct(request.getDataProductUri()); + DataProductGetResponse.Builder responseBuilder = DataProductGetResponse.newBuilder(); + responseBuilder.setDataProduct(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); } @Override - public void removeDataProduct(DataProductDeleteRequest request, - StreamObserver responseObserver) { - super.removeDataProduct(request, responseObserver); + public void removeDataProduct(DataProductDeleteRequest request, StreamObserver responseObserver) { + catalogService.deleteDataProduct(request.getDataProductUri()); + responseObserver.onNext(DataProductDeleteResponse.newBuilder().build()); + responseObserver.onCompleted(); } @Override - public void registerReplicaLocation(DataReplicaCreateRequest request, - StreamObserver responseObserver) { + public void registerReplicaLocation(DataReplicaCreateRequest request, StreamObserver responseObserver) { logger.info("Creating Replica for a data product {}", request.getDataReplica()); if (request.getDataReplica() == null) { logger.debug("No Data Replica Location"); } - DataReplicaLocation result = dataCatalogService.createDataReplica(request.getDataReplica()); + DataReplicaLocation result = catalogService.createDataReplica(request.getDataReplica()); DataReplicaCreateResponse.Builder responseBuilder = DataReplicaCreateResponse.newBuilder(); responseBuilder.setDataReplica(result); @@ -77,10 +99,9 @@ public void registerReplicaLocation(DataReplicaCreateRequest request, } @Override - public void updateReplicaLocation(DataReplicaUpdateRequest request, - StreamObserver responseObserver) { + public void updateReplicaLocation(DataReplicaUpdateRequest request, StreamObserver responseObserver) { - DataReplicaLocation result = dataCatalogService.updateDataReplica(request.getDataReplica()); + DataReplicaLocation result = catalogService.updateDataReplica(request.getDataReplica()); DataReplicaUpdateResponse.Builder responseBuilder = DataReplicaUpdateResponse.newBuilder(); responseBuilder.setDataReplica(result); @@ -89,33 +110,32 @@ public void updateReplicaLocation(DataReplicaUpdateRequest request, } @Override - public void getReplicaLocation(DataReplicaGetRequest request, - StreamObserver responseObserver) { + public void getReplicaLocation(DataReplicaGetRequest request, StreamObserver responseObserver) { logger.info("Loading Replica for a Replica ID : {}", request.getDataReplicaId()); - DataReplicaLocation result = dataCatalogService.getDataReplica(request.getDataReplicaId()); + DataReplicaLocation result = catalogService.getDataReplica(request.getDataReplicaId()); DataReplicaGetResponse.Builder responseBuilder = DataReplicaGetResponse.newBuilder(); responseBuilder.setDataReplica(result); responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); } + responseObserver.onCompleted(); + } @Override - public void removeReplicaLocation(DataReplicaDeleteRequest request, - StreamObserver responseObserver) { - super.removeReplicaLocation(request, responseObserver); + public void removeReplicaLocation(DataReplicaDeleteRequest request, StreamObserver responseObserver) { + catalogService.deleteDataReplica(request.getDataReplicaId()); + responseObserver.onNext(DataReplicaDeleteResponse.newBuilder().build()); + responseObserver.onCompleted(); } @Override - public void getAllReplicaLocation(AllDataReplicaGetRequest request, - StreamObserver responseObserver) { + public void getAllReplicaLocation(AllDataReplicaGetRequest request, StreamObserver responseObserver) { super.getAllReplicaLocation(request, responseObserver); } @Override - public void removeAllReplicaLocation(AllDataReplicaDeleteRequest request, - StreamObserver responseObserver) { + public void removeAllReplicaLocation(AllDataReplicaDeleteRequest request, StreamObserver responseObserver) { super.removeAllReplicaLocation(request, responseObserver); } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java index 4e49ede..5d39094 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java @@ -1,5 +1,6 @@ package org.apache.airavata.replicacatalog.catalogapi.service.impl; +import jakarta.persistence.EntityNotFoundException; import jakarta.transaction.Transactional; import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; @@ -35,6 +36,50 @@ public class ReplicaCatalogServiceImp implements IReplicaCatalogService { DataReplicaMapper replicaMapper = new DataReplicaMapper(); + + @Override + public DataProduct createDataProduct(DataProduct dataProduct) { + DataProductEntity dataProductEntity = new DataProductEntity(); + + if (dataProduct != null && dataProduct.getProductUri() != null) { + dataProductEntity.setProductUri(dataProduct.getProductUri()); + } + dataProductMapper.mapModelToEntity(dataProduct, dataProductEntity); + + if (dataProductEntity.getProductUri() == null) { + dataProductEntity.setProductUri(UUID.randomUUID().toString()); + } + DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); + return toDataProduct(savedDataProductEntity); + } + + @Override + public DataProduct updateDataProduct(DataProduct dataProduct) { + + DataProductEntity dataProductEntity = dataProductRepository.findByProductUri(dataProduct.getProductUri()); + if (dataProductEntity == null) { + logger.warn("Data product not exists"); + throw new EntityNotFoundException("Could not find a data product with the ID: " + dataProduct.getProductUri()); + } + dataProductMapper.mapModelToEntity(dataProduct, dataProductEntity); + DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); + + return toDataProduct(savedDataProductEntity); + } + + @Override + public DataProduct getDataProduct(String productUri) { + DataProductEntity dataProductEntity = dataProductRepository.findByProductUri(productUri); + return toDataProduct(dataProductEntity); + } + + @Override + public void deleteDataProduct(String productUri) { + dataProductRepository.deleteByProductUri(productUri); + } + + + @Override public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation) { @@ -76,38 +121,7 @@ public DataReplicaLocation getDataReplica(String replicaId) { @Override public void deleteDataReplica(String replicaId) { - - } - - @Override - public DataProduct createDataProduct(DataProduct dataProduct) { - DataProductEntity dataProductEntity = new DataProductEntity(); - - if (dataProduct != null && dataProduct.getProductUri() != null) { - dataProductEntity.setProductUri(dataProduct.getProductUri()); - } - dataProductMapper.mapModelToEntity(dataProduct, dataProductEntity); - - if (dataProductEntity.getProductUri() == null) { - dataProductEntity.setProductUri(UUID.randomUUID().toString()); - } - DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); - return toDataProduct(savedDataProductEntity); - } - - @Override - public DataProduct updateDataProduct(DataProduct dataProduct) { - return null; - } - - @Override - public DataProduct getDataProduct(String productUri) { - return null; - } - - @Override - public void deleteDataProduct(String productUri) { - + dataReplicaLocationRepository.deleteByReplicaId(replicaId); } diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto index 9909885..ef3bf79 100644 --- a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto @@ -73,15 +73,16 @@ enum DataProductType { // Entity which used to model a replica entry message DataReplicaLocation { string data_replica_id = 1; // Unique replica id - string data_product_id = 2; - string storage_resource_id = 3; // Replica storage id (TO link the storage entry) - string replica_name = 4; // Replica user friendly name - string replica_description = 5; // Replica description - int64 creation_time = 6; // Replica creation date and time - int64 last_modified_time = 7; // Last modified time of replica - int64 valid_until_time = 8; // Validity time of the replica - StorageType storage_type = 9; // Storage type of the replica - map metadata = 10; // Replica's metadata information + string data_product_id = 2; // related data uri + string parent_replica_id = 3; // If there is parent replica (Like directory structure) + string storage_resource_id = 4; // Replica storage id (TO link the storage entry) + string replica_name = 5; // Replica user friendly name + string replica_description = 6; // Replica description + int64 creation_time = 7; // Replica creation date and time + int64 last_modified_time = 8; // Last modified time of replica + int64 valid_until_time = 9; // Validity time of the replica + StorageType storage_type = 10; // Storage type of the replica + map metadata = 11; // Replica's metadata information } message DataReplicaCreateRequest { From 14b26025258dc8fe038229f7be61b778faddbe4a Mon Sep 17 00:00:00 2001 From: jayancv Date: Wed, 5 Apr 2023 23:13:19 +0530 Subject: [PATCH 5/8] Update resource service implementations --- .../catalogapi/mapper/DataProductMapper.java | 7 +- .../repository/GenericResourceRepository.java | 4 +- .../resource/service/IResourceService.java | 4 + .../resource/service/ResourceAPIService.java | 44 ++++++++--- .../resource/service/SQLResourceService.java | 74 ++++++++++++++++--- .../proto/resource/common/StorageCommon.proto | 2 +- 6 files changed, 113 insertions(+), 22 deletions(-) diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java index c33e0a5..5c149f7 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java @@ -30,7 +30,6 @@ public void mapModelToEntity(DataProduct dataProduct, DataProductEntity dataProd // For new data product entities if (dataProductEntity.getProductUri() == null) { dataProductEntity.setProductUri(dataProduct.getProductUri()); - dataProductEntity.setCreationTime(new Timestamp(System.currentTimeMillis())); } dataProductEntity.setProductName(dataProduct.getProductName()); dataProductEntity.setProductDescription(dataProduct.getProductDescription()); @@ -38,6 +37,12 @@ public void mapModelToEntity(DataProduct dataProduct, DataProductEntity dataProd dataProductEntity.setParentProductUri(dataProduct.getParentProductUri()); dataProductEntity.setLastModifiedTime(new Timestamp(System.currentTimeMillis())); + if (dataProduct.getCreationTime() < 1) { + dataProductEntity.setCreationTime(new Timestamp(System.currentTimeMillis())); + } else { + dataProductEntity.setCreationTime(new Timestamp(dataProduct.getCreationTime())); + } + //TODO Map metadata mapMetaData(dataProduct, dataProductEntity); } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java index 189cb1b..8c3ab5c 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java @@ -27,7 +27,7 @@ @Repository public interface GenericResourceRepository extends CrudRepository { Optional findByResourceId(String resourceId); - List findByStorageId(String storageId); - List findByReplicaId(String replicaId); + Optional findByReplicaId(String replicaId); void deleteByStorageIdAndStorageType(String storageId, GenericResourceEntity.StorageType storageType); + Optional deleteByResourceId(String resourceId); } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java index 2e6d407..3ffe118 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java @@ -43,6 +43,10 @@ public interface IResourceService { public GenericResource getGenericResource(GenericResourceGetRequest request) throws Exception; + public GenericResource updateGenericResource(GenericResourceUpdateRequest request) throws Exception; + + public GenericResource deleteGenericResource(GenericResourceDeleteRequest request) throws Exception; + // public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception; // public Optional getS3Storage(S3StorageGetRequest request) throws Exception; // public S3Storage createS3Storage(S3StorageCreateRequest request) throws Exception; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java index 9f2ad79..6c4df13 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java @@ -1,16 +1,13 @@ package org.apache.airavata.replicacatalog.resource.service; import io.grpc.stub.StreamObserver; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateResponse; -import org.apache.airavata.replicacatalog.catalogapi.service.IReplicaCatalogService; -import org.apache.airavata.replicacatalog.resource.mapper.ResourceStorageMapper; -import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; -import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; -import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; -import org.apache.airavata.replicacatalog.resource.repository.StorageSecretRepository; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceDeleteRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceDeleteResponse; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceUpdateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceUpdateResponse; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; @@ -20,8 +17,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import java.util.UUID; - @GRpcService public class ResourceAPIService extends StorageCommonServiceGrpc.StorageCommonServiceImplBase { private static final Logger logger = LoggerFactory.getLogger(ResourceAPIService.class); @@ -56,6 +51,37 @@ public void getGenericResource(GenericResourceGetRequest request, StreamObserver responseObserver.onCompleted(); } + @Override + public void updateGenericResource(GenericResourceUpdateRequest request, StreamObserver responseObserver) { + + GenericResource genericResource = null; + try { + genericResource = resourceService.updateGenericResource(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + + GenericResourceUpdateResponse.Builder responseBuilder = GenericResourceUpdateResponse.newBuilder(); + responseBuilder.setResourceId(genericResource.getResourceId()); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void deleteGenericResource(GenericResourceDeleteRequest request, StreamObserver responseObserver) { + GenericResource genericResource = null; + try { + genericResource = resourceService.deleteGenericResource(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + + GenericResourceDeleteResponse.Builder responseBuilder = GenericResourceDeleteResponse.newBuilder(); + responseBuilder.setStatus(genericResource != null); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + @Override public void registerSecretForStorage(SecretForStorageCreateRequest request, StreamObserver responseObserver) { SecretForStorage secretForStorage = null; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java index b1cced0..509220a 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java @@ -94,17 +94,22 @@ public GenericResource createGenericResource(GenericResourceCreateRequest reques @Override public GenericResource getGenericResource(GenericResourceGetRequest request) throws Exception { - List savedGenericResourceEntityList = resourceRepository.findByReplicaId(request.getReplicaId()); - if(savedGenericResourceEntityList.isEmpty()){ + Optional savedGenericResourceEntityList = Optional.empty(); + if (!request.getReplicaId().isBlank()) { + savedGenericResourceEntityList = resourceRepository.findByReplicaId(request.getReplicaId()); + } else if (!request.getResourceId().isBlank()) { + savedGenericResourceEntityList = resourceRepository.findByResourceId(request.getResourceId()); + } + if (savedGenericResourceEntityList.isEmpty()) { return null; } - GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(0); - Optional storage = null; + GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(); + Optional storage = Optional.empty(); StorageWrapper wrapper = null; - if (StorageType.S3.name().equals(resourceEntity.getStorageType().name() )) { + if (StorageType.S3.name().equals(resourceEntity.getStorageType().name())) { storage = s3StorageRepository.findById(resourceEntity.getStorageId()); - if(storage.isPresent()){ - S3Storage s3 =mapper.map(storage.get(),S3Storage.newBuilder().getClass()).build(); + if (storage.isPresent()) { + S3Storage s3 = mapper.map(storage.get(), S3Storage.newBuilder().getClass()).build(); wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); } } @@ -114,6 +119,51 @@ public GenericResource getGenericResource(GenericResourceGetRequest request) thr return responseBuilder.build(); } + @Override + public GenericResource updateGenericResource(GenericResourceUpdateRequest request) throws Exception { + + Optional savedGenericResourceEntityList = Optional.empty(); + if (!request.getResourceId().isBlank()) { + + savedGenericResourceEntityList = resourceRepository.findByResourceId(request.getResourceId()); + + } else if (request.getGenericResource().hasStorage()) { + + savedGenericResourceEntityList = resourceRepository.findByResourceId(request.getGenericResource().getResourceId()); + + } + if (savedGenericResourceEntityList.isEmpty()) { + return null; + } + GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(); + Optional storage = null; + StorageWrapper wrapper = null; + if (StorageType.S3.name().equals(resourceEntity.getStorageType().name() )) { + storage = s3StorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + boolean updated = updateS3Storage(storage.get(), request.getGenericResource().getStorage().getS3Storage()); + if (updated) { + S3Storage s3 = mapper.map(storage.get(), S3Storage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); + } + } + } + + resourceStorageMapper.mapGenericStorageModelToEntity(request.getGenericResource(), resourceEntity); + + GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); + + GenericResource.Builder responseBuilder = GenericResource.newBuilder(); + resourceStorageMapper.mapGenericStorageEntityToModel(savedDataProductEntity, wrapper, responseBuilder); + return responseBuilder.build(); + } + + @Override + public GenericResource deleteGenericResource(GenericResourceDeleteRequest request) throws Exception { + resourceRepository.deleteByResourceId(request.getResourceId()); + return null; + } + @Override public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception { Optional resourceSecEtyOp = resourceSecretRepository.findByStorageId(request.getStorageId()); @@ -226,8 +276,14 @@ public S3Storage createS3Storage(S3Storage request) throws Exception { } - public boolean updateS3Storage(S3StorageUpdateRequest request) throws Exception { - s3StorageRepository.save(mapper.map(request, S3StorageEntity.class)); + public boolean updateS3Storage(S3StorageEntity entity, S3Storage storage) throws Exception { + + entity.setEndpoint(storage.getEndpoint()); + entity.setName(storage.getName()); + entity.setBucketName(storage.getBucketName()); + entity.setRegion(storage.getRegion()); + entity.setUseTLS(storage.getUseTLS()); + s3StorageRepository.save(entity); return true; } diff --git a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto index 2269564..1df0c03 100644 --- a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto @@ -99,7 +99,7 @@ message GenericResourceUpdateRequest { org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; } - GenericResource storage = 5; + GenericResource genericResource = 5; } From 773936a0cc13cc27aab480ac977ae4e187473dbe Mon Sep 17 00:00:00 2001 From: jayancv Date: Sun, 9 Apr 2023 20:49:15 +0530 Subject: [PATCH 6/8] removed product register from replica catalog --- .../airavata/ReplicaCatalogAPIClient.java | 22 +-- .../catalogapi/mapper/DataProductMapper.java | 70 -------- .../catalogapi/mapper/DataReplicaMapper.java | 15 +- .../catalogapi/model/DataProductEntity.java | 163 ------------------ .../model/DataProductMetadataEntity.java | 71 -------- .../model/DataProductMetadataPK.java | 76 -------- .../model/DataReplicaLocationEntity.java | 35 ---- .../repository/DataProductRepository.java | 15 -- .../service/IReplicaCatalogService.java | 11 -- .../service/ReplicaCatalogAPIService.java | 49 ------ .../impl/ReplicaCatalogServiceImp.java | 63 +------ .../resource/model/StorageSecretEntity.java | 13 +- .../proto/catalogapi/ReplicaCatalogAPI.proto | 44 ----- .../catalogapi/ReplicaCatalogAPIService.proto | 22 --- .../proto/resource/common/StorageCommon.proto | 5 +- 15 files changed, 28 insertions(+), 646 deletions(-) delete mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java delete mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java delete mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java delete mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java delete mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java diff --git a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java index 9f25a68..e9c877f 100644 --- a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java +++ b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws InterruptedException { Airavata MFT copied a 100GB astrological data file to Amazon S3 bucket User can access that S3 bucket data file via Replica catalog details - 1) Register Data Product + 1) Register Data Product in data catalog 2) Register Replica Location 3) Register Location storage details 4) Register location storage credentials @@ -67,17 +67,12 @@ public static void main(String[] args) throws InterruptedException { try { ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient(channel); - DataProduct dataProduct = DataProduct.newBuilder() - .setDataProductType(DataProductType.FILE) - .setProductName("ASTRO") - .setProductUri("613") - .setProductDescription("Astro Data").build(); - DataProduct productResult = client.createDataProduct(dataProduct); + String testUri = "TestUri"; DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder() .setReplicaName("ASTRO S3") .setReplicaDescription("S3 replica") - .setDataProductId(productResult.getProductUri()) + .setDataProductId(testUri) .setCreationTime(System.currentTimeMillis()).build(); DataReplicaLocation replicaResult = client.createReplicaLocation(replicaLocation); @@ -105,7 +100,9 @@ public static void main(String[] args) throws InterruptedException { SecretForStorage secretForStorage = SecretForStorage.newBuilder() .setStorageId(resourceResult.getStorage().getS3Storage().getStorageId()) - .setSecretId(secretResult.getSecret().getS3Secret().getSecretId()).setStorageType(StorageType.S3).build(); + .setSecretId(secretResult.getSecret().getS3Secret().getSecretId()) + .setStorageType(StorageType.S3) + .setUserIdentity("userID#12345").build(); SecretForStorageCreateRequest createRequest = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage).build(); SecretForStorage secretForStorageResult = client.createCommonStorage(channel, createRequest); @@ -142,13 +139,6 @@ public static void main(String[] args) throws InterruptedException { } } - - public DataProduct createDataProduct(DataProduct dataProduct) { - DataProductCreateRequest request = DataProductCreateRequest.newBuilder().setDataProduct(dataProduct).build(); - DataProductCreateResponse response = blockingApiStub.registerDataProduct(request); - return response.getDataProduct(); - } - public DataReplicaLocation createReplicaLocation(DataReplicaLocation replicaLocation) { DataReplicaCreateRequest request = DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java deleted file mode 100644 index 5c149f7..0000000 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataProductMapper.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.apache.airavata.replicacatalog.catalogapi.mapper; - -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; -import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; -import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; -import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Map to/from - * {@link DataReplicaLocationEntity} - * <-> {@link org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation} - */ -@Component -public class DataProductMapper { - - @Autowired - DataProductRepository dataProductRepository; - - public void mapModelToEntity(DataProduct dataProduct, DataProductEntity dataProductEntity) { - - // For new data product entities - if (dataProductEntity.getProductUri() == null) { - dataProductEntity.setProductUri(dataProduct.getProductUri()); - } - dataProductEntity.setProductName(dataProduct.getProductName()); - dataProductEntity.setProductDescription(dataProduct.getProductDescription()); - dataProductEntity.setDataProductType(dataProduct.getDataProductType()); - dataProductEntity.setParentProductUri(dataProduct.getParentProductUri()); - dataProductEntity.setLastModifiedTime(new Timestamp(System.currentTimeMillis())); - - if (dataProduct.getCreationTime() < 1) { - dataProductEntity.setCreationTime(new Timestamp(System.currentTimeMillis())); - } else { - dataProductEntity.setCreationTime(new Timestamp(dataProduct.getCreationTime())); - } - - //TODO Map metadata - mapMetaData(dataProduct, dataProductEntity); - } - - public void mapEntityToModel(DataProductEntity dataProductEntity, DataProduct.Builder dataProductBuilder) { - - dataProductBuilder - .setProductUri(dataProductEntity.getProductUri()) - .setProductName(dataProductEntity.getProductName()) - .setDataProductType(dataProductEntity.getDataProductType()) - .setParentProductUri(dataProductEntity.getParentProductUri()) - .setCreationTime(dataProductEntity.getCreationTime().getTime()).build(); - - } - - private void mapMetaData(DataProduct dataProduct, DataProductEntity dataProductEntity) { - Map newMetaData = new HashMap<>(); - dataProduct.getMetadataMap().keySet().forEach(m -> { - newMetaData.put(m, dataProduct.getMetadataMap().get(m)); - }); - dataProductEntity.setProductMetadata(newMetaData); - } - - -} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java index 5fa3961..eab93ea 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; -import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; -import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -22,7 +20,7 @@ public class DataReplicaMapper { DataReplicaLocationRepository replicaLocationRepository; @Autowired - DataProductRepository dataProductRepository; + public void mapModelToEntity(DataReplicaLocation model, DataReplicaLocationEntity entity) { @@ -32,8 +30,7 @@ public void mapModelToEntity(DataReplicaLocation model, DataReplicaLocationEntit entity.setReplicaDescription(model.getReplicaDescription()); entity.setCreationTime(new Timestamp(System.currentTimeMillis())); // TODO: handle parent data product not found - DataProductEntity parentDataEntity = dataProductRepository.findByProductUri(model.getDataProductId()); - entity.setDataProduct(parentDataEntity); +// entity.setDataProduct(parentDataEntity); } @@ -45,9 +42,9 @@ public void mapEntityToModel(DataReplicaLocationEntity dataReplicaLocationEntity .setDataProductId( dataReplicaLocationEntity.getProductUri() ) .setCreationTime( dataReplicaLocationEntity.getCreationTime().getTime() ); - if ( dataReplicaLocationEntity.getDataProduct() != null ) - { - dataProductBuilder.setDataProductId( dataReplicaLocationEntity.getDataProduct().getProductUri() ); - } +// if ( dataReplicaLocationEntity.getDataProduct() != null ) +// { +// dataProductBuilder.setDataProductId( dataReplicaLocationEntity.getDataProduct().getProductUri() ); +// } } } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java deleted file mode 100644 index 02e2e19..0000000 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductEntity.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.replicacatalog.catalogapi.model; - - -import jakarta.persistence.*; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductType; - -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.List; -import java.util.Map; - -/** - * The persistent class for the data_product database table. - */ -@Entity -@Table(name = "DATA_PRODUCT") -public class DataProductEntity implements Serializable { - private static final long serialVersionUID = 1L; - - @Id - @Column(name = "PRODUCT_URI") - private String productUri; - - @Column(name = "PRODUCT_NAME") - private String productName; - - @Column(name = "PRODUCT_DESCRIPTION") - private String productDescription; - - @Column(name = "OWNER_NAME") - private String ownerName; - - @Column(name = "PARENT_PRODUCT_URI") - private String parentProductUri; - - @Column(name = "PRODUCT_SIZE") - private int productSize; - - @Column(name = "CREATION_TIME") - private Timestamp creationTime; - - @Column(name = "LAST_MODIFIED_TIME") - private Timestamp lastModifiedTime; - - @Column(name = "PRODUCT_TYPE") - @Enumerated(EnumType.STRING) - private DataProductType dataProductType; - - @ElementCollection(fetch = FetchType.EAGER) - @CollectionTable(name="DATA_PRODUCT_METADATA", joinColumns = @JoinColumn(name="PRODUCT_URI")) - @MapKeyColumn(name = "METADATA_KEY") - @Column(name = "METADATA_VALUE") - private Map productMetadata; - - @OneToMany(targetEntity = DataReplicaLocationEntity.class, cascade = CascadeType.ALL, - mappedBy = "dataProduct", fetch = FetchType.EAGER) - private List replicaLocations; - - public String getProductUri() { - return productUri; - } - - public void setProductUri(String productUri) { - this.productUri = productUri; - } - - public String getProductName() { - return productName; - } - - public void setProductName(String productName) { - this.productName = productName; - } - - public String getProductDescription() { - return productDescription; - } - - public void setProductDescription(String productDescription) { - this.productDescription = productDescription; - } - - public String getOwnerName() { - return ownerName; - } - - public void setOwnerName(String ownerName) { - this.ownerName = ownerName; - } - - public String getParentProductUri() { - return parentProductUri; - } - - public void setParentProductUri(String parentProductUri) { - this.parentProductUri = parentProductUri; - } - - public int getProductSize() { - return productSize; - } - - public void setProductSize(int productSize) { - this.productSize = productSize; - } - - public Timestamp getCreationTime() { - return creationTime; - } - - public void setCreationTime(Timestamp creationTime) { - this.creationTime = creationTime; - } - - public Timestamp getLastModifiedTime() { - return lastModifiedTime; - } - - public void setLastModifiedTime(Timestamp lastModifiedTime) { - this.lastModifiedTime = lastModifiedTime; - } - - public DataProductType getDataProductType() { - return dataProductType; - } - - public void setDataProductType(DataProductType dataProductType) { - this.dataProductType = dataProductType; - } - - public Map getProductMetadata() { return productMetadata; } - - public void setProductMetadata(Map productMetadata) { this.productMetadata = productMetadata; } - - public List getReplicaLocations() { - return replicaLocations; - } - - public void setReplicaLocations(List replicaLocations) { - this.replicaLocations = replicaLocations; - } - -} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java deleted file mode 100644 index cb4da3d..0000000 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataEntity.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.airavata.replicacatalog.catalogapi.model; - -import jakarta.persistence.*; - -import java.io.Serializable; - -/** - * The persistent class for the data_product_metadata database table. - */ -@Entity -@Table(name = "DATA_PRODUCT_METADATA") -@IdClass(DataProductMetadataPK.class) -public class DataProductMetadataEntity implements Serializable { - private static final long serialVersionUID = 1L; - - @Id - @Column(name = "PRODUCT_URI") - private String productUri; - - @Id - @Column(name = "METADATA_KEY") - private String metadataKey; - - @Column(name = "METADATA_VALUE") - private String metadataValue; - - public String getProductUri() { - return productUri; - } - - public void setProductUri(String productUri) { - this.productUri = productUri; - } - - public String getMetadataKey() { - return metadataKey; - } - - public void setMetadataKey(String metadataKey) { - this.metadataKey = metadataKey; - } - - public String getMetadataValue() { - return metadataValue; - } - - public void setMetadataValue(String metadataValue) { - this.metadataValue = metadataValue; - } - -} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java deleted file mode 100644 index 07ac45b..0000000 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataProductMetadataPK.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.airavata.replicacatalog.catalogapi.model; - -import java.io.Serializable; - -/** - * The primary key class for the data_product_metadata database table. - */ -public class DataProductMetadataPK implements Serializable { - //default serial version id, required for serializable classes. - private static final long serialVersionUID = 1L; - - private String productUri; - private String metadataKey; - - public DataProductMetadataPK() { - } - - public String getProductUri() { - return productUri; - } - - public void setProductUri(String productUri) { - this.productUri = productUri; - } - - public String getMetadataKey() { - return metadataKey; - } - - public void setMetadataKey(String metadataKey) { - this.metadataKey = metadataKey; - } - - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof DataProductMetadataPK)) { - return false; - } - DataProductMetadataPK castOther = (DataProductMetadataPK) other; - return - this.productUri.equals(castOther.productUri) - && this.metadataKey.equals(castOther.metadataKey); - } - - public int hashCode() { - final int prime = 31; - int hash = 17; - hash = hash * prime + this.productUri.hashCode(); - hash = hash * prime + this.metadataKey.hashCode(); - - return hash; - } - -} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java index f68bf3f..b6b0fea 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java @@ -62,24 +62,12 @@ public class DataReplicaLocationEntity implements Serializable { @Column(name = "VALID_UNTIL_TIME") private Timestamp validUntilTime; -// @Column(name = "REPLICA_LOCATION_CATEGORY") -// @Enumerated(EnumType.STRING) -// private ReplicaLocationCategory replicaLocationCategory; -// -// @Column(name = "REPLICA_PERSISTENT_TYPE") -// @Enumerated(EnumType.STRING) -// private ReplicaPersistentType replicaPersistentType; - @ElementCollection(fetch = FetchType.EAGER) @CollectionTable(name = "DATA_REPLICA_METADATA", joinColumns = @JoinColumn(name = "REPLICA_ID")) @MapKeyColumn(name = "METADATA_KEY") @Column(name = "METADATA_VALUE") private Map replicaMetadata; - @ManyToOne(targetEntity = DataProductEntity.class) - @JoinColumn(name = "PRODUCT_URI", nullable = false, updatable = false, insertable = false) - private DataProductEntity dataProduct; - public String getReplicaId() { return replicaId; } @@ -152,22 +140,6 @@ public void setValidUntilTime(Timestamp validUntilTime) { this.validUntilTime = validUntilTime; } -// public ReplicaLocationCategory getReplicaLocationCategory() { -// return replicaLocationCategory; -// } -// -// public void setReplicaLocationCategory(ReplicaLocationCategory replicaLocationCategory) { -// this.replicaLocationCategory = replicaLocationCategory; -// } -// -// public ReplicaPersistentType getReplicaPersistentType() { -// return replicaPersistentType; -// } -// -// public void setReplicaPersistentType(ReplicaPersistentType replicaPersistentType) { -// this.replicaPersistentType = replicaPersistentType; -// } - public Map getReplicaMetadata() { return replicaMetadata; } @@ -176,11 +148,4 @@ public void setReplicaMetadata(Map replicaMetadata) { this.replicaMetadata = replicaMetadata; } - public DataProductEntity getDataProduct() { - return dataProduct; - } - - public void setDataProduct(DataProductEntity dataProduct) { - this.dataProduct = dataProduct; - } } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java deleted file mode 100644 index af99525..0000000 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataProductRepository.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.airavata.replicacatalog.catalogapi.repository; - -import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.transaction.annotation.Transactional; - -@Transactional(readOnly = true) -public interface DataProductRepository extends JpaRepository { - - DataProductEntity findByProductUri(String productUri); - - @Transactional - void deleteByProductUri(String productUri); - -} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java index bb27bf7..ab4638e 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java @@ -1,20 +1,9 @@ package org.apache.airavata.replicacatalog.catalogapi.service; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; -import java.util.List; - public interface IReplicaCatalogService { - DataProduct createDataProduct(DataProduct dataProduct); - - DataProduct updateDataProduct(DataProduct dataProduct); - - DataProduct getDataProduct(String productUri); - - void deleteDataProduct(String productUri); - DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica); DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica); diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java index 9e1d7d6..900cee8 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java @@ -6,15 +6,6 @@ import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaDeleteResponse; import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetRequest; import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetResponse; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateRequest; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateResponse; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductDeleteRequest; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductDeleteResponse; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductGetRequest; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductGetResponse; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductUpdateRequest; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProductUpdateResponse; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaDeleteRequest; @@ -24,7 +15,6 @@ import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateRequest; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateResponse; -import org.apache.airavata.replicacatalog.catalogapi.mapper.DataProductMapper; import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; @@ -38,51 +28,12 @@ public class ReplicaCatalogAPIService extends ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceImplBase { private static final Logger logger = LoggerFactory.getLogger(ReplicaCatalogAPIService.class); - - @Autowired - DataProductMapper dataProductMapper = new DataProductMapper(); - @Autowired DataReplicaMapper replicaMapper = new DataReplicaMapper(); @Autowired IReplicaCatalogService catalogService; - - @Override - public void registerDataProduct(DataProductCreateRequest request, StreamObserver responseObserver) { - DataProduct result = catalogService.createDataProduct(request.getDataProduct()); - DataProductCreateResponse.Builder responseBuilder = DataProductCreateResponse.newBuilder(); - responseBuilder.setDataProduct(result); - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void updateDataProduct(DataProductUpdateRequest request, StreamObserver responseObserver) { - DataProduct result = catalogService.updateDataProduct(request.getDataProduct()); - DataProductUpdateResponse.Builder responseBuilder = DataProductUpdateResponse.newBuilder(); - responseBuilder.setDataProduct(result); - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void getDataProduct(DataProductGetRequest request, StreamObserver responseObserver) { - DataProduct result = catalogService.getDataProduct(request.getDataProductUri()); - DataProductGetResponse.Builder responseBuilder = DataProductGetResponse.newBuilder(); - responseBuilder.setDataProduct(result); - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void removeDataProduct(DataProductDeleteRequest request, StreamObserver responseObserver) { - catalogService.deleteDataProduct(request.getDataProductUri()); - responseObserver.onNext(DataProductDeleteResponse.newBuilder().build()); - responseObserver.onCompleted(); - } - @Override public void registerReplicaLocation(DataReplicaCreateRequest request, StreamObserver responseObserver) { diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java index 5d39094..fd9767d 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java @@ -1,14 +1,11 @@ package org.apache.airavata.replicacatalog.catalogapi.service.impl; -import jakarta.persistence.EntityNotFoundException; +import java.util.UUID; + import jakarta.transaction.Transactional; -import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; -import org.apache.airavata.replicacatalog.catalogapi.mapper.DataProductMapper; import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; -import org.apache.airavata.replicacatalog.catalogapi.model.DataProductEntity; import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; -import org.apache.airavata.replicacatalog.catalogapi.repository.DataProductRepository; import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; import org.apache.airavata.replicacatalog.catalogapi.service.IReplicaCatalogService; import org.slf4j.Logger; @@ -16,70 +13,19 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.UUID; - @Service @Transactional public class ReplicaCatalogServiceImp implements IReplicaCatalogService { private final static Logger logger = LoggerFactory.getLogger(ReplicaCatalogServiceImp.class); - @Autowired - DataProductRepository dataProductRepository; - @Autowired DataReplicaLocationRepository dataReplicaLocationRepository; - @Autowired - DataProductMapper dataProductMapper = new DataProductMapper(); - @Autowired DataReplicaMapper replicaMapper = new DataReplicaMapper(); - @Override - public DataProduct createDataProduct(DataProduct dataProduct) { - DataProductEntity dataProductEntity = new DataProductEntity(); - - if (dataProduct != null && dataProduct.getProductUri() != null) { - dataProductEntity.setProductUri(dataProduct.getProductUri()); - } - dataProductMapper.mapModelToEntity(dataProduct, dataProductEntity); - - if (dataProductEntity.getProductUri() == null) { - dataProductEntity.setProductUri(UUID.randomUUID().toString()); - } - DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); - return toDataProduct(savedDataProductEntity); - } - - @Override - public DataProduct updateDataProduct(DataProduct dataProduct) { - - DataProductEntity dataProductEntity = dataProductRepository.findByProductUri(dataProduct.getProductUri()); - if (dataProductEntity == null) { - logger.warn("Data product not exists"); - throw new EntityNotFoundException("Could not find a data product with the ID: " + dataProduct.getProductUri()); - } - dataProductMapper.mapModelToEntity(dataProduct, dataProductEntity); - DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity); - - return toDataProduct(savedDataProductEntity); - } - - @Override - public DataProduct getDataProduct(String productUri) { - DataProductEntity dataProductEntity = dataProductRepository.findByProductUri(productUri); - return toDataProduct(dataProductEntity); - } - - @Override - public void deleteDataProduct(String productUri) { - dataProductRepository.deleteByProductUri(productUri); - } - - - @Override public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation) { @@ -131,10 +77,5 @@ private DataReplicaLocation toDataReplicaLocation(DataReplicaLocationEntity save return builder.build(); } - private DataProduct toDataProduct(DataProductEntity savedDataProductEntity) { - DataProduct.Builder builder = DataProduct.newBuilder(); - dataProductMapper.mapEntityToModel(savedDataProductEntity, builder); - return builder.build(); - } } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java index dbe23c0..c01c185 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java @@ -33,8 +33,6 @@ public class StorageSecretEntity { @GenericGenerator(name = "uuid", strategy = "uuid2") private String id; - // Identifier can add here - @Column(name = "STORAGE_ID") private String storageId; @@ -44,6 +42,9 @@ public class StorageSecretEntity { @Column(name = "STORAGE_TYPE") private String type; + @Column(name = "USER_IDENTIFIER") + private String userIdentifier; + public String getId() { return id; } @@ -75,4 +76,12 @@ public String getType() { public void setType(String type) { this.type = type; } + + public String getUserIdentifier() { + return userIdentifier; + } + + public void setUserIdentifier(String userIdentifier) { + this.userIdentifier = userIdentifier; + } } diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto index ef3bf79..79c264f 100644 --- a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto @@ -150,47 +150,3 @@ message AllDataReplicaDeleteResponse{ int32 replica_location_count = 1; } - -// Entity which used to model a DataProduct entry -message DataProduct{ - string product_uri = 1; // Unique id - string parent_product_uri = 2; // - string product_name = 3; // Product user friendly name - string product_description = 4; // Product description - int64 creation_time = 5; // Product creation date and time - int64 last_modified_time = 6; // Last modified time of replica - int64 valid_until_time = 7; // Validity time of the replica - DataProductType data_product_type = 8; // Product type of the DataProduct - map metadata = 9; // Product metadata information - repeated DataReplicaLocation replica_locations = 10; //Link to replicas - -} - -message DataProductCreateRequest { - DataProduct data_product = 1; -} -message DataProductCreateResponse{ - DataProduct data_product = 1; -} - -message DataProductUpdateRequest { - DataProduct data_product = 1; -} -message DataProductUpdateResponse{ - DataProduct data_product = 1; -} - -message DataProductGetRequest { - string data_product_uri = 1; -} -message DataProductGetResponse { - DataProduct data_product = 1; -} - -message DataProductDeleteRequest { - string data_product_uri = 1; -} -message DataProductDeleteResponse { -} - - diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto index d607fd4..a95dc92 100644 --- a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto @@ -54,26 +54,4 @@ service ReplicaCatalogAPIService { */ rpc removeAllReplicaLocation(AllDataReplicaDeleteRequest) returns (AllDataReplicaDeleteResponse){} - - /* - * Register a new data product to the Replica Catalog - */ - rpc registerDataProduct(DataProductCreateRequest) returns (DataProductCreateResponse){} - - /* - * Update existing Data Product details in the ReplicaCatalog - */ - rpc updateDataProduct(DataProductUpdateRequest) returns (DataProductUpdateResponse){} - - /* - * Retrieving existing Data Product details in the ReplicaCatalog - */ - rpc getDataProduct(DataProductGetRequest) returns (DataProductGetResponse){} - - /* - * Removing a Data Product in the ReplicaCatalog - */ - rpc removeDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){} - - } diff --git a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto index 1df0c03..5da9b38 100644 --- a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto @@ -120,8 +120,9 @@ message SecretForStorage { string storage_id = 1; string secret_id = 2; string replica_id = 3; - StorageType storage_type = 4; - Error error = 5; + string user_identity = 4; + StorageType storage_type = 5; + Error error = 6; } message SecretForStorageCreateRequest { From 83563299368e4fe0277e5589c51ec4dac337e515 Mon Sep 17 00:00:00 2001 From: jayancv Date: Sun, 16 Apr 2023 23:00:43 +0530 Subject: [PATCH 7/8] Added GCS storage and update the test case --- pom.xml | 1 + replica-catalog-api/client/pom.xml | 6 + .../airavata/ReplicaCatalogAPIClient.java | 78 ++-- .../airavata/ReplicaCatalogAPIClientTest.java | 361 +++++++++++------- replica-catalog-api/server/pom.xml | 6 + .../catalogapi/mapper/DataReplicaMapper.java | 7 +- .../DataReplicaLocationRepository.java | 5 + .../service/IReplicaCatalogService.java | 13 +- .../service/ReplicaCatalogAPIService.java | 66 +++- .../impl/ReplicaCatalogServiceImp.java | 35 +- .../exception/EntityNotFoundException.java | 25 ++ .../exception/InvalidDataException.java | 9 + .../exception/StorageNotSupportException.java | 9 + .../mapper/ResourceStorageMapper.java | 34 +- .../model/storage/FTPStorageEntity.java | 77 ++++ .../model/storage/GCSStorageEntity.java | 73 ++++ .../model/storage/LocalStorageEntity.java | 66 ++++ .../model/{ => storage}/S3StorageEntity.java | 2 +- .../storage/FTPStorageRepository.java | 31 ++ .../storage/GCSStorageRepository.java | 31 ++ .../storage/LocalStorageRepository.java | 31 ++ .../{ => storage}/S3StorageRepository.java | 4 +- .../resource/service/SQLResourceService.java | 107 +++++- .../sceret/model/FTPSecretEntity.java | 67 ++++ .../sceret/model/GCSSecretEntity.java | 87 +++++ .../repository/FTPSecretRepository.java | 29 ++ .../repository/GCSSecretRepository.java | 29 ++ .../sceret/service/SQLSecretService.java | 21 +- .../proto/catalogapi/ReplicaCatalogAPI.proto | 42 +- .../src/main/proto/resource/FTPStorage.proto | 67 ++++ .../src/main/proto/resource/GCSStorage.proto | 66 ++++ .../main/proto/resource/LocalStorage.proto | 66 ++++ .../proto/resource/{s3 => }/S3Storage.proto | 0 .../resource/{common => }/StorageCommon.proto | 9 +- .../src/main/proto/secret/FTPCredential.proto | 55 +++ .../src/main/proto/secret/GCSCredential.proto | 59 +++ .../proto/secret/{s3 => }/S3Credential.proto | 0 .../secret/{common => }/SecretCommon.proto | 6 +- 38 files changed, 1472 insertions(+), 208 deletions(-) create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java rename replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/{ => storage}/S3StorageEntity.java (97%) create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java rename replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/{ => storage}/S3StorageRepository.java (88%) create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java create mode 100644 replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java create mode 100644 replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto rename replica-catalog-api/stubs/src/main/proto/resource/{s3 => }/S3Storage.proto (100%) rename replica-catalog-api/stubs/src/main/proto/resource/{common => }/StorageCommon.proto (92%) create mode 100644 replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto create mode 100644 replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto rename replica-catalog-api/stubs/src/main/proto/secret/{s3 => }/S3Credential.proto (100%) rename replica-catalog-api/stubs/src/main/proto/secret/{common => }/SecretCommon.proto (92%) diff --git a/pom.xml b/pom.xml index d217819..a05e80b 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ under the License. 3.1.1 2.14.1 5.5.1 + 5.8.1 diff --git a/replica-catalog-api/client/pom.xml b/replica-catalog-api/client/pom.xml index bd56f10..9d746a1 100644 --- a/replica-catalog-api/client/pom.xml +++ b/replica-catalog-api/client/pom.xml @@ -23,6 +23,12 @@ slf4j-api ${slf4j.version} + + org.junit.jupiter + junit-jupiter-engine + ${jupiter.version} + test + diff --git a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java index e9c877f..8e70d78 100644 --- a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java +++ b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java @@ -15,7 +15,6 @@ import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; -import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommon; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; @@ -31,22 +30,30 @@ public class ReplicaCatalogAPIClient { private static final Logger logger = LoggerFactory.getLogger(ReplicaCatalogAPIClient.class); - private final ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub blockingApiStub; + private final ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub replicaServiceAPI; + private StorageCommonServiceGrpc.StorageCommonServiceBlockingStub storageServiceAPI = null; + private SecretCommonServiceGrpc.SecretCommonServiceBlockingStub secretServiceAPI = null; public ReplicaCatalogAPIClient(Channel channel) { - blockingApiStub = ReplicaCatalogAPIServiceGrpc.newBlockingStub(channel); + replicaServiceAPI = ReplicaCatalogAPIServiceGrpc.newBlockingStub(channel); } - public ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub getBlockingApiStub() { - return blockingApiStub; + public ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub getReplicaServiceAPI() { + return replicaServiceAPI; } public StorageCommonServiceGrpc.StorageCommonServiceBlockingStub getBlockingStorageStub(Channel channel) { - return StorageCommonServiceGrpc.newBlockingStub(channel); + if (storageServiceAPI == null) { + storageServiceAPI = StorageCommonServiceGrpc.newBlockingStub(channel); + } + return storageServiceAPI; } public SecretCommonServiceGrpc.SecretCommonServiceBlockingStub getBlockingSecretStub(Channel channel) { - return SecretCommonServiceGrpc.newBlockingStub(channel); + if (secretServiceAPI == null) { + secretServiceAPI = SecretCommonServiceGrpc.newBlockingStub(channel); + } + return secretServiceAPI; } public static void main(String[] args) throws InterruptedException { @@ -72,7 +79,7 @@ public static void main(String[] args) throws InterruptedException { DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder() .setReplicaName("ASTRO S3") .setReplicaDescription("S3 replica") - .setDataProductId(testUri) + .setDataProductUri(testUri) .setCreationTime(System.currentTimeMillis()).build(); DataReplicaLocation replicaResult = client.createReplicaLocation(replicaLocation); @@ -108,18 +115,23 @@ public static void main(String[] args) throws InterruptedException { System.out.println( MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}]", - (Object[]) new String[]{replicaResult.getDataReplicaId(),secretForStorageResult.getStorageId(),secretForStorageResult.getSecretId()})); + (Object[]) new String[]{replicaResult.getDataReplicaId(), secretForStorageResult.getStorageId(), secretForStorageResult.getSecretId()})); DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(replicaResult.getDataReplicaId()).build(); - DataReplicaGetResponse replicaResponse = client.blockingApiStub.getReplicaLocation(replicaGetRequest); + DataReplicaGetResponse replicaResponse = client.replicaServiceAPI.getReplicaLocation(replicaGetRequest); if (replicaResponse != null && replicaResponse.getDataReplica() != null) { - SecretForStorage secretForStorage1= client.getStorageSecretIds(channel,replicaResponse.getDataReplica().getDataReplicaId()); - if(secretForStorage1!= null){ - System.out.println( - MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}]", - (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(),secretForStorage1.getStorageId(),secretForStorage1.getSecretId()})); - - } + SecretForStorage secretForStorage1 = null; + try { + secretForStorage1 = client.getStorageSecretIds(channel, replicaResponse.getDataReplica().getDataReplicaId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + if (secretForStorage1 != null) { + System.out.println( + MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}]", + (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(), secretForStorage1.getStorageId(), secretForStorage1.getSecretId()})); + + } } /* @@ -140,9 +152,8 @@ public static void main(String[] args) throws InterruptedException { } public DataReplicaLocation createReplicaLocation(DataReplicaLocation replicaLocation) { - DataReplicaCreateRequest request = - DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); - DataReplicaCreateResponse response = blockingApiStub.registerReplicaLocation(request); + DataReplicaCreateRequest request = DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); + DataReplicaCreateResponse response = replicaServiceAPI.registerReplicaLocation(request); return response.getDataReplica(); } @@ -159,12 +170,19 @@ public GenericResource createStorage(Channel channel, GenericResource resource) case StorageWrapper.S3STORAGE_FIELD_NUMBER: storageType = StorageType.S3; break; - + case StorageWrapper.LOCALSTORAGE_FIELD_NUMBER: + storageType = StorageType.LOCAL; + break; + case StorageWrapper.GCSSTORAGE_FIELD_NUMBER: + storageType = StorageType.GCS; + break; + case StorageWrapper.FTPSTORAGE_FIELD_NUMBER: + storageType = StorageType.FTP; + break; default: break; } - GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder(). - setResource(resource).setStorageType(storageType).build(); + GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder().setResource(resource).build(); GenericResource response = getBlockingStorageStub(channel).createGenericResource(request); return response; } @@ -174,11 +192,19 @@ public StorageSecret createSecret(Channel channel, SecretCreateRequest createReq return response; } - public SecretForStorage getStorageSecretIds(Channel channel, String replicaId){ + public SecretForStorage getStorageSecretIds(Channel channel, String replicaId) throws Exception { GenericResourceGetRequest resourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaId).build(); GenericResource resource = getBlockingStorageStub(channel).getGenericResource(resourceGetRequest); - SecretForStorageGetRequest request = SecretForStorageGetRequest.newBuilder().setStorageId(resource.getStorage().getS3Storage().getStorageId()).build(); - SecretForStorage secretForStorage = getBlockingStorageStub(channel).getSecretForStorage(request); + String storageId = null; + if (resource.getStorage().hasS3Storage()) { + storageId = resource.getStorage().getS3Storage().getStorageId(); + } else if (resource.getStorage().hasGcsStorage()) { + storageId = resource.getStorage().getGcsStorage().getStorageId(); + } else { + throw new Exception("Not Supported storage type"); + } + SecretForStorageGetRequest request = SecretForStorageGetRequest.newBuilder().setStorageId(storageId).build(); + SecretForStorage secretForStorage = getBlockingStorageStub(channel).getSecretForStorage(request); return secretForStorage; } diff --git a/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java index 5b2a5b9..24f6cec 100644 --- a/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java +++ b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java @@ -1,129 +1,232 @@ -//package org.apache.airavata; -// -// -//import java.text.MessageFormat; -//import java.util.concurrent.TimeUnit; -// -//import io.grpc.Channel; -//import io.grpc.ManagedChannel; -//import io.grpc.ManagedChannelBuilder; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataProduct; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateRequest; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataProductCreateResponse; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataProductType; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; -//import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; -//import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; -//import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; -//import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; -//import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; -//import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommon; -//import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; -//import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; -//import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; -//import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; -// -//class ReplicaCatalogAPIClientTest -//{ -// public void testCase1() -// { -// String target = "localhost:6565"; -// -// /* -// --- Sample scenario --- -// Airavata MFT copied a 1000GB astrological data Folder which contains 100 data files to Amazon S3 bucket -// User can access that S3 bucket data folder via Replica catalog details -// -// 1) Register Data Product -// 2) Register Replica Location -// 3) Register Location storage details -// 4) Register location storage credentials -// -// */ -// ManagedChannel channel = ManagedChannelBuilder.forTarget( target ).usePlaintext().build(); -// try -// { -// ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient( channel ); -// -// DataProduct dataProduct = DataProduct.newBuilder().setDataProductType( DataProductType.FILE ) -// .setProductName( "ASTRO" ).setProductUri( "613" ).setProductDescription( "Astro Data" ).build(); -// DataProduct productResult = client.createDataProduct( dataProduct ); -// -// DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder().setReplicaName( "ASTRO S3" ) -// .setReplicaDescription( "S3 replica" ).setDataProductId( productResult.getProductUri() ).build(); -// -// DataReplicaLocation replicaResult = client.createReplicaLocation( replicaLocation ); -// -// GenericResource resource = client.createStorage( channel, null ); -// -// StorageSecret secret = client.createSecret( channel, null ); -// -// client.createCommonStorage( channel, null ); -// -// -// DataReplicaLocation result = client.createReplicaLocation( replicaLocation ); -// System.out.println( -// MessageFormat.format( "Created data product with id [{0}]", result.getDataReplicaId() ) ); -// -// } -// finally -// { -// channel.shutdownNow().awaitTermination( 5, TimeUnit.SECONDS ); -// } -// } -// -// public DataReplicaLocation createReplicaLocation( DataReplicaLocation replicaLocation ) -// { -// DataReplicaCreateRequest request = -// DataReplicaCreateRequest.newBuilder().setDataReplica( replicaLocation ).build(); -// DataReplicaCreateResponse response = blockingApiStub.registerReplicaLocation( request ); -// return response.getDataReplica(); -// } -// -// public DataProduct createDataProduct( DataProduct dataProduct ) -// { -// DataProductCreateRequest request = DataProductCreateRequest.newBuilder().setDataProduct( dataProduct ).build(); -// DataProductCreateResponse response = blockingApiStub.registerDataProduct( request ); -// return response.getDataProduct(); -// } -// -// public SecretForStorage createCommonStorage( Channel channel, StorageCommon storageCommon ) -// { -// SecretForStorage request = SecretForStorage.newBuilder() -// .setStorageId( "-1" ).setSecretId( "-1" ).setStorageType( StorageType.S3 ).build(); -// SecretForStorage response = getBlockingStorageStub( channel ).registerSecretForStorage( request ); -// return response; -// } -// -// public GenericResource createStorage( Channel channel, GenericResource resource ) -// { -// S3Storage storage = S3Storage.newBuilder().setStorageId( "-1" ).setBucketName( "bucket" ).setName( "Name" ) -// .setEndpoint( "end" ).setRegion( "us-west" ).build(); -// -// GenericResource resource1 = GenericResource.newBuilder().setResourceId( "-1" ).setS3Storage( storage ) -// .setFile( FileResource.newBuilder().setResourcePath( "path" ).build() ).build(); -// -// GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder().setStorageId( "-1" ). -// setResource( resource1 ).setStorageType( GenericResourceCreateRequest.StorageType.S3 ).build(); -// GenericResource response = getBlockingResourceStub( channel ).createGenericResource( request ); -// return response; -// } -// -// public StorageSecret createSecret( Channel channel, GenericResource resource ) -// { -// S3Secret secret = S3Secret.newBuilder().setSecretId( "-1" ).setAccessKey( "access" ).setSecretKey( "secKey" ) -// .setSessionToken( "token" ).build(); -// StorageSecret -// storageSecret = StorageSecret.newBuilder().setSecretId( "-1" ).setS3Secret( secret ) -// .setStorageType( org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3 ).build(); -// -// -// StorageSecret request = StorageSecret.newBuilder().setSecretId( "-1" ).setS3Secret( secret ). -// setStorageType( org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3 ).build(); -// StorageSecret response = getBlockingSecretStub( channel ).registerSecret( storageSecret ); -// return storageSecret; -// } -// -// -//} \ No newline at end of file +package org.apache.airavata; + + +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; +import org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage; +import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretWrapper; +import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret; +import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; +import org.junit.jupiter.api.Test; + +class ReplicaCatalogAPIClientTest { + + @Test + public void testCase1() throws InterruptedException { + String target = "localhost:6565"; + + /* + --- Sample scenario 1--- + Airavata MFT copied a 100GB astrological data file to Amazon S3 bucket and another copy to GCS location + User can access that S3 bucket data file and Google cloud storage file via Replica catalog details + + 1) Register Data Product in data catalog + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + ManagedChannel channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build(); + try { + ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient(channel); + + String testUri = "TestUri"; + + DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder() + .setReplicaName("ASTRO S3") + .setReplicaDescription("S3 replica") + .setDataProductUri(testUri) + .setStorageType(org.apache.airavata.replicacatalog.catalog.stubs.StorageType.S3) + .setCreationTime(System.currentTimeMillis()).build(); + DataReplicaLocation replicaResult = createReplicaLocation(client, replicaLocation); + + S3Storage storage = S3Storage.newBuilder() + .setName("ASTRO S3") + .setBucketName("arn:aws:s3:::mftjayan") + .setRegion("us-east-1") + .setEndpoint("https://s3.us-east-1.amazonaws.com").build(); + + GenericResource resource = GenericResource.newBuilder() + .setReplicaId(replicaResult.getDataReplicaId()) + .setStorage(StorageWrapper.newBuilder().setS3Storage(storage).build()) + .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); + GenericResource resourceResult = createStorage(client, channel, resource); + + S3Secret secret = S3Secret.newBuilder() + .setAccessKey("access") + .setSecretKey("secKey") + .setSessionToken("token").build(); + StorageSecret storageSecret = StorageSecret.newBuilder() + .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) + .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3).build(); + SecretCreateRequest secretCreateRequest = SecretCreateRequest.newBuilder().setSecret(storageSecret).build(); + StorageSecret secretResult = createSecret(client, channel, secretCreateRequest); + + SecretForStorage secretForStorage = SecretForStorage.newBuilder() + .setStorageId(resourceResult.getStorage().getS3Storage().getStorageId()) + .setSecretId(secretResult.getSecret().getS3Secret().getSecretId()).setStorageType(StorageType.S3) + .setUserIdentity("userID#12345").build(); + SecretForStorageCreateRequest createRequest = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage).build(); + SecretForStorage secretForStorageResult = createCommonStorage(client, channel, createRequest); + + // GCS replica + DataReplicaLocation replicaLocation2 = DataReplicaLocation.newBuilder() + .setReplicaName("ASTRO GCS") + .setReplicaDescription("GCS replica") + .setDataProductUri(testUri) + .setStorageType(org.apache.airavata.replicacatalog.catalog.stubs.StorageType.GCS) + .setCreationTime(System.currentTimeMillis()).build(); + DataReplicaLocation replicaResult2 = createReplicaLocation(client, replicaLocation2); + + GCSStorage gcsStorage = GCSStorage.newBuilder() + .setName("ASTRO GCS") + .setBucketName("mftjayan").build(); + + GenericResource resource2 = GenericResource.newBuilder() + .setReplicaId(replicaResult2.getDataReplicaId()) + .setStorage(StorageWrapper.newBuilder().setGcsStorage(gcsStorage).build()) + .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); + GenericResource resourceResult2 = createStorage(client, channel, resource2); + + GCSSecret gcsSecret = GCSSecret.newBuilder() + .setClientEmail("accessEmail@gmail.com") + .setPrivateKey("secKey") + .setProjectId("1258").build(); + StorageSecret storageSecret2 = StorageSecret.newBuilder() + .setSecret(SecretWrapper.newBuilder().setGcsSecret(gcsSecret).build()) + .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.GCS).build(); + SecretCreateRequest secretCreateRequest2 = SecretCreateRequest.newBuilder().setSecret(storageSecret2).build(); + StorageSecret secretResult2 = createSecret(client, channel, secretCreateRequest2); + + SecretForStorage secretForStorage2 = SecretForStorage.newBuilder() + .setStorageId(resourceResult2.getStorage().getGcsStorage().getStorageId()) + .setSecretId(secretResult2.getSecret().getGcsSecret().getSecretId()) + .setStorageType(StorageType.GCS) + .setUserIdentity("userID#12345").build(); + SecretForStorageCreateRequest createRequest2 = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage2).build(); + SecretForStorage secretForStorageResult2 = createCommonStorage(client, channel, createRequest2); + + + System.out.println( + MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}] in S3", + (Object[]) new String[]{replicaResult.getDataReplicaId(), secretForStorageResult.getStorageId(), secretForStorageResult.getSecretId()})); + + System.out.println( + MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}] in GCS", + (Object[]) new String[]{replicaResult2.getDataReplicaId(), secretForStorageResult2.getStorageId(), secretForStorageResult2.getSecretId()})); + + + AllDataReplicaGetRequest replicasGetRequest = AllDataReplicaGetRequest.newBuilder().setDataProductUri(testUri).build(); + AllDataReplicaGetResponse replicas = client.getReplicaServiceAPI().getAllReplicaLocation(replicasGetRequest); + System.out.println(MessageFormat.format("Load replicas with product url [{0}] ", (Object[]) new String[]{testUri})); + + replicas.getReplicaListList().forEach(r -> { + + DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(r.getDataReplicaId()).build(); + DataReplicaGetResponse replicaResponse = client.getReplicaServiceAPI().getReplicaLocation(replicaGetRequest); + if (replicaResponse != null) { + SecretForStorage secretForStorage1 = null; + try { + secretForStorage1 = getStorageSecretIds(client, channel, replicaResponse.getDataReplica().getDataReplicaId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + if (secretForStorage1 != null) { + System.out.println( + MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}] Storage Type [{3}]", + (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(), secretForStorage1.getStorageId(), secretForStorage1.getSecretId(), + secretForStorage1.getStorageType().name()})); + + } + } + } + + ); + + + /* + --- Sample scenario 2--- + Airavata MFT copied a 1000GB astrological data Folder which contains 100 data files to Amazon S3 bucket + User can access that S3 bucket data folder via Replica catalog details + + 1) Register Data Product + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + }catch (Exception e){ + + } finally { + channel.shutdownNow().awaitTermination(1800, TimeUnit.SECONDS); + } + } + + public DataReplicaLocation createReplicaLocation(ReplicaCatalogAPIClient client, DataReplicaLocation replicaLocation) throws Exception { + DataReplicaCreateRequest request = DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); + DataReplicaCreateResponse response = client.getReplicaServiceAPI().registerReplicaLocation(request); + if(response.hasError() && !response.getError().getMessage().isBlank()){ + throw new Exception(response.getError().getMessage()); + } + return response.getDataReplica(); + } + + + public SecretForStorage createCommonStorage(ReplicaCatalogAPIClient client, Channel channel, SecretForStorageCreateRequest createRequest) { + + SecretForStorage response = client.getBlockingStorageStub(channel).registerSecretForStorage(createRequest); + return response; + } + + public GenericResource createStorage(ReplicaCatalogAPIClient client, Channel channel, GenericResource resource) { + GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder().setResource(resource).build(); + GenericResource response = client.getBlockingStorageStub(channel).createGenericResource(request); + return response; + } + + public StorageSecret createSecret(ReplicaCatalogAPIClient client, Channel channel, SecretCreateRequest createRequest) { + StorageSecret response = client.getBlockingSecretStub(channel).registerSecret(createRequest); + return response; + } + + public SecretForStorage getStorageSecretIds(ReplicaCatalogAPIClient client, Channel channel, String replicaId) throws Exception { + GenericResourceGetRequest resourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaId).build(); + GenericResource resource = client.getBlockingStorageStub(channel).getGenericResource(resourceGetRequest); + String storageId = null; + if (resource.getStorage().hasS3Storage()) { + storageId = resource.getStorage().getS3Storage().getStorageId(); + } else if (resource.getStorage().hasGcsStorage()) { + storageId = resource.getStorage().getGcsStorage().getStorageId(); + } else { + throw new Exception("Not Supported storage type"); + } + SecretForStorageGetRequest request = SecretForStorageGetRequest.newBuilder().setStorageId(storageId).build(); + SecretForStorage secretForStorage = client.getBlockingStorageStub(channel).getSecretForStorage(request); + return secretForStorage; + } + + +} \ No newline at end of file diff --git a/replica-catalog-api/server/pom.xml b/replica-catalog-api/server/pom.xml index 7f7120a..65c67c0 100644 --- a/replica-catalog-api/server/pom.xml +++ b/replica-catalog-api/server/pom.xml @@ -68,6 +68,12 @@ dozer ${dozer.version} + + org.junit.jupiter + junit-jupiter-engine + ${jupiter.version} + test + diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java index eab93ea..94b1012 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java @@ -19,14 +19,11 @@ public class DataReplicaMapper { @Autowired DataReplicaLocationRepository replicaLocationRepository; - @Autowired - - public void mapModelToEntity(DataReplicaLocation model, DataReplicaLocationEntity entity) { entity.setReplicaName(model.getReplicaName()); entity.setReplicaId(model.getDataReplicaId()); - entity.setProductUri(model.getDataProductId()); + entity.setProductUri(model.getDataProductUri()); entity.setReplicaDescription(model.getReplicaDescription()); entity.setCreationTime(new Timestamp(System.currentTimeMillis())); // TODO: handle parent data product not found @@ -39,7 +36,7 @@ public void mapEntityToModel(DataReplicaLocationEntity dataReplicaLocationEntity dataProductBuilder .setDataReplicaId( dataReplicaLocationEntity.getReplicaId() ) .setReplicaName( dataReplicaLocationEntity.getReplicaName() ) - .setDataProductId( dataReplicaLocationEntity.getProductUri() ) + .setDataProductUri( dataReplicaLocationEntity.getProductUri() ) .setCreationTime( dataReplicaLocationEntity.getCreationTime().getTime() ); // if ( dataReplicaLocationEntity.getDataProduct() != null ) diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java index 9fa5cb7..29ab52a 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java @@ -20,6 +20,9 @@ */ package org.apache.airavata.replicacatalog.catalogapi.repository; +import java.util.List; +import java.util.Optional; + import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.transaction.annotation.Transactional; @@ -29,6 +32,8 @@ public interface DataReplicaLocationRepository extends JpaRepository> findByProductUri(String replicaId); + @Transactional void deleteByReplicaId(String replicaId); diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java index ab4638e..93a5bdc 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java @@ -1,15 +1,20 @@ package org.apache.airavata.replicacatalog.catalogapi.service; +import java.util.List; + import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.ReplicaGroupEntry; public interface IReplicaCatalogService { - DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica); + DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica) throws Exception; + + DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica) throws Exception; - DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica); + DataReplicaLocation getDataReplica(String replicaId) throws Exception; - DataReplicaLocation getDataReplica(String replicaId); + void deleteDataReplica(String replicaId) throws Exception; - void deleteDataReplica(String replicaId); + List getDataReplicas(String productUri) throws Exception; } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java index 900cee8..b9ae1ee 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java @@ -1,5 +1,8 @@ package org.apache.airavata.replicacatalog.catalogapi.service; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + import io.grpc.stub.StreamObserver; import org.apache.airavata.replicacatalog.catalog.service.ReplicaCatalogAPIServiceGrpc; import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaDeleteRequest; @@ -15,7 +18,11 @@ import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateRequest; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.Error; +import org.apache.airavata.replicacatalog.catalog.stubs.ErrorCode; +import org.apache.airavata.replicacatalog.catalog.stubs.ReplicaGroupEntry; import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; +import org.apache.airavata.replicacatalog.exception.InvalidDataException; import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,13 +45,31 @@ public class ReplicaCatalogAPIService extends ReplicaCatalogAPIServiceGrpc.Repli public void registerReplicaLocation(DataReplicaCreateRequest request, StreamObserver responseObserver) { logger.info("Creating Replica for a data product {}", request.getDataReplica()); - if (request.getDataReplica() == null) { + Error.Builder errorBuilder = Error.newBuilder(); + DataReplicaLocation result = DataReplicaLocation.newBuilder().build(); + if (!request.hasDataReplica()) { logger.debug("No Data Replica Location"); + errorBuilder.setMessage("Replica details not available in the DataReplicaCreateRequest") + .setCode(ErrorCode.INVALID_DATA).build(); + } else if (request.getDataReplica().getDataProductUri().isBlank()){ + logger.debug("No data product uri for the replica location"); + errorBuilder.setMessage("Data product uri not available in the replica location in the DataReplicaCreateRequest") + .setCode(ErrorCode.INVALID_DATA).build(); + }else if (request.getDataReplica().getStorageType().getNumber()==0){ + logger.debug("No storage type for the replica location"); + errorBuilder.setMessage("Storage type not available in the replica location in the DataReplicaCreateRequest") + .setCode(ErrorCode.INVALID_DATA).build(); + } + else { + try { + result = catalogService.createDataReplica(request.getDataReplica()); + } catch (Exception e) { + logger.error("Error in creating Replica location",e); + } } - DataReplicaLocation result = catalogService.createDataReplica(request.getDataReplica()); - DataReplicaCreateResponse.Builder responseBuilder = DataReplicaCreateResponse.newBuilder(); responseBuilder.setDataReplica(result); + responseBuilder.setError(errorBuilder.build()); responseObserver.onNext(responseBuilder.build()); responseObserver.onCompleted(); } @@ -52,7 +77,12 @@ public void registerReplicaLocation(DataReplicaCreateRequest request, StreamObse @Override public void updateReplicaLocation(DataReplicaUpdateRequest request, StreamObserver responseObserver) { - DataReplicaLocation result = catalogService.updateDataReplica(request.getDataReplica()); + DataReplicaLocation result = null; + try { + result = catalogService.updateDataReplica(request.getDataReplica()); + } catch (Exception e) { + throw new RuntimeException(e); + } DataReplicaUpdateResponse.Builder responseBuilder = DataReplicaUpdateResponse.newBuilder(); responseBuilder.setDataReplica(result); @@ -65,7 +95,12 @@ public void getReplicaLocation(DataReplicaGetRequest request, StreamObserver responseObserver) { - catalogService.deleteDataReplica(request.getDataReplicaId()); + try { + catalogService.deleteDataReplica(request.getDataReplicaId()); + } catch (Exception e) { + throw new RuntimeException(e); + } responseObserver.onNext(DataReplicaDeleteResponse.newBuilder().build()); responseObserver.onCompleted(); } @Override public void getAllReplicaLocation(AllDataReplicaGetRequest request, StreamObserver responseObserver) { - super.getAllReplicaLocation(request, responseObserver); + List replicaLocations = null; + try { + replicaLocations = catalogService.getDataReplicas(request.getDataProductUri()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + AllDataReplicaGetResponse.Builder responseBuilder = AllDataReplicaGetResponse.newBuilder(); + replicaLocations.forEach(entry -> { + responseBuilder.setDataProductUri(request.getDataProductUri()).addReplicaList( entry); + }); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } @Override diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java index fd9767d..e581592 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java @@ -1,9 +1,13 @@ package org.apache.airavata.replicacatalog.catalogapi.service.impl; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import java.util.UUID; import jakarta.transaction.Transactional; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.ReplicaGroupEntry; import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; @@ -27,7 +31,7 @@ public class ReplicaCatalogServiceImp implements IReplicaCatalogService { @Override - public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation) { + public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation) throws Exception{ DataReplicaLocationEntity dataReplicaLocationEntity = new DataReplicaLocationEntity(); @@ -44,7 +48,7 @@ public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation } @Override - public DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplicaLocation) { + public DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplicaLocation) throws Exception{ DataReplicaLocationEntity dataReplicaLocationEntity = dataReplicaLocationRepository.findByReplicaId(dataReplicaLocation.getDataReplicaId()); if (dataReplicaLocationEntity == null) { @@ -56,7 +60,7 @@ public DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplicaLoca } @Override - public DataReplicaLocation getDataReplica(String replicaId) { + public DataReplicaLocation getDataReplica(String replicaId) throws Exception { DataReplicaLocationEntity dataReplicaLocationEntity = dataReplicaLocationRepository.findByReplicaId(replicaId); if (dataReplicaLocationEntity == null) { @@ -66,12 +70,31 @@ public DataReplicaLocation getDataReplica(String replicaId) { } @Override - public void deleteDataReplica(String replicaId) { + public void deleteDataReplica(String replicaId) throws Exception { dataReplicaLocationRepository.deleteByReplicaId(replicaId); } - - private DataReplicaLocation toDataReplicaLocation(DataReplicaLocationEntity savedDataLocationEntity) { + @Override + public List getDataReplicas(String productUri) throws Exception{ + Optional> dataReplicaLocationEntities = dataReplicaLocationRepository.findByProductUri(productUri); + if (!dataReplicaLocationEntities.isPresent() || dataReplicaLocationEntities.get().isEmpty()) { + logger.debug("Data Replica Location not exists"); + } + List dataReplicaLocations = new ArrayList<>(); + dataReplicaLocationEntities.get().forEach(r -> { + ReplicaGroupEntry groupEntry = null; + try { + groupEntry = ReplicaGroupEntry.newBuilder() + .setDataReplicaId(r.getReplicaId()) + .addFiles(toDataReplicaLocation(r)).build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + dataReplicaLocations.add(groupEntry); + }); + return dataReplicaLocations; + } + private DataReplicaLocation toDataReplicaLocation(DataReplicaLocationEntity savedDataLocationEntity) throws Exception{ DataReplicaLocation.Builder builder = DataReplicaLocation.newBuilder(); replicaMapper.mapEntityToModel(savedDataLocationEntity, builder); return builder.build(); diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java new file mode 100644 index 0000000..0acc2d3 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java @@ -0,0 +1,25 @@ +package org.apache.airavata.replicacatalog.exception; + +public class EntityNotFoundException extends RuntimeException { + + public EntityNotFoundException() { + } + + public EntityNotFoundException(String message) { + super(message); + } + + public EntityNotFoundException(Throwable cause) { + super(cause); + } + + public EntityNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java new file mode 100644 index 0000000..4d23073 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java @@ -0,0 +1,9 @@ +package org.apache.airavata.replicacatalog.exception; + +public class InvalidDataException extends Exception { + + public InvalidDataException(String message) { + super(message); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java new file mode 100644 index 0000000..5ddde9e --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java @@ -0,0 +1,9 @@ +package org.apache.airavata.replicacatalog.exception; + +public class StorageNotSupportException extends RuntimeException { + + public StorageNotSupportException(String message) { + super(message); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java index d50422f..d817c6b 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java @@ -23,17 +23,17 @@ public class ResourceStorageMapper { @Autowired GenericResourceRepository genericResourceRepository; - public void mapGenericStorageModelToEntity(GenericResource storage, GenericResourceEntity resourceEntity) { + public void mapGenericStorageModelToEntity(GenericResource storage, GenericResourceEntity resourceEntity) throws Exception { resourceEntity.setReplicaId(storage.getReplicaId()); resourceEntity.setStorageId(storage.getResourceId()); - resourceEntity.setStorageType(GenericResourceEntity.StorageType.S3); + resourceEntity.setStorageType(resolveStorage(storage.getStorage())); resourceEntity.setResourcePath(storage.getFile().getResourcePath()); resourceEntity.setResourceType(GenericResourceEntity.ResourceType.FILE); // TODO } - public void mapGenericStorageEntityToModel(GenericResourceEntity resourceEntity, StorageWrapper wrapper, GenericResource.Builder builder) { + public void mapGenericStorageEntityToModel(GenericResourceEntity resourceEntity, StorageWrapper wrapper, GenericResource.Builder builder) throws Exception { builder.setResourceId(resourceEntity.getResourceId()); builder.setReplicaId(resourceEntity.getReplicaId()); @@ -56,4 +56,32 @@ public void mapStorageSecretEntityToModel(StorageSecretEntity resourceEntity, Se } + + protected GenericResourceEntity.StorageType resolveStorage (StorageWrapper wrapper){ + if(wrapper.hasS3Storage()){ + return GenericResourceEntity.StorageType.S3; + } else if (wrapper.hasGcsStorage()){ + return GenericResourceEntity.StorageType.GCS; + } else if (wrapper.hasFtpStorage()) { + return GenericResourceEntity.StorageType.FTP; + } + + return null; + } + + public StorageType resolveStorage(String type){ + GenericResourceEntity.StorageType storageType = GenericResourceEntity.StorageType.valueOf(type); + switch (storageType) { + case S3: + return StorageType.S3; + case GCS: + return StorageType.GCS; + case FTP: + return StorageType.FTP; + default: + throw new RuntimeException( + "Unexpected storage type: " + type); + } + } + } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java new file mode 100644 index 0000000..4130115 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "FTP_STORAGE") +public class FTPStorageEntity { + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "HOST") + private String host; + + @Column(name = "PORT") + private int port; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java new file mode 100644 index 0000000..569fcd6 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "GCS_STORAGE") +public class GCSStorageEntity { + + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator( name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "BUCKET_NAME") + private String bucketName; + + @Column(name = "STORAGE_NAME") + private String name; + + public String getStorageId() + { + return storageId; + } + + public void setStorageId( String storageId ) + { + this.storageId = storageId; + } + + public String getBucketName() + { + return bucketName; + } + + public void setBucketName( String bucketName ) + { + this.bucketName = bucketName; + } + + public String getName() + { + return name; + } + + public void setName( String name ) + { + this.name = name; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java new file mode 100644 index 0000000..1cd5fb6 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "LOCAL_STORAGE") +public class LocalStorageEntity { + + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "AGENT_ID") + private String agentId; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAgentId() { + return agentId; + } + + public void setAgentId(String agentId) { + this.agentId = agentId; + } +} \ No newline at end of file diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/S3StorageEntity.java similarity index 97% rename from replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java rename to replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/S3StorageEntity.java index 81f04f3..c6b3b7b 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/S3StorageEntity.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/S3StorageEntity.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.airavata.replicacatalog.resource.model; +package org.apache.airavata.replicacatalog.resource.model.storage; import jakarta.persistence.Column; import jakarta.persistence.Entity; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java new file mode 100644 index 0000000..94ea319 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import java.util.List; + +import org.apache.airavata.replicacatalog.resource.model.storage.FTPStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface FTPStorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java new file mode 100644 index 0000000..9288750 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import java.util.List; + +import org.apache.airavata.replicacatalog.resource.model.storage.GCSStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface GCSStorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java new file mode 100644 index 0000000..3d1d9f0 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import java.util.List; + +import org.apache.airavata.replicacatalog.resource.model.storage.LocalStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface LocalStorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/S3StorageRepository.java similarity index 88% rename from replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java rename to replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/S3StorageRepository.java index 4590b0b..aecebb0 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/S3StorageRepository.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/S3StorageRepository.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.airavata.replicacatalog.resource.repository; +package org.apache.airavata.replicacatalog.resource.repository.storage; -import org.apache.airavata.replicacatalog.resource.model.S3StorageEntity; +import org.apache.airavata.replicacatalog.resource.model.storage.S3StorageEntity; import org.springframework.data.domain.Pageable; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Repository; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java index 509220a..de00848 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java @@ -18,18 +18,24 @@ package org.apache.airavata.replicacatalog.resource.service; +import org.apache.airavata.replicacatalog.exception.InvalidDataException; +import org.apache.airavata.replicacatalog.exception.StorageNotSupportException; import org.apache.airavata.replicacatalog.resource.mapper.ResourceStorageMapper; import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; import org.apache.airavata.replicacatalog.resource.model.ResolveStorageEntity; -import org.apache.airavata.replicacatalog.resource.model.S3StorageEntity; +import org.apache.airavata.replicacatalog.resource.model.storage.GCSStorageEntity; +import org.apache.airavata.replicacatalog.resource.model.storage.S3StorageEntity; import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; import org.apache.airavata.replicacatalog.resource.repository.ResolveStorageRepository; -import org.apache.airavata.replicacatalog.resource.repository.S3StorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.storage.GCSStorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.storage.LocalStorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.storage.S3StorageRepository; import org.apache.airavata.replicacatalog.resource.repository.StorageSecretRepository; import org.apache.airavata.replicacatalog.resource.stubs.common.*; import org.apache.airavata.replicacatalog.resource.stubs.common.Error; +import org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage; import org.apache.airavata.replicacatalog.resource.stubs.s3.*; import org.dozer.DozerBeanMapper; import org.slf4j.Logger; @@ -53,6 +59,12 @@ public class SQLResourceService implements IResourceService { @Autowired private S3StorageRepository s3StorageRepository; + @Autowired + private GCSStorageRepository gcsStorageRepository; + + @Autowired + private LocalStorageRepository localStorageRepository; + @Autowired private StorageSecretRepository resourceSecretRepository; @@ -74,16 +86,54 @@ public void destroy() { @Override public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception { + + StorageType storageType = null; + switch (request.getResource().getStorage().getStorageCase().getNumber()) { + case StorageWrapper.S3STORAGE_FIELD_NUMBER: + storageType = StorageType.S3; + break; + case StorageWrapper.LOCALSTORAGE_FIELD_NUMBER: + storageType = StorageType.LOCAL; + break; + case StorageWrapper.GCSSTORAGE_FIELD_NUMBER: + storageType = StorageType.GCS; + break; + case StorageWrapper.FTPSTORAGE_FIELD_NUMBER: + storageType = StorageType.FTP; + break; + default: + break; + } + + GenericResourceEntity resourceEntity = new GenericResourceEntity(); + String storageId = ""; + StorageWrapper wrapper = null; + if (storageType == StorageType.S3) { + S3Storage storage = createS3Storage(request.getResource().getStorage().getS3Storage()); + if (storage != null) { + storageId = storage.getStorageId(); + wrapper = StorageWrapper.newBuilder().setS3Storage(storage).build(); + } + + } else if (storageType == StorageType.GCS) { + GCSStorage storage = createGCSStorage(request.getResource().getStorage().getGcsStorage()); + if (storage != null) { + storageId = storage.getStorageId(); + wrapper = StorageWrapper.newBuilder().setGcsStorage(storage).build(); + } - S3Storage storage = null; - if (request.getStorageType() == StorageType.S3) { - storage = createS3Storage(request.getResource().getStorage().getS3Storage()); + } else { + throw new StorageNotSupportException( "Storage type not supported/implemented"); } - StorageWrapper wrapper = StorageWrapper.newBuilder().setS3Storage(storage).build(); + + if (wrapper == null) { + throw new InvalidDataException("Storage not created."); + } + resourceEntity.setResourceId(UUID.randomUUID().toString()); resourceStorageMapper.mapGenericStorageModelToEntity(request.getResource(), resourceEntity); - resourceEntity.setStorageId(storage.getStorageId()); + resourceEntity.setStorageId(storageId); GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); GenericResource.Builder responseBuilder = GenericResource.newBuilder(); @@ -104,14 +154,21 @@ public GenericResource getGenericResource(GenericResourceGetRequest request) thr return null; } GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(); - Optional storage = Optional.empty(); StorageWrapper wrapper = null; if (StorageType.S3.name().equals(resourceEntity.getStorageType().name())) { + Optional storage = Optional.empty(); storage = s3StorageRepository.findById(resourceEntity.getStorageId()); if (storage.isPresent()) { S3Storage s3 = mapper.map(storage.get(), S3Storage.newBuilder().getClass()).build(); wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); } + } else if (StorageType.GCS.name().equals(resourceEntity.getStorageType().name())) { + Optional storage = Optional.empty(); + storage = gcsStorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + GCSStorage gcs = mapper.map(storage.get(), GCSStorage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setGcsStorage(gcs).build(); + } } GenericResource.Builder responseBuilder = GenericResource.newBuilder(); @@ -136,9 +193,9 @@ public GenericResource updateGenericResource(GenericResourceUpdateRequest reques return null; } GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(); - Optional storage = null; StorageWrapper wrapper = null; if (StorageType.S3.name().equals(resourceEntity.getStorageType().name() )) { + Optional storage = null; storage = s3StorageRepository.findById(resourceEntity.getStorageId()); if (storage.isPresent()) { boolean updated = updateS3Storage(storage.get(), request.getGenericResource().getStorage().getS3Storage()); @@ -147,8 +204,17 @@ public GenericResource updateGenericResource(GenericResourceUpdateRequest reques wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); } } + } else if (StorageType.GCS.name().equals(resourceEntity.getStorageType().name())) { + Optional storage = null; + storage = gcsStorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + boolean updated = updateGCSStorage(storage.get(), request.getGenericResource().getStorage().getGcsStorage()); + if (updated) { + GCSStorage gcs = mapper.map(storage.get(), GCSStorage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setGcsStorage(gcs).build(); + } + } } - resourceStorageMapper.mapGenericStorageModelToEntity(request.getGenericResource(), resourceEntity); GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); @@ -172,6 +238,7 @@ public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) StorageSecretEntity storageSecretEntity = resourceSecEtyOp.get(); resultBuilder.setSecretId(storageSecretEntity.getSecretId()); resultBuilder.setStorageId(storageSecretEntity.getStorageId()); + resultBuilder.setStorageType(resourceStorageMapper.resolveStorage( storageSecretEntity.getType())); } else { resultBuilder.setError(Error.NOT_FOUND); } @@ -295,6 +362,26 @@ public boolean deleteS3Storage(S3StorageDeleteRequest request) throws Exception } + public GCSStorage createGCSStorage(GCSStorage request) throws Exception { + GCSStorageEntity savedEntity = gcsStorageRepository.save(mapper.map(request, GCSStorageEntity.class)); + + ResolveStorageEntity storageTypeEty = new ResolveStorageEntity(); + storageTypeEty.setStorageId(savedEntity.getStorageId()); + storageTypeEty.setStorageType(ResolveStorageEntity.StorageType.GCS); + storageTypeEty.setStorageName(savedEntity.getName()); + resolveStorageRepository.save(storageTypeEty); + + return mapper.map(savedEntity, GCSStorage.newBuilder().getClass()).build(); + } + + + public boolean updateGCSStorage(GCSStorageEntity entity, GCSStorage storage) throws Exception { + entity.setName(storage.getName()); + entity.setBucketName(storage.getBucketName()); + gcsStorageRepository.save(entity); + return true; + } + @Override public StorageTypeResolveResponse resolveStorageType(StorageTypeResolveRequest request) throws Exception { Optional resolveStorageOp = resolveStorageRepository.getByStorageId(request.getStorageId()); diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java new file mode 100644 index 0000000..8d02d06 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "FTP_SECRET") +public class FTPSecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "USER_ID") + private String userId; + + @Column(name = "PASSWORD") + private String password; + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java new file mode 100644 index 0000000..339cb64 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "GCS_SECRET") +public class GCSSecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator( name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "PROJECT_ID") + private String projectId; + + @Column(name = "PRIVATE_KEY", length = 3000) + private String privateKey; + + @Column(name = "CLIENT_EMAIL") + private String clientEmail; + + + public String getSecretId() + { + return secretId; + } + + public void setSecretId( String secretId ) + { + this.secretId = secretId; + } + + public String getProjectId() + { + return projectId; + } + + public void setProjectId( String projectId ) + { + this.projectId = projectId; + } + + public String getPrivateKey() + { + return privateKey; + } + + public void setPrivateKey( String privateKey ) + { + this.privateKey = privateKey; + } + + public String getClientEmail() + { + return clientEmail; + } + + public void setClientEmail( String clientEmail ) + { + this.clientEmail = clientEmail; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java new file mode 100644 index 0000000..1a8088c --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.repository; + +import java.util.Optional; + +import org.apache.airavata.replicacatalog.sceret.model.FTPSecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface FTPSecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java new file mode 100644 index 0000000..6a98676 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.repository; + +import java.util.Optional; + +import org.apache.airavata.replicacatalog.sceret.model.GCSSecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface GCSSecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java index 59db567..533579e 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java @@ -19,9 +19,12 @@ import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; +import org.apache.airavata.replicacatalog.sceret.model.GCSSecretEntity; import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; +import org.apache.airavata.replicacatalog.sceret.repository.GCSSecretRepository; import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; import org.apache.airavata.replicacatalog.secret.stubs.common.*; +import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret; import org.apache.airavata.replicacatalog.secret.stubs.s3.*; import org.dozer.DozerBeanMapper; import org.slf4j.Logger; @@ -43,6 +46,9 @@ public class SQLSecretService implements ISecretService { @Autowired private S3SecretRepository s3SecretRepository; + @Autowired + private GCSSecretRepository gcsSecretRepository; + private DozerBeanMapper mapper = new DozerBeanMapper(); @Override @@ -64,9 +70,12 @@ public StorageSecret getSecretForStorage(SecretGetRequest request) throws Except public StorageSecret registerSecretForStorage(SecretCreateRequest request) throws Exception { StorageSecret stoReq = request.getSecret(); StorageSecret.Builder storageSecret = StorageSecret.newBuilder(); - if (stoReq.getStorageType().name().equals(StorageType.S3.name())) { - S3Secret secret= createS3Secret(stoReq.getSecret().getS3Secret()); - return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret( secret )).setSecretId(secret.getSecretId()).build(); + if (StorageType.S3.name().equals(stoReq.getStorageType().name())) { + S3Secret secret = createS3Secret(stoReq.getSecret().getS3Secret()); + return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret(secret)).setSecretId(secret.getSecretId()).build(); + } else if (StorageType.GCS.name().equals(stoReq.getStorageType().name())) { + GCSSecret secret = createGCSSecret(stoReq.getSecret().getGcsSecret()); + return storageSecret.setSecret(SecretWrapper.newBuilder().setGcsSecret(secret)).setSecretId(secret.getSecretId()).build(); } return null; } @@ -107,4 +116,10 @@ public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { return true; } + + public GCSSecret createGCSSecret(GCSSecret request) throws Exception { + GCSSecretEntity savedEntity = gcsSecretRepository.save(mapper.map(request, GCSSecretEntity.class)); + return mapper.map(savedEntity, GCSSecret.newBuilder().getClass()).build(); + } + } diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto index 79c264f..4f9a992 100644 --- a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto @@ -58,11 +58,17 @@ enum Permission { } enum StorageType { - LOCAL = 0; + NONE = 0; S3 = 1; - GCS = 2; - AZURE = 3; - + FTP = 2; + LOCAL = 3; + GCS = 4; + BOX = 5; + DROPBOX = 6; + AZURE = 7; + SWIFT = 8; + ODATA = 9; + SCP = 10; } enum DataProductType { @@ -70,10 +76,21 @@ enum DataProductType { COLLECTION = 1; } +enum ErrorCode { + INVALID_DATA = 0; + INTERNAL_ERROR = 1; + MAPPING_ERROR = 2; +} + +message Error { + ErrorCode code = 1; + string message = 2; +} + // Entity which used to model a replica entry message DataReplicaLocation { string data_replica_id = 1; // Unique replica id - string data_product_id = 2; // related data uri + string data_product_uri = 2; // related data uri string parent_replica_id = 3; // If there is parent replica (Like directory structure) string storage_resource_id = 4; // Replica storage id (TO link the storage entry) string replica_name = 5; // Replica user friendly name @@ -90,6 +107,7 @@ message DataReplicaCreateRequest { } message DataReplicaCreateResponse{ DataReplicaLocation data_replica = 1; + Error error = 2; } message DataReplicaGroupEntryCreateRequest { @@ -97,36 +115,41 @@ message DataReplicaGroupEntryCreateRequest { } message DataReplicaGroupEntryCreateResponse{ ReplicaGroupEntry data_replica_group = 1; -} + Error error = 2; +} message DataReplicaUpdateRequest { DataReplicaLocation data_replica = 1; } message DataReplicaUpdateResponse{ DataReplicaLocation data_replica = 1; + Error error = 2; } message DataReplicaGetRequest { string data_replica_id = 1; } message DataReplicaGetResponse { DataReplicaLocation data_replica = 1; + Error error = 2; } message DataReplicaDeleteRequest { string data_replica_id = 1; } message DataReplicaDeleteResponse { + Error error = 1; } message AllDataReplicaGetRequest { - string data_product_id = 1; + string data_product_uri = 1; int32 page_number = 2; int32 page_size = 3; } message AllDataReplicaGetResponse { - string data_product_id = 1; + string data_product_uri = 1; repeated ReplicaGroupEntry replica_list = 2; + Error error = 3; } message ReplicaGroupEntry { @@ -143,10 +166,11 @@ message ReplicaListEntry { } message AllDataReplicaDeleteRequest { - string data_product_id = 1; + string data_product_uri = 1; } message AllDataReplicaDeleteResponse{ int32 replica_location_count = 1; + Error error = 2; } diff --git a/replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto b/replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto new file mode 100644 index 0000000..cd971c8 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.ftp; + +message FTPStorage { + string storage_id = 1; + string host = 2; + int32 port = 3; + string name = 4; +} + +message FTPStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message FTPStorageListResponse { + repeated FTPStorage storages = 1; +} + +message FTPStorageGetRequest { + string storage_id = 1; +} + +message FTPStorageCreateRequest { + string host = 1; + int32 port = 2; + string storage_id = 3; + string name = 4; +} + +message FTPStorageUpdateRequest { + string storage_id = 1; + string host = 2; + int32 port = 3; + string name = 4; +} + +message FTPStorageUpdateResponse { + string storage_id = 1; +} + +message FTPStorageDeleteRequest { + string storage_id = 1; +} + +message FTPStorageDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto b/replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto new file mode 100644 index 0000000..0296ac1 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.gcs; + +message GCSStorage { + string storage_id = 1; + string bucket_name = 2; + string name = 3; +} + +message GCSStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message GCSStorageListResponse { + repeated GCSStorage storages = 1; +} + +message GCSStorageGetRequest { + string storage_id = 1; +} + +message GCSStorageCreateRequest { + string bucket_name = 1; + string storage_id = 2; + string name = 3; +} + +message GCSStorageUpdateRequest { + string storage_id = 1; + string bucketName = 2; + string name = 3; +} + +message GCSStorageUpdateResponse { + string storage_id = 1; +} + +message GCSStorageDeleteRequest { + string storage_id = 1; +} + +message GCSStorageDeleteResponse { + bool status = 1; +} + + diff --git a/replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto b/replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto new file mode 100644 index 0000000..e48cd81 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.local; + +message LocalStorage { + string storage_id = 1; + string name = 2; + string agent_id = 3; + +} + +message LocalStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message LocalStorageListResponse { + repeated LocalStorage storages = 1; +} + +message LocalStorageGetRequest { + string storage_id = 1; +} + +message LocalStorageCreateRequest { + string agentId = 1; + string storage_id = 2; + string name = 3; +} + +message LocalStorageUpdateRequest { + string storage_id = 1; + string name = 2; + string agent_id = 3; +} + +message LocalStorageUpdateResponse { + string storage_id = 1; +} + +message LocalStorageDeleteRequest { + string storage_id = 1; +} + +message LocalStorageDeleteResponse { + bool status = 1; +} + diff --git a/replica-catalog-api/stubs/src/main/proto/resource/s3/S3Storage.proto b/replica-catalog-api/stubs/src/main/proto/resource/S3Storage.proto similarity index 100% rename from replica-catalog-api/stubs/src/main/proto/resource/s3/S3Storage.proto rename to replica-catalog-api/stubs/src/main/proto/resource/S3Storage.proto diff --git a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto similarity index 92% rename from replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto rename to replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto index 5da9b38..9c2e28b 100644 --- a/replica-catalog-api/stubs/src/main/proto/resource/common/StorageCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto @@ -20,7 +20,10 @@ syntax = "proto3"; option java_multiple_files = true; package org.apache.airavata.replicacatalog.resource.stubs.common; -import "resource/s3/S3Storage.proto"; +import "resource/S3Storage.proto"; +import "resource/FTPStorage.proto"; +import "resource/LocalStorage.proto"; +import "resource/GCSStorage.proto"; enum StorageType { @@ -78,6 +81,9 @@ message GenericResource { message StorageWrapper { oneof storage { org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage s3Storage = 1; + org.apache.airavata.replicacatalog.resource.stubs.local.LocalStorage localStorage = 2; + org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage gcsStorage = 3; + org.apache.airavata.replicacatalog.resource.stubs.ftp.FTPStorage ftpStorage = 4; } } @@ -89,7 +95,6 @@ message GenericResourceGetRequest { message GenericResourceCreateRequest { string storage_id = 1; GenericResource resource = 2; - StorageType storage_type = 3; } message GenericResourceUpdateRequest { diff --git a/replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto b/replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto new file mode 100644 index 0000000..632e51a --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.secret.stubs.ftp; + + +message FTPSecret { + string secretId = 1; + string userId = 2; + string password = 3; +} + +message FTPSecretGetRequest { + string secretId = 1; +} + +message FTPSecretCreateRequest { + string userId = 1; + string password = 2; +} + +message FTPSecretUpdateRequest { + string secretId = 1; + string userId = 2; + string password = 3; +} + +message FTPSecretUpdateResponse { + string secretId = 1; +} + +message FTPSecretDeleteRequest { + string secretId = 1; +} + +message FTPSecretDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto b/replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto new file mode 100644 index 0000000..0336ecb --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.secret.stubs.gcs; + + +// This represents GCS Service account credentials. https://cloud.google.com/iam/docs/service-accounts +message GCSSecret { + string secretId = 1; + string projectId = 2; + string privateKey = 3; + string clientEmail = 4; +} + +message GCSSecretGetRequest { + string secretId = 1; +} + +message GCSSecretCreateRequest { + string projectId = 1; + string privateKey = 2; + string clientEmail = 3; +} + +message GCSSecretUpdateRequest { + string secretId = 1; + string projectId = 2; + string privateKey = 3; + string clientEmail = 4; +} + +message GCSSecretUpdateResponse { + string secretId = 1; +} + +message GCSSecretDeleteRequest { + string secretId = 1; +} + +message GCSSecretDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/s3/S3Credential.proto b/replica-catalog-api/stubs/src/main/proto/secret/S3Credential.proto similarity index 100% rename from replica-catalog-api/stubs/src/main/proto/secret/s3/S3Credential.proto rename to replica-catalog-api/stubs/src/main/proto/secret/S3Credential.proto diff --git a/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto b/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto similarity index 92% rename from replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto rename to replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto index 1e502aa..4d851e9 100644 --- a/replica-catalog-api/stubs/src/main/proto/secret/common/SecretCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto @@ -20,7 +20,9 @@ syntax = "proto3"; option java_multiple_files = true; package org.apache.airavata.replicacatalog.secret.stubs.common; -import "secret/s3/S3Credential.proto"; +import "secret/S3Credential.proto"; +import "secret/FTPCredential.proto"; +import "secret/GCSCredential.proto"; enum StorageType { S3 = 0; @@ -61,6 +63,8 @@ message StorageSecret { message SecretWrapper { oneof secret { org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret s3Secret = 1; + org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret gcsSecret = 2; + org.apache.airavata.replicacatalog.secret.stubs.ftp.FTPSecret ftpSecret = 3; } } From 15e8bc915be1fe674493f36c7848db42daa5fe68 Mon Sep 17 00:00:00 2001 From: jayancv Date: Thu, 20 Apr 2023 22:44:38 +0530 Subject: [PATCH 8/8] Load stored storage and secret --- .../airavata/ReplicaCatalogAPIClient.java | 3 +- .../airavata/ReplicaCatalogAPIClientTest.java | 50 +++++++++++-------- .../mapper/ResourceStorageMapper.java | 2 +- .../resource/service/SQLResourceService.java | 1 + .../sceret/service/ISecretService.java | 8 +-- .../sceret/service/SQLSecretService.java | 27 ++++++++-- .../service/StorageSecretAPIService.java | 13 +++++ .../main/proto/resource/StorageCommon.proto | 36 ++++++------- .../src/main/proto/secret/SecretCommon.proto | 29 ++++++----- 9 files changed, 106 insertions(+), 63 deletions(-) diff --git a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java index 8e70d78..bafc8ff 100644 --- a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java +++ b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java @@ -16,7 +16,6 @@ import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; -import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; @@ -101,7 +100,7 @@ public static void main(String[] args) throws InterruptedException { .setSessionToken("token").build(); StorageSecret storageSecret = StorageSecret.newBuilder() .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) - .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3).build(); + .setStorageType(StorageType.S3).build(); SecretCreateRequest secretCreateRequest = SecretCreateRequest.newBuilder().setSecret(storageSecret).build(); StorageSecret secretResult = client.createSecret(channel, secretCreateRequest); diff --git a/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java index 24f6cec..5a6a86b 100644 --- a/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java +++ b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java @@ -14,6 +14,7 @@ import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetRequest; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetResponse; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; @@ -21,11 +22,11 @@ import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; -import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; import org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage; import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretGetRequest; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretWrapper; import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret; @@ -81,7 +82,7 @@ public void testCase1() throws InterruptedException { .setSessionToken("token").build(); StorageSecret storageSecret = StorageSecret.newBuilder() .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) - .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.S3).build(); + .setStorageType(StorageType.S3).build(); SecretCreateRequest secretCreateRequest = SecretCreateRequest.newBuilder().setSecret(storageSecret).build(); StorageSecret secretResult = createSecret(client, channel, secretCreateRequest); @@ -117,7 +118,7 @@ public void testCase1() throws InterruptedException { .setProjectId("1258").build(); StorageSecret storageSecret2 = StorageSecret.newBuilder() .setSecret(SecretWrapper.newBuilder().setGcsSecret(gcsSecret).build()) - .setStorageType(org.apache.airavata.replicacatalog.secret.stubs.common.StorageType.GCS).build(); + .setStorageType(StorageType.GCS).build(); SecretCreateRequest secretCreateRequest2 = SecretCreateRequest.newBuilder().setSecret(storageSecret2).build(); StorageSecret secretResult2 = createSecret(client, channel, secretCreateRequest2); @@ -145,24 +146,33 @@ public void testCase1() throws InterruptedException { replicas.getReplicaListList().forEach(r -> { - DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(r.getDataReplicaId()).build(); - DataReplicaGetResponse replicaResponse = client.getReplicaServiceAPI().getReplicaLocation(replicaGetRequest); - if (replicaResponse != null) { - SecretForStorage secretForStorage1 = null; - try { - secretForStorage1 = getStorageSecretIds(client, channel, replicaResponse.getDataReplica().getDataReplicaId()); - } catch (Exception e) { - throw new RuntimeException(e); - } - if (secretForStorage1 != null) { - System.out.println( - MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}] Storage Type [{3}]", - (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(), secretForStorage1.getStorageId(), secretForStorage1.getSecretId(), - secretForStorage1.getStorageType().name()})); - - } - } + DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(r.getDataReplicaId()).build(); + DataReplicaGetResponse replicaResponse = client.getReplicaServiceAPI().getReplicaLocation(replicaGetRequest); + if (replicaResponse != null) { + SecretForStorage secretForStorage1 = null; + try { + secretForStorage1 = getStorageSecretIds(client, channel, replicaResponse.getDataReplica().getDataReplicaId()); + SecretGetRequest secretGetRequest = SecretGetRequest.newBuilder().setSecretId(secretForStorage1.getSecretId()).setStorageType(secretForStorage1.getStorageType()).build(); + StorageSecret secretLoad = client.getBlockingSecretStub(channel).getSecret(secretGetRequest); + System.out.println(secretLoad); + + + GenericResourceGetRequest genericResourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaResponse.getDataReplica().getDataReplicaId()).build(); + GenericResource resource1 = client.getBlockingStorageStub(channel).getGenericResource(genericResourceGetRequest); + System.out.println(resource1); + + } catch (Exception e) { + throw new RuntimeException(e); + } + if (secretForStorage1 != null) { + System.out.println( + MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}] Storage Type [{3}]", + (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(), secretForStorage1.getStorageId(), secretForStorage1.getSecretId(), + secretForStorage1.getStorageType().name()})); + } + } + } ); diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java index d817c6b..7f95cb5 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java @@ -1,13 +1,13 @@ package org.apache.airavata.replicacatalog.resource.mapper; import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; -import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java index de00848..7f6e127 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java @@ -18,6 +18,7 @@ package org.apache.airavata.replicacatalog.resource.service; +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; import org.apache.airavata.replicacatalog.exception.InvalidDataException; import org.apache.airavata.replicacatalog.exception.StorageNotSupportException; import org.apache.airavata.replicacatalog.resource.mapper.ResourceStorageMapper; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java index d637329..49aef84 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java @@ -26,11 +26,11 @@ public interface ISecretService { public void init(); public void destroy(); - public StorageSecret getSecretForStorage(SecretGetRequest request) throws Exception; + public StorageSecret getSecret(SecretGetRequest request) throws Exception; public StorageSecret registerSecretForStorage(SecretCreateRequest request) throws Exception; - public boolean deleteSecretForStorage(SecretDeleteRequest request) throws Exception; - public SecretListResponse searchStorages(SecretSearchRequest request) throws Exception; - public SecretListResponse listStorage(SecretListRequest request) throws Exception; + public boolean deleteSecret(SecretDeleteRequest request) throws Exception; + public SecretListResponse searchSecrets(SecretSearchRequest request) throws Exception; + public SecretListResponse listSecrets(SecretListRequest request) throws Exception; // public Optional getS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretGetRequest request) throws Exception; // public org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret createS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretCreateRequest request) throws Exception; diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java index 533579e..0cb9967 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java @@ -17,7 +17,7 @@ package org.apache.airavata.replicacatalog.sceret.service; -import org.apache.airavata.replicacatalog.resource.stubs.common.StorageType; +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; import org.apache.airavata.replicacatalog.sceret.model.GCSSecretEntity; import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; @@ -25,6 +25,7 @@ import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; import org.apache.airavata.replicacatalog.secret.stubs.common.*; import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret; +import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecretGetRequest; import org.apache.airavata.replicacatalog.secret.stubs.s3.*; import org.dozer.DozerBeanMapper; import org.slf4j.Logger; @@ -62,8 +63,20 @@ public void destroy() { } @Override - public StorageSecret getSecretForStorage(SecretGetRequest request) throws Exception { + public StorageSecret getSecret(SecretGetRequest request) throws Exception { + String id = request.getSecretId(); + StorageSecret.Builder storageSecret = StorageSecret.newBuilder(); + if (StorageType.S3.name().equals(request.getStorageType().name())) { + S3SecretGetRequest getRequest = S3SecretGetRequest.newBuilder().setSecretId(id).build(); + Optional secret = getS3Secret(getRequest); + return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret(secret.get())).setSecretId(id).build(); + } else if (StorageType.GCS.name().equals(request.getStorageType().name())) { + GCSSecretGetRequest getRequest = GCSSecretGetRequest.newBuilder().setSecretId(id).build(); + Optional secret = getGCSSecret(getRequest); + return storageSecret.setSecret(SecretWrapper.newBuilder().setGcsSecret(secret.get())).setSecretId(id).build(); + } return null; + } @Override @@ -81,17 +94,17 @@ public StorageSecret registerSecretForStorage(SecretCreateRequest request) throw } @Override - public boolean deleteSecretForStorage(SecretDeleteRequest request) throws Exception { + public boolean deleteSecret(SecretDeleteRequest request) throws Exception { return false; } @Override - public SecretListResponse searchStorages(SecretSearchRequest request) throws Exception { + public SecretListResponse searchSecrets(SecretSearchRequest request) throws Exception { return null; } @Override - public SecretListResponse listStorage(SecretListRequest request) throws Exception { + public SecretListResponse listSecrets(SecretListRequest request) throws Exception { return null; } @@ -122,4 +135,8 @@ public GCSSecret createGCSSecret(GCSSecret request) throws Exception { return mapper.map(savedEntity, GCSSecret.newBuilder().getClass()).build(); } + public Optional getGCSSecret(GCSSecretGetRequest request) throws Exception { + Optional secretEty = gcsSecretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(entity -> mapper.map(entity, GCSSecret.newBuilder().getClass()).build()); + } } diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java index 295591b..c570e4c 100644 --- a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java @@ -8,6 +8,7 @@ import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretGetRequest; import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; import org.lognet.springboot.grpc.GRpcService; import org.slf4j.Logger; @@ -39,4 +40,16 @@ public void registerSecret(SecretCreateRequest request, StreamObserver responseObserver) { + StorageSecret secretResult = null; + try { + secretResult = secretService.getSecret(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(secretResult); + responseObserver.onCompleted(); + } } diff --git a/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto index 9c2e28b..2a0c929 100644 --- a/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto @@ -24,20 +24,21 @@ import "resource/S3Storage.proto"; import "resource/FTPStorage.proto"; import "resource/LocalStorage.proto"; import "resource/GCSStorage.proto"; - - -enum StorageType { - S3 = 0; - SCP = 1; - FTP = 2; - LOCAL = 3; - BOX = 4; - DROPBOX = 5; - GCS = 6; - AZURE = 7; - SWIFT = 8; - ODATA = 9; -} +import "catalogapi/ReplicaCatalogAPI.proto"; + + +//enum StorageType { +// S3 = 0; +// SCP = 1; +// FTP = 2; +// LOCAL = 3; +// BOX = 4; +// DROPBOX = 5; +// GCS = 6; +// AZURE = 7; +// SWIFT = 8; +// ODATA = 9; +//} enum Error { NOT_FOUND = 0; @@ -124,10 +125,9 @@ message GenericResourceDeleteResponse { message SecretForStorage { string storage_id = 1; string secret_id = 2; - string replica_id = 3; - string user_identity = 4; - StorageType storage_type = 5; - Error error = 6; + string user_identity = 3; + StorageType storage_type = 4; + Error error = 5; } message SecretForStorageCreateRequest { diff --git a/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto b/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto index 4d851e9..8785fdc 100644 --- a/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto +++ b/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto @@ -23,19 +23,20 @@ package org.apache.airavata.replicacatalog.secret.stubs.common; import "secret/S3Credential.proto"; import "secret/FTPCredential.proto"; import "secret/GCSCredential.proto"; - -enum StorageType { - S3 = 0; - SCP = 1; - FTP = 2; - LOCAL = 3; - BOX = 4; - DROPBOX = 5; - GCS = 6; - AZURE = 7; - SWIFT = 8; - ODATA = 9; -} +import "catalogapi/ReplicaCatalogAPI.proto"; + +//enum StorageType { +// S3 = 0; +// SCP = 1; +// FTP = 2; +// LOCAL = 3; +// BOX = 4; +// DROPBOX = 5; +// GCS = 6; +// AZURE = 7; +// SWIFT = 8; +// ODATA = 9; +//} enum Error { NOT_FOUND = 0; @@ -75,10 +76,12 @@ message SecretCreateRequest { message SecretGetRequest { string secret_id = 1; + StorageType storage_type = 2; } message SecretDeleteRequest { string secret_id = 1; + StorageType storage_type = 2; } message SecretDeleteResponse {