diff --git a/pom.xml b/pom.xml index 2fd997b..a05e80b 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ under the License. data-catalog-api + replica-catalog-api @@ -47,6 +48,10 @@ under the License. 1.52.1 ${grpc.version} 6.0.53 + 3.1.1 + 2.14.1 + 5.5.1 + 5.8.1 diff --git a/replica-catalog-api/client/pom.xml b/replica-catalog-api/client/pom.xml new file mode 100644 index 0000000..9d746a1 --- /dev/null +++ b/replica-catalog-api/client/pom.xml @@ -0,0 +1,35 @@ + + + 4.0.0 + + org.apache.airavata + replica-catalog-api + 0.1-SNAPSHOT + + + replica-catalog-api-client + + + + org.apache.airavata + replica-catalog-api-stubs + 0.1-SNAPSHOT + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.junit.jupiter + junit-jupiter-engine + ${jupiter.version} + test + + + + + \ No newline at end of file diff --git a/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java new file mode 100644 index 0000000..bafc8ff --- /dev/null +++ b/replica-catalog-api/client/src/main/java/org/apache/airavata/ReplicaCatalogAPIClient.java @@ -0,0 +1,211 @@ +package org.apache.airavata; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder;; +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +import org.apache.airavata.replicacatalog.catalog.service.ReplicaCatalogAPIServiceGrpc; +import org.apache.airavata.replicacatalog.catalog.stubs.*; +import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; +import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretWrapper; +import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReplicaCatalogAPIClient { + private static final Logger logger = LoggerFactory.getLogger(ReplicaCatalogAPIClient.class); + + private final ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub replicaServiceAPI; + private StorageCommonServiceGrpc.StorageCommonServiceBlockingStub storageServiceAPI = null; + private SecretCommonServiceGrpc.SecretCommonServiceBlockingStub secretServiceAPI = null; + + public ReplicaCatalogAPIClient(Channel channel) { + replicaServiceAPI = ReplicaCatalogAPIServiceGrpc.newBlockingStub(channel); + } + + public ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceBlockingStub getReplicaServiceAPI() { + return replicaServiceAPI; + } + + public StorageCommonServiceGrpc.StorageCommonServiceBlockingStub getBlockingStorageStub(Channel channel) { + if (storageServiceAPI == null) { + storageServiceAPI = StorageCommonServiceGrpc.newBlockingStub(channel); + } + return storageServiceAPI; + } + + public SecretCommonServiceGrpc.SecretCommonServiceBlockingStub getBlockingSecretStub(Channel channel) { + if (secretServiceAPI == null) { + secretServiceAPI = SecretCommonServiceGrpc.newBlockingStub(channel); + } + return secretServiceAPI; + } + + public static void main(String[] args) throws InterruptedException { + String target = "localhost:6565"; + + /* + --- Sample scenario 1--- + Airavata MFT copied a 100GB astrological data file to Amazon S3 bucket + User can access that S3 bucket data file via Replica catalog details + + 1) Register Data Product in data catalog + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + ManagedChannel channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build(); + try { + ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient(channel); + + String testUri = "TestUri"; + + DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder() + .setReplicaName("ASTRO S3") + .setReplicaDescription("S3 replica") + .setDataProductUri(testUri) + .setCreationTime(System.currentTimeMillis()).build(); + DataReplicaLocation replicaResult = client.createReplicaLocation(replicaLocation); + + S3Storage storage = S3Storage.newBuilder() + .setName("ASTRO S3") + .setBucketName("arn:aws:s3:::mftjayan") + .setRegion("us-east-1") + .setEndpoint("https://s3.us-east-1.amazonaws.com").build(); + + GenericResource resource = GenericResource.newBuilder() + .setReplicaId(replicaResult.getDataReplicaId()) + .setStorage(StorageWrapper.newBuilder().setS3Storage(storage).build()) + .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); + GenericResource resourceResult = client.createStorage(channel, resource); + + S3Secret secret = S3Secret.newBuilder() + .setAccessKey("access") + .setSecretKey("secKey") + .setSessionToken("token").build(); + StorageSecret storageSecret = StorageSecret.newBuilder() + .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) + .setStorageType(StorageType.S3).build(); + SecretCreateRequest secretCreateRequest = SecretCreateRequest.newBuilder().setSecret(storageSecret).build(); + StorageSecret secretResult = client.createSecret(channel, secretCreateRequest); + + SecretForStorage secretForStorage = SecretForStorage.newBuilder() + .setStorageId(resourceResult.getStorage().getS3Storage().getStorageId()) + .setSecretId(secretResult.getSecret().getS3Secret().getSecretId()) + .setStorageType(StorageType.S3) + .setUserIdentity("userID#12345").build(); + SecretForStorageCreateRequest createRequest = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage).build(); + SecretForStorage secretForStorageResult = client.createCommonStorage(channel, createRequest); + + System.out.println( + MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}]", + (Object[]) new String[]{replicaResult.getDataReplicaId(), secretForStorageResult.getStorageId(), secretForStorageResult.getSecretId()})); + + DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(replicaResult.getDataReplicaId()).build(); + DataReplicaGetResponse replicaResponse = client.replicaServiceAPI.getReplicaLocation(replicaGetRequest); + if (replicaResponse != null && replicaResponse.getDataReplica() != null) { + SecretForStorage secretForStorage1 = null; + try { + secretForStorage1 = client.getStorageSecretIds(channel, replicaResponse.getDataReplica().getDataReplicaId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + if (secretForStorage1 != null) { + System.out.println( + MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}]", + (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(), secretForStorage1.getStorageId(), secretForStorage1.getSecretId()})); + + } + } + + /* + --- Sample scenario 2--- + Airavata MFT copied a 1000GB astrological data Folder which contains 100 data files to Amazon S3 bucket + User can access that S3 bucket data folder via Replica catalog details + + 1) Register Data Product + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + + } finally { + channel.shutdownNow().awaitTermination(180, TimeUnit.SECONDS); + } + } + + public DataReplicaLocation createReplicaLocation(DataReplicaLocation replicaLocation) { + DataReplicaCreateRequest request = DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); + DataReplicaCreateResponse response = replicaServiceAPI.registerReplicaLocation(request); + return response.getDataReplica(); + } + + + public SecretForStorage createCommonStorage(Channel channel, SecretForStorageCreateRequest createRequest) { + + SecretForStorage response = getBlockingStorageStub(channel).registerSecretForStorage(createRequest); + return response; + } + + public GenericResource createStorage(Channel channel, GenericResource resource) { + StorageType storageType = null; + switch (resource.getStorage().getStorageCase().getNumber()) { + case StorageWrapper.S3STORAGE_FIELD_NUMBER: + storageType = StorageType.S3; + break; + case StorageWrapper.LOCALSTORAGE_FIELD_NUMBER: + storageType = StorageType.LOCAL; + break; + case StorageWrapper.GCSSTORAGE_FIELD_NUMBER: + storageType = StorageType.GCS; + break; + case StorageWrapper.FTPSTORAGE_FIELD_NUMBER: + storageType = StorageType.FTP; + break; + default: + break; + } + GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder().setResource(resource).build(); + GenericResource response = getBlockingStorageStub(channel).createGenericResource(request); + return response; + } + + public StorageSecret createSecret(Channel channel, SecretCreateRequest createRequest) { + StorageSecret response = getBlockingSecretStub(channel).registerSecret(createRequest); + return response; + } + + public SecretForStorage getStorageSecretIds(Channel channel, String replicaId) throws Exception { + GenericResourceGetRequest resourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaId).build(); + GenericResource resource = getBlockingStorageStub(channel).getGenericResource(resourceGetRequest); + String storageId = null; + if (resource.getStorage().hasS3Storage()) { + storageId = resource.getStorage().getS3Storage().getStorageId(); + } else if (resource.getStorage().hasGcsStorage()) { + storageId = resource.getStorage().getGcsStorage().getStorageId(); + } else { + throw new Exception("Not Supported storage type"); + } + SecretForStorageGetRequest request = SecretForStorageGetRequest.newBuilder().setStorageId(storageId).build(); + SecretForStorage secretForStorage = getBlockingStorageStub(channel).getSecretForStorage(request); + return secretForStorage; + } + + +} \ No newline at end of file diff --git a/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java new file mode 100644 index 0000000..5a6a86b --- /dev/null +++ b/replica-catalog-api/client/src/test/java/org/apache/airavata/ReplicaCatalogAPIClientTest.java @@ -0,0 +1,242 @@ +package org.apache.airavata; + + +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; +import org.apache.airavata.replicacatalog.resource.stubs.common.FileResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; +import org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage; +import org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretGetRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretWrapper; +import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret; +import org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret; +import org.junit.jupiter.api.Test; + +class ReplicaCatalogAPIClientTest { + + @Test + public void testCase1() throws InterruptedException { + String target = "localhost:6565"; + + /* + --- Sample scenario 1--- + Airavata MFT copied a 100GB astrological data file to Amazon S3 bucket and another copy to GCS location + User can access that S3 bucket data file and Google cloud storage file via Replica catalog details + + 1) Register Data Product in data catalog + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + ManagedChannel channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build(); + try { + ReplicaCatalogAPIClient client = new ReplicaCatalogAPIClient(channel); + + String testUri = "TestUri"; + + DataReplicaLocation replicaLocation = DataReplicaLocation.newBuilder() + .setReplicaName("ASTRO S3") + .setReplicaDescription("S3 replica") + .setDataProductUri(testUri) + .setStorageType(org.apache.airavata.replicacatalog.catalog.stubs.StorageType.S3) + .setCreationTime(System.currentTimeMillis()).build(); + DataReplicaLocation replicaResult = createReplicaLocation(client, replicaLocation); + + S3Storage storage = S3Storage.newBuilder() + .setName("ASTRO S3") + .setBucketName("arn:aws:s3:::mftjayan") + .setRegion("us-east-1") + .setEndpoint("https://s3.us-east-1.amazonaws.com").build(); + + GenericResource resource = GenericResource.newBuilder() + .setReplicaId(replicaResult.getDataReplicaId()) + .setStorage(StorageWrapper.newBuilder().setS3Storage(storage).build()) + .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); + GenericResource resourceResult = createStorage(client, channel, resource); + + S3Secret secret = S3Secret.newBuilder() + .setAccessKey("access") + .setSecretKey("secKey") + .setSessionToken("token").build(); + StorageSecret storageSecret = StorageSecret.newBuilder() + .setSecret(SecretWrapper.newBuilder().setS3Secret(secret).build()) + .setStorageType(StorageType.S3).build(); + SecretCreateRequest secretCreateRequest = SecretCreateRequest.newBuilder().setSecret(storageSecret).build(); + StorageSecret secretResult = createSecret(client, channel, secretCreateRequest); + + SecretForStorage secretForStorage = SecretForStorage.newBuilder() + .setStorageId(resourceResult.getStorage().getS3Storage().getStorageId()) + .setSecretId(secretResult.getSecret().getS3Secret().getSecretId()).setStorageType(StorageType.S3) + .setUserIdentity("userID#12345").build(); + SecretForStorageCreateRequest createRequest = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage).build(); + SecretForStorage secretForStorageResult = createCommonStorage(client, channel, createRequest); + + // GCS replica + DataReplicaLocation replicaLocation2 = DataReplicaLocation.newBuilder() + .setReplicaName("ASTRO GCS") + .setReplicaDescription("GCS replica") + .setDataProductUri(testUri) + .setStorageType(org.apache.airavata.replicacatalog.catalog.stubs.StorageType.GCS) + .setCreationTime(System.currentTimeMillis()).build(); + DataReplicaLocation replicaResult2 = createReplicaLocation(client, replicaLocation2); + + GCSStorage gcsStorage = GCSStorage.newBuilder() + .setName("ASTRO GCS") + .setBucketName("mftjayan").build(); + + GenericResource resource2 = GenericResource.newBuilder() + .setReplicaId(replicaResult2.getDataReplicaId()) + .setStorage(StorageWrapper.newBuilder().setGcsStorage(gcsStorage).build()) + .setFile(FileResource.newBuilder().setResourcePath("/astro.zip").build()).build(); + GenericResource resourceResult2 = createStorage(client, channel, resource2); + + GCSSecret gcsSecret = GCSSecret.newBuilder() + .setClientEmail("accessEmail@gmail.com") + .setPrivateKey("secKey") + .setProjectId("1258").build(); + StorageSecret storageSecret2 = StorageSecret.newBuilder() + .setSecret(SecretWrapper.newBuilder().setGcsSecret(gcsSecret).build()) + .setStorageType(StorageType.GCS).build(); + SecretCreateRequest secretCreateRequest2 = SecretCreateRequest.newBuilder().setSecret(storageSecret2).build(); + StorageSecret secretResult2 = createSecret(client, channel, secretCreateRequest2); + + SecretForStorage secretForStorage2 = SecretForStorage.newBuilder() + .setStorageId(resourceResult2.getStorage().getGcsStorage().getStorageId()) + .setSecretId(secretResult2.getSecret().getGcsSecret().getSecretId()) + .setStorageType(StorageType.GCS) + .setUserIdentity("userID#12345").build(); + SecretForStorageCreateRequest createRequest2 = SecretForStorageCreateRequest.newBuilder().setSecretForStorage(secretForStorage2).build(); + SecretForStorage secretForStorageResult2 = createCommonStorage(client, channel, createRequest2); + + + System.out.println( + MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}] in S3", + (Object[]) new String[]{replicaResult.getDataReplicaId(), secretForStorageResult.getStorageId(), secretForStorageResult.getSecretId()})); + + System.out.println( + MessageFormat.format("Created data replica with id [{0}], Storage id [{1}], Secret id [{2}] in GCS", + (Object[]) new String[]{replicaResult2.getDataReplicaId(), secretForStorageResult2.getStorageId(), secretForStorageResult2.getSecretId()})); + + + AllDataReplicaGetRequest replicasGetRequest = AllDataReplicaGetRequest.newBuilder().setDataProductUri(testUri).build(); + AllDataReplicaGetResponse replicas = client.getReplicaServiceAPI().getAllReplicaLocation(replicasGetRequest); + System.out.println(MessageFormat.format("Load replicas with product url [{0}] ", (Object[]) new String[]{testUri})); + + replicas.getReplicaListList().forEach(r -> { + + DataReplicaGetRequest replicaGetRequest = DataReplicaGetRequest.newBuilder().setDataReplicaId(r.getDataReplicaId()).build(); + DataReplicaGetResponse replicaResponse = client.getReplicaServiceAPI().getReplicaLocation(replicaGetRequest); + if (replicaResponse != null) { + SecretForStorage secretForStorage1 = null; + try { + secretForStorage1 = getStorageSecretIds(client, channel, replicaResponse.getDataReplica().getDataReplicaId()); + SecretGetRequest secretGetRequest = SecretGetRequest.newBuilder().setSecretId(secretForStorage1.getSecretId()).setStorageType(secretForStorage1.getStorageType()).build(); + StorageSecret secretLoad = client.getBlockingSecretStub(channel).getSecret(secretGetRequest); + System.out.println(secretLoad); + + + GenericResourceGetRequest genericResourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaResponse.getDataReplica().getDataReplicaId()).build(); + GenericResource resource1 = client.getBlockingStorageStub(channel).getGenericResource(genericResourceGetRequest); + System.out.println(resource1); + + } catch (Exception e) { + throw new RuntimeException(e); + } + if (secretForStorage1 != null) { + System.out.println( + MessageFormat.format("Loaded data replica with id [{0}], Storage id [{1}], Secret id [{2}] Storage Type [{3}]", + (Object[]) new String[]{replicaResponse.getDataReplica().getDataReplicaId(), secretForStorage1.getStorageId(), secretForStorage1.getSecretId(), + secretForStorage1.getStorageType().name()})); + + } + } + } + + ); + + + /* + --- Sample scenario 2--- + Airavata MFT copied a 1000GB astrological data Folder which contains 100 data files to Amazon S3 bucket + User can access that S3 bucket data folder via Replica catalog details + + 1) Register Data Product + 2) Register Replica Location + 3) Register Location storage details + 4) Register location storage credentials + + */ + }catch (Exception e){ + + } finally { + channel.shutdownNow().awaitTermination(1800, TimeUnit.SECONDS); + } + } + + public DataReplicaLocation createReplicaLocation(ReplicaCatalogAPIClient client, DataReplicaLocation replicaLocation) throws Exception { + DataReplicaCreateRequest request = DataReplicaCreateRequest.newBuilder().setDataReplica(replicaLocation).build(); + DataReplicaCreateResponse response = client.getReplicaServiceAPI().registerReplicaLocation(request); + if(response.hasError() && !response.getError().getMessage().isBlank()){ + throw new Exception(response.getError().getMessage()); + } + return response.getDataReplica(); + } + + + public SecretForStorage createCommonStorage(ReplicaCatalogAPIClient client, Channel channel, SecretForStorageCreateRequest createRequest) { + + SecretForStorage response = client.getBlockingStorageStub(channel).registerSecretForStorage(createRequest); + return response; + } + + public GenericResource createStorage(ReplicaCatalogAPIClient client, Channel channel, GenericResource resource) { + GenericResourceCreateRequest request = GenericResourceCreateRequest.newBuilder().setResource(resource).build(); + GenericResource response = client.getBlockingStorageStub(channel).createGenericResource(request); + return response; + } + + public StorageSecret createSecret(ReplicaCatalogAPIClient client, Channel channel, SecretCreateRequest createRequest) { + StorageSecret response = client.getBlockingSecretStub(channel).registerSecret(createRequest); + return response; + } + + public SecretForStorage getStorageSecretIds(ReplicaCatalogAPIClient client, Channel channel, String replicaId) throws Exception { + GenericResourceGetRequest resourceGetRequest = GenericResourceGetRequest.newBuilder().setReplicaId(replicaId).build(); + GenericResource resource = client.getBlockingStorageStub(channel).getGenericResource(resourceGetRequest); + String storageId = null; + if (resource.getStorage().hasS3Storage()) { + storageId = resource.getStorage().getS3Storage().getStorageId(); + } else if (resource.getStorage().hasGcsStorage()) { + storageId = resource.getStorage().getGcsStorage().getStorageId(); + } else { + throw new Exception("Not Supported storage type"); + } + SecretForStorageGetRequest request = SecretForStorageGetRequest.newBuilder().setStorageId(storageId).build(); + SecretForStorage secretForStorage = client.getBlockingStorageStub(channel).getSecretForStorage(request); + return secretForStorage; + } + + +} \ No newline at end of file diff --git a/replica-catalog-api/pom.xml b/replica-catalog-api/pom.xml new file mode 100644 index 0000000..32f2816 --- /dev/null +++ b/replica-catalog-api/pom.xml @@ -0,0 +1,20 @@ + + + 4.0.0 + + org.apache.airavata + airavata-data-catalog + 0.1-SNAPSHOT + + + replica-catalog-api + pom + + stubs + server + client + + + \ No newline at end of file diff --git a/replica-catalog-api/server/pom.xml b/replica-catalog-api/server/pom.xml new file mode 100644 index 0000000..65c67c0 --- /dev/null +++ b/replica-catalog-api/server/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + + org.apache.airavata + replica-catalog-api + 0.1-SNAPSHOT + + + replica-catalog-api-server + + + + + org.springframework.boot + spring-boot-starter-parent + ${spring.boot.version} + pom + import + + + + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + io.github.lognet + grpc-spring-boot-starter + ${grpc.spring.boot} + + + org.springframework.boot + spring-boot-starter + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.postgresql + postgresql + + + org.apache.airavata + replica-catalog-api-stubs + 0.1-SNAPSHOT + + + io.hypersistence + hypersistence-utils-hibernate-60 + ${hypersistence.60.version} + + + com.fasterxml.jackson.module + jackson-module-jakarta-xmlbind-annotations + ${jackson.jakarta.version} + + + net.sf.dozer + dozer + ${dozer.version} + + + org.junit.jupiter + junit-jupiter-engine + ${jupiter.version} + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + + + + \ No newline at end of file diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java new file mode 100644 index 0000000..f66e4c2 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/ReplicaCatalogApiServiceApplication.java @@ -0,0 +1,13 @@ +package org.apache.airavata.replicacatalog; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ReplicaCatalogApiServiceApplication { + + public static void main(String[] args) { + SpringApplication.run(ReplicaCatalogApiServiceApplication.class, args); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java new file mode 100644 index 0000000..94b1012 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/mapper/DataReplicaMapper.java @@ -0,0 +1,47 @@ +package org.apache.airavata.replicacatalog.catalogapi.mapper; + +import java.sql.Timestamp; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link DataReplicaLocation} + */ +@Component +public class DataReplicaMapper { + + @Autowired + DataReplicaLocationRepository replicaLocationRepository; + + public void mapModelToEntity(DataReplicaLocation model, DataReplicaLocationEntity entity) { + + entity.setReplicaName(model.getReplicaName()); + entity.setReplicaId(model.getDataReplicaId()); + entity.setProductUri(model.getDataProductUri()); + entity.setReplicaDescription(model.getReplicaDescription()); + entity.setCreationTime(new Timestamp(System.currentTimeMillis())); + // TODO: handle parent data product not found +// entity.setDataProduct(parentDataEntity); + + } + + public void mapEntityToModel(DataReplicaLocationEntity dataReplicaLocationEntity, DataReplicaLocation.Builder dataProductBuilder) { + + dataProductBuilder + .setDataReplicaId( dataReplicaLocationEntity.getReplicaId() ) + .setReplicaName( dataReplicaLocationEntity.getReplicaName() ) + .setDataProductUri( dataReplicaLocationEntity.getProductUri() ) + .setCreationTime( dataReplicaLocationEntity.getCreationTime().getTime() ); + +// if ( dataReplicaLocationEntity.getDataProduct() != null ) +// { +// dataProductBuilder.setDataProductId( dataReplicaLocationEntity.getDataProduct().getProductUri() ); +// } + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java new file mode 100644 index 0000000..b6b0fea --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaLocationEntity.java @@ -0,0 +1,151 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import jakarta.persistence.*; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Map; + +/** + * The persistent class for the data_replica_location database table. + */ +@Entity +@Table(name = "DATA_REPLICA_LOCATION") +public class DataReplicaLocationEntity implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "REPLICA_ID") + private String replicaId; + + @Column(name = "PRODUCT_URI") + private String productUri; + + @Column(name = "REPLICA_NAME") + private String replicaName; + + @Column(name = "REPLICA_DESCRIPTION") + private String replicaDescription; + + @Column(name = "STORAGE_RESOURCE_ID") + private String storageResourceId; + + @Column(name = "FILE_PATH") + private String filePath; + + @Column(name = "CREATION_TIME") + private Timestamp creationTime; + + @Column(name = "LAST_MODIFIED_TIME") + private Timestamp lastModifiedTime; + + @Column(name = "VALID_UNTIL_TIME") + private Timestamp validUntilTime; + + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "DATA_REPLICA_METADATA", joinColumns = @JoinColumn(name = "REPLICA_ID")) + @MapKeyColumn(name = "METADATA_KEY") + @Column(name = "METADATA_VALUE") + private Map replicaMetadata; + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getProductUri() { + return productUri; + } + + public void setProductUri(String productUri) { + this.productUri = productUri; + } + + public String getReplicaName() { + return replicaName; + } + + public void setReplicaName(String replicaName) { + this.replicaName = replicaName; + } + + public String getReplicaDescription() { + return replicaDescription; + } + + public void setReplicaDescription(String replicaDescription) { + this.replicaDescription = replicaDescription; + } + + public String getStorageResourceId() { + return storageResourceId; + } + + public void setStorageResourceId(String storageResourceId) { + this.storageResourceId = storageResourceId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public Timestamp getCreationTime() { + return creationTime; + } + + public void setCreationTime(Timestamp creationTime) { + this.creationTime = creationTime; + } + + public Timestamp getLastModifiedTime() { + return lastModifiedTime; + } + + public void setLastModifiedTime(Timestamp lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + public Timestamp getValidUntilTime() { + return validUntilTime; + } + + public void setValidUntilTime(Timestamp validUntilTime) { + this.validUntilTime = validUntilTime; + } + + public Map getReplicaMetadata() { + return replicaMetadata; + } + + public void setReplicaMetadata(Map replicaMetadata) { + this.replicaMetadata = replicaMetadata; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java new file mode 100644 index 0000000..cef6df4 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataEntity.java @@ -0,0 +1,71 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import jakarta.persistence.*; + +import java.io.Serializable; + +/** + * The persistent class for the data_replica_metadata database table. + */ +@Entity +@Table(name = "DATA_REPLICA_METADATA") +@IdClass(DataReplicaMetadataPK.class) +public class DataReplicaMetadataEntity implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "REPLICA_ID") + private String replicaId; + + @Id + @Column(name = "METADATA_KEY") + private String metadataKey; + + @Column(name = "METADATA_VALUE") + private String metadataValue; + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getMetadataKey() { + return metadataKey; + } + + public void setMetadataKey(String metadataKey) { + this.metadataKey = metadataKey; + } + + public String getMetadataValue() { + return metadataValue; + } + + public void setMetadataValue(String metadataValue) { + this.metadataValue = metadataValue; + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java new file mode 100644 index 0000000..0fa59c4 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/model/DataReplicaMetadataPK.java @@ -0,0 +1,74 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.model; + +import java.io.Serializable; + +/** + * The primary key class for the data_replica_metadata database table. + */ +public class DataReplicaMetadataPK implements Serializable { + private static final long serialVersionUID = 1L; + + private String replicaId; + private String metadataKey; + + public DataReplicaMetadataPK() { + } + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getMetadataKey() { + return metadataKey; + } + + public void setMetadataKey(String metadataKey) { + this.metadataKey = metadataKey; + } + + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof DataReplicaMetadataPK)) { + return false; + } + DataReplicaMetadataPK castOther = (DataReplicaMetadataPK) other; + return + this.replicaId.equals(castOther.replicaId) + && this.metadataKey.equals(castOther.metadataKey); + } + + public int hashCode() { + final int prime = 31; + int hash = 17; + hash = hash * prime + this.replicaId.hashCode(); + hash = hash * prime + this.metadataKey.hashCode(); + + return hash; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java new file mode 100644 index 0000000..29ab52a --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/repository/DataReplicaLocationRepository.java @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.replicacatalog.catalogapi.repository; + +import java.util.List; +import java.util.Optional; + +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.transaction.annotation.Transactional; + +@Transactional(readOnly = true) +public interface DataReplicaLocationRepository extends JpaRepository { + + DataReplicaLocationEntity findByReplicaId(String replicaId); + + Optional> findByProductUri(String replicaId); + + @Transactional + void deleteByReplicaId(String replicaId); + + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java new file mode 100644 index 0000000..93a5bdc --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/IReplicaCatalogService.java @@ -0,0 +1,20 @@ +package org.apache.airavata.replicacatalog.catalogapi.service; + +import java.util.List; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.ReplicaGroupEntry; + +public interface IReplicaCatalogService { + + DataReplicaLocation createDataReplica(DataReplicaLocation dataReplica) throws Exception; + + DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplica) throws Exception; + + DataReplicaLocation getDataReplica(String replicaId) throws Exception; + + void deleteDataReplica(String replicaId) throws Exception; + + List getDataReplicas(String productUri) throws Exception; + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java new file mode 100644 index 0000000..b9ae1ee --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIService.java @@ -0,0 +1,146 @@ +package org.apache.airavata.replicacatalog.catalogapi.service; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.catalog.service.ReplicaCatalogAPIServiceGrpc; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaDeleteRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaDeleteResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.AllDataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaCreateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaDeleteRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaDeleteResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaGetResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateRequest; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaUpdateResponse; +import org.apache.airavata.replicacatalog.catalog.stubs.Error; +import org.apache.airavata.replicacatalog.catalog.stubs.ErrorCode; +import org.apache.airavata.replicacatalog.catalog.stubs.ReplicaGroupEntry; +import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; +import org.apache.airavata.replicacatalog.exception.InvalidDataException; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * API level services + */ +@GRpcService +public class ReplicaCatalogAPIService extends ReplicaCatalogAPIServiceGrpc.ReplicaCatalogAPIServiceImplBase { + private static final Logger logger = LoggerFactory.getLogger(ReplicaCatalogAPIService.class); + + @Autowired + DataReplicaMapper replicaMapper = new DataReplicaMapper(); + + @Autowired + IReplicaCatalogService catalogService; + + @Override + public void registerReplicaLocation(DataReplicaCreateRequest request, StreamObserver responseObserver) { + + logger.info("Creating Replica for a data product {}", request.getDataReplica()); + Error.Builder errorBuilder = Error.newBuilder(); + DataReplicaLocation result = DataReplicaLocation.newBuilder().build(); + if (!request.hasDataReplica()) { + logger.debug("No Data Replica Location"); + errorBuilder.setMessage("Replica details not available in the DataReplicaCreateRequest") + .setCode(ErrorCode.INVALID_DATA).build(); + } else if (request.getDataReplica().getDataProductUri().isBlank()){ + logger.debug("No data product uri for the replica location"); + errorBuilder.setMessage("Data product uri not available in the replica location in the DataReplicaCreateRequest") + .setCode(ErrorCode.INVALID_DATA).build(); + }else if (request.getDataReplica().getStorageType().getNumber()==0){ + logger.debug("No storage type for the replica location"); + errorBuilder.setMessage("Storage type not available in the replica location in the DataReplicaCreateRequest") + .setCode(ErrorCode.INVALID_DATA).build(); + } + else { + try { + result = catalogService.createDataReplica(request.getDataReplica()); + } catch (Exception e) { + logger.error("Error in creating Replica location",e); + } + } + DataReplicaCreateResponse.Builder responseBuilder = DataReplicaCreateResponse.newBuilder(); + responseBuilder.setDataReplica(result); + responseBuilder.setError(errorBuilder.build()); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void updateReplicaLocation(DataReplicaUpdateRequest request, StreamObserver responseObserver) { + + DataReplicaLocation result = null; + try { + result = catalogService.updateDataReplica(request.getDataReplica()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DataReplicaUpdateResponse.Builder responseBuilder = DataReplicaUpdateResponse.newBuilder(); + responseBuilder.setDataReplica(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getReplicaLocation(DataReplicaGetRequest request, StreamObserver responseObserver) { + + logger.info("Loading Replica for a Replica ID : {}", request.getDataReplicaId()); + + DataReplicaLocation result = null; + try { + result = catalogService.getDataReplica(request.getDataReplicaId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DataReplicaGetResponse.Builder responseBuilder = DataReplicaGetResponse.newBuilder(); + responseBuilder.setDataReplica(result); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void removeReplicaLocation(DataReplicaDeleteRequest request, StreamObserver responseObserver) { + try { + catalogService.deleteDataReplica(request.getDataReplicaId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(DataReplicaDeleteResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + + @Override + public void getAllReplicaLocation(AllDataReplicaGetRequest request, StreamObserver responseObserver) { + List replicaLocations = null; + try { + replicaLocations = catalogService.getDataReplicas(request.getDataProductUri()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + AllDataReplicaGetResponse.Builder responseBuilder = AllDataReplicaGetResponse.newBuilder(); + replicaLocations.forEach(entry -> { + responseBuilder.setDataProductUri(request.getDataProductUri()).addReplicaList( entry); + }); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + + } + + @Override + public void removeAllReplicaLocation(AllDataReplicaDeleteRequest request, StreamObserver responseObserver) { + super.removeAllReplicaLocation(request, responseObserver); + } + + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java new file mode 100644 index 0000000..e581592 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/catalogapi/service/impl/ReplicaCatalogServiceImp.java @@ -0,0 +1,104 @@ +package org.apache.airavata.replicacatalog.catalogapi.service.impl; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import jakarta.transaction.Transactional; +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.ReplicaGroupEntry; +import org.apache.airavata.replicacatalog.catalogapi.mapper.DataReplicaMapper; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.catalogapi.repository.DataReplicaLocationRepository; +import org.apache.airavata.replicacatalog.catalogapi.service.IReplicaCatalogService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@Transactional +public class ReplicaCatalogServiceImp implements IReplicaCatalogService { + private final static Logger logger = LoggerFactory.getLogger(ReplicaCatalogServiceImp.class); + + @Autowired + DataReplicaLocationRepository dataReplicaLocationRepository; + + @Autowired + DataReplicaMapper replicaMapper = new DataReplicaMapper(); + + + + @Override + public DataReplicaLocation createDataReplica(DataReplicaLocation replicaLocation) throws Exception{ + + DataReplicaLocationEntity dataReplicaLocationEntity = new DataReplicaLocationEntity(); + + if (replicaLocation != null && replicaLocation.getDataReplicaId() != null) { + dataReplicaLocationEntity.setReplicaId(replicaLocation.getDataReplicaId()); + } + replicaMapper.mapModelToEntity(replicaLocation, dataReplicaLocationEntity); + if (dataReplicaLocationEntity.getReplicaId() == null || dataReplicaLocationEntity.getReplicaId().isEmpty()) { + dataReplicaLocationEntity.setReplicaId(UUID.randomUUID().toString()); + } + DataReplicaLocationEntity savedDataReplicaLocationEntity = dataReplicaLocationRepository.save(dataReplicaLocationEntity); + return toDataReplicaLocation(savedDataReplicaLocationEntity); + + } + + @Override + public DataReplicaLocation updateDataReplica(DataReplicaLocation dataReplicaLocation) throws Exception{ + + DataReplicaLocationEntity dataReplicaLocationEntity = dataReplicaLocationRepository.findByReplicaId(dataReplicaLocation.getDataReplicaId()); + if (dataReplicaLocationEntity == null) { + logger.debug("Data Replica Location not exists"); + } + replicaMapper.mapModelToEntity(dataReplicaLocation, dataReplicaLocationEntity); + DataReplicaLocationEntity savedDataReplicaLocationEntity = dataReplicaLocationRepository.save(dataReplicaLocationEntity); + return toDataReplicaLocation(savedDataReplicaLocationEntity); + } + + @Override + public DataReplicaLocation getDataReplica(String replicaId) throws Exception { + + DataReplicaLocationEntity dataReplicaLocationEntity = dataReplicaLocationRepository.findByReplicaId(replicaId); + if (dataReplicaLocationEntity == null) { + logger.debug("Data Replica Location not exists"); + } + return toDataReplicaLocation(dataReplicaLocationEntity); + } + + @Override + public void deleteDataReplica(String replicaId) throws Exception { + dataReplicaLocationRepository.deleteByReplicaId(replicaId); + } + + @Override + public List getDataReplicas(String productUri) throws Exception{ + Optional> dataReplicaLocationEntities = dataReplicaLocationRepository.findByProductUri(productUri); + if (!dataReplicaLocationEntities.isPresent() || dataReplicaLocationEntities.get().isEmpty()) { + logger.debug("Data Replica Location not exists"); + } + List dataReplicaLocations = new ArrayList<>(); + dataReplicaLocationEntities.get().forEach(r -> { + ReplicaGroupEntry groupEntry = null; + try { + groupEntry = ReplicaGroupEntry.newBuilder() + .setDataReplicaId(r.getReplicaId()) + .addFiles(toDataReplicaLocation(r)).build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + dataReplicaLocations.add(groupEntry); + }); + return dataReplicaLocations; + } + private DataReplicaLocation toDataReplicaLocation(DataReplicaLocationEntity savedDataLocationEntity) throws Exception{ + DataReplicaLocation.Builder builder = DataReplicaLocation.newBuilder(); + replicaMapper.mapEntityToModel(savedDataLocationEntity, builder); + return builder.build(); + } + + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java new file mode 100644 index 0000000..0acc2d3 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/EntityNotFoundException.java @@ -0,0 +1,25 @@ +package org.apache.airavata.replicacatalog.exception; + +public class EntityNotFoundException extends RuntimeException { + + public EntityNotFoundException() { + } + + public EntityNotFoundException(String message) { + super(message); + } + + public EntityNotFoundException(Throwable cause) { + super(cause); + } + + public EntityNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java new file mode 100644 index 0000000..4d23073 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/InvalidDataException.java @@ -0,0 +1,9 @@ +package org.apache.airavata.replicacatalog.exception; + +public class InvalidDataException extends Exception { + + public InvalidDataException(String message) { + super(message); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java new file mode 100644 index 0000000..5ddde9e --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/exception/StorageNotSupportException.java @@ -0,0 +1,9 @@ +package org.apache.airavata.replicacatalog.exception; + +public class StorageNotSupportException extends RuntimeException { + + public StorageNotSupportException(String message) { + super(message); + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java new file mode 100644 index 0000000..7f95cb5 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/mapper/ResourceStorageMapper.java @@ -0,0 +1,87 @@ +package org.apache.airavata.replicacatalog.resource.mapper; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageWrapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link DataReplicaLocation} + */ +@Component +public class ResourceStorageMapper { + + @Autowired + GenericResourceRepository genericResourceRepository; + + public void mapGenericStorageModelToEntity(GenericResource storage, GenericResourceEntity resourceEntity) throws Exception { + + resourceEntity.setReplicaId(storage.getReplicaId()); + resourceEntity.setStorageId(storage.getResourceId()); + resourceEntity.setStorageType(resolveStorage(storage.getStorage())); + resourceEntity.setResourcePath(storage.getFile().getResourcePath()); + resourceEntity.setResourceType(GenericResourceEntity.ResourceType.FILE); + // TODO + } + + public void mapGenericStorageEntityToModel(GenericResourceEntity resourceEntity, StorageWrapper wrapper, GenericResource.Builder builder) throws Exception { + + builder.setResourceId(resourceEntity.getResourceId()); + builder.setReplicaId(resourceEntity.getReplicaId()); + if (wrapper != null) { + builder.setStorage(wrapper); + } + } + + public void mapStorageSecretModelToEntity(SecretForStorage storage, StorageSecretEntity resourceEntity) { + resourceEntity.setStorageId(storage.getStorageId()); + resourceEntity.setSecretId(storage.getSecretId()); + resourceEntity.setType(storage.getStorageType().name()); + // TODO + } + + public void mapStorageSecretEntityToModel(StorageSecretEntity resourceEntity, SecretForStorage.Builder dataProductBuilder) { + + dataProductBuilder.setStorageType(StorageType.valueOf(resourceEntity.getType())).setSecretId(resourceEntity.getSecretId()) + .setStorageId(resourceEntity.getStorageId()); + + } + + + protected GenericResourceEntity.StorageType resolveStorage (StorageWrapper wrapper){ + if(wrapper.hasS3Storage()){ + return GenericResourceEntity.StorageType.S3; + } else if (wrapper.hasGcsStorage()){ + return GenericResourceEntity.StorageType.GCS; + } else if (wrapper.hasFtpStorage()) { + return GenericResourceEntity.StorageType.FTP; + } + + return null; + } + + public StorageType resolveStorage(String type){ + GenericResourceEntity.StorageType storageType = GenericResourceEntity.StorageType.valueOf(type); + switch (storageType) { + case S3: + return StorageType.S3; + case GCS: + return StorageType.GCS; + case FTP: + return StorageType.FTP; + default: + throw new RuntimeException( + "Unexpected storage type: " + type); + } + } + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java new file mode 100644 index 0000000..091bd71 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/GenericResourceEntity.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + + +@Entity +@Table(name = "REPLICA_RESOURCE") +public class GenericResourceEntity { + + public enum ResourceType { + FILE, DIRECTORY; + } + + public enum StorageType { + S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT, ODATA; + } + + @Id + @Column(name = "RESOURCE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String resourceId; + + @Column(name = "RESOURCE_PATH") + private String resourcePath; + + @Column(name = "RESOURCE_TYPE") + private ResourceType resourceType; + + @Column(name = "REPLICA_ID") + private String replicaId; + + @Column(name = "STORAGE_ID") + private String storageId; + + @Column(name = "STORAGE_TYPE") + private StorageType storageType; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getResourcePath() { + return resourcePath; + } + + public void setResourcePath(String resourcePath) { + this.resourcePath = resourcePath; + } + + public ResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(ResourceType resourceType) { + this.resourceType = resourceType; + } + + public String getReplicaId() { + return replicaId; + } + + public void setReplicaId(String replicaId) { + this.replicaId = replicaId; + } + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java new file mode 100644 index 0000000..b0a1574 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/ResolveStorageEntity.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + + +@Entity +@Table(name = "REPLICA_STORAGE") +public class ResolveStorageEntity { + + public enum StorageType { + S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT, ODATA; + } + + @Id + @Column(name = "STORAGE_ID") + private String storageId; + + @Column(name = "STORAGE_TYPE") + private StorageType storageType; + + @Column(name = "STORAGE_NAME") + private String storageName; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + public String getStorageName() { + return storageName; + } + + public void setStorageName(String storageName) { + this.storageName = storageName; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java new file mode 100644 index 0000000..c01c185 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/StorageSecretEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + +@Entity +@Table(name = "STORAGE_SECRET") +public class StorageSecretEntity { + @Id + @Column(name = "STORAGE_SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String id; + + @Column(name = "STORAGE_ID") + private String storageId; + + @Column(name = "SECRET_ID") + private String secretId; + + @Column(name = "STORAGE_TYPE") + private String type; + + @Column(name = "USER_IDENTIFIER") + private String userIdentifier; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getUserIdentifier() { + return userIdentifier; + } + + public void setUserIdentifier(String userIdentifier) { + this.userIdentifier = userIdentifier; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java new file mode 100644 index 0000000..4130115 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/FTPStorageEntity.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "FTP_STORAGE") +public class FTPStorageEntity { + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "HOST") + private String host; + + @Column(name = "PORT") + private int port; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java new file mode 100644 index 0000000..569fcd6 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/GCSStorageEntity.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "GCS_STORAGE") +public class GCSStorageEntity { + + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator( name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "BUCKET_NAME") + private String bucketName; + + @Column(name = "STORAGE_NAME") + private String name; + + public String getStorageId() + { + return storageId; + } + + public void setStorageId( String storageId ) + { + this.storageId = storageId; + } + + public String getBucketName() + { + return bucketName; + } + + public void setBucketName( String bucketName ) + { + this.bucketName = bucketName; + } + + public String getName() + { + return name; + } + + public void setName( String name ) + { + this.name = name; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java new file mode 100644 index 0000000..1cd5fb6 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/LocalStorageEntity.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "LOCAL_STORAGE") +public class LocalStorageEntity { + + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "AGENT_ID") + private String agentId; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAgentId() { + return agentId; + } + + public void setAgentId(String agentId) { + this.agentId = agentId; + } +} \ No newline at end of file diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/S3StorageEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/S3StorageEntity.java new file mode 100644 index 0000000..c6b3b7b --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/model/storage/S3StorageEntity.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.model.storage; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "S3_STORAGE") +public class S3StorageEntity { + + @Id + @Column(name = "STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "STORAGE_NAME") + private String name; + + @Column(name = "BUCKET_NAME") + private String bucketName; + + @Column(name = "REGION") + private String region; + + @Column(name = "ENDPOINT") + private String endpoint; + + @Column(name = "USE_TLS") + private boolean useTLS; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public boolean isUseTLS() { + return useTLS; + } + + public void setUseTLS(boolean useTLS) { + this.useTLS = useTLS; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java new file mode 100644 index 0000000..8c3ab5c --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/GenericResourceRepository.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface GenericResourceRepository extends CrudRepository { + Optional findByResourceId(String resourceId); + Optional findByReplicaId(String replicaId); + void deleteByStorageIdAndStorageType(String storageId, GenericResourceEntity.StorageType storageType); + Optional deleteByResourceId(String resourceId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java new file mode 100644 index 0000000..c04e019 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/ResolveStorageRepository.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.ResolveStorageEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface ResolveStorageRepository extends CrudRepository { + Optional getByStorageId(String storageID); + List getByStorageName(String storageName); + List getByStorageType(ResolveStorageEntity.StorageType storageType); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java new file mode 100644 index 0000000..f87e761 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/StorageSecretRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository; + +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface StorageSecretRepository extends CrudRepository { + + Optional findByStorageId(String storageId); + void deleteByStorageId(String resourceId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java new file mode 100644 index 0000000..94ea319 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/FTPStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import java.util.List; + +import org.apache.airavata.replicacatalog.resource.model.storage.FTPStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface FTPStorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java new file mode 100644 index 0000000..9288750 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/GCSStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import java.util.List; + +import org.apache.airavata.replicacatalog.resource.model.storage.GCSStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface GCSStorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java new file mode 100644 index 0000000..3d1d9f0 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/LocalStorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import java.util.List; + +import org.apache.airavata.replicacatalog.resource.model.storage.LocalStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface LocalStorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/S3StorageRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/S3StorageRepository.java new file mode 100644 index 0000000..aecebb0 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/repository/storage/S3StorageRepository.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.repository.storage; + +import org.apache.airavata.replicacatalog.resource.model.storage.S3StorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface S3StorageRepository extends CrudRepository { + List findAll(Pageable pageable); + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java new file mode 100644 index 0000000..3ffe118 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/IResourceService.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.service; + + + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.resource.stubs.common.*; +import org.apache.airavata.replicacatalog.resource.stubs.s3.*; + +import java.util.Optional; + +public interface IResourceService { + + public void init(); + public void destroy(); + public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception; + + public SecretForStorage registerSecretForStorage(SecretForStorageCreateRequest request) throws Exception; + + public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) throws Exception; + + public StorageListResponse searchStorages(StorageSearchRequest request) throws Exception; + + public StorageListResponse listStorage(StorageListRequest request) throws Exception; + + public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception; + + public GenericResource getGenericResource(GenericResourceGetRequest request) throws Exception; + + public GenericResource updateGenericResource(GenericResourceUpdateRequest request) throws Exception; + + public GenericResource deleteGenericResource(GenericResourceDeleteRequest request) throws Exception; + +// public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception; +// public Optional getS3Storage(S3StorageGetRequest request) throws Exception; +// public S3Storage createS3Storage(S3StorageCreateRequest request) throws Exception; +// public boolean updateS3Storage(S3StorageUpdateRequest request) throws Exception; +// public boolean deleteS3Storage(S3StorageDeleteRequest request) throws Exception; +// + + StorageTypeResolveResponse resolveStorageType(StorageTypeResolveRequest request) throws Exception; +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java new file mode 100644 index 0000000..6c4df13 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/ResourceAPIService.java @@ -0,0 +1,108 @@ +package org.apache.airavata.replicacatalog.resource.service; + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResource; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceDeleteRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceDeleteResponse; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceUpdateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.GenericResourceUpdateResponse; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageCreateRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorageGetRequest; +import org.apache.airavata.replicacatalog.resource.stubs.common.StorageCommonServiceGrpc; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +@GRpcService +public class ResourceAPIService extends StorageCommonServiceGrpc.StorageCommonServiceImplBase { + private static final Logger logger = LoggerFactory.getLogger(ResourceAPIService.class); + + + @Autowired + IResourceService resourceService; + + @Override + public void createGenericResource(GenericResourceCreateRequest request, StreamObserver responseObserver) { + logger.info("Creating Storage {}", request.getStorageId()); + GenericResource resource = null; + try { + resource = resourceService.createGenericResource(request); + } catch (Exception e) { + logger.error("Error {0}", e); + } + + responseObserver.onNext(resource); + responseObserver.onCompleted(); + } + + @Override + public void getGenericResource(GenericResourceGetRequest request, StreamObserver responseObserver) { + GenericResource genericResource = null; + try { + genericResource = resourceService.getGenericResource(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(genericResource); + responseObserver.onCompleted(); + } + + @Override + public void updateGenericResource(GenericResourceUpdateRequest request, StreamObserver responseObserver) { + + GenericResource genericResource = null; + try { + genericResource = resourceService.updateGenericResource(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + + GenericResourceUpdateResponse.Builder responseBuilder = GenericResourceUpdateResponse.newBuilder(); + responseBuilder.setResourceId(genericResource.getResourceId()); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void deleteGenericResource(GenericResourceDeleteRequest request, StreamObserver responseObserver) { + GenericResource genericResource = null; + try { + genericResource = resourceService.deleteGenericResource(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + + GenericResourceDeleteResponse.Builder responseBuilder = GenericResourceDeleteResponse.newBuilder(); + responseBuilder.setStatus(genericResource != null); + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void registerSecretForStorage(SecretForStorageCreateRequest request, StreamObserver responseObserver) { + SecretForStorage secretForStorage = null; + try { + secretForStorage = resourceService.registerSecretForStorage(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(secretForStorage); + responseObserver.onCompleted(); + } + + @Override + public void getSecretForStorage(SecretForStorageGetRequest request, StreamObserver responseObserver) { + SecretForStorage secretForStorage = null; + try { + secretForStorage = resourceService.getSecretForStorage(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(secretForStorage); + responseObserver.onCompleted(); + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java new file mode 100644 index 0000000..7f6e127 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/resource/service/SQLResourceService.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.resource.service; + + +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; +import org.apache.airavata.replicacatalog.exception.InvalidDataException; +import org.apache.airavata.replicacatalog.exception.StorageNotSupportException; +import org.apache.airavata.replicacatalog.resource.mapper.ResourceStorageMapper; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.model.ResolveStorageEntity; +import org.apache.airavata.replicacatalog.resource.model.storage.GCSStorageEntity; +import org.apache.airavata.replicacatalog.resource.model.storage.S3StorageEntity; +import org.apache.airavata.replicacatalog.resource.model.StorageSecretEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.repository.ResolveStorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.storage.GCSStorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.storage.LocalStorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.storage.S3StorageRepository; +import org.apache.airavata.replicacatalog.resource.repository.StorageSecretRepository; + +import org.apache.airavata.replicacatalog.resource.stubs.common.*; +import org.apache.airavata.replicacatalog.resource.stubs.common.Error; +import org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage; +import org.apache.airavata.replicacatalog.resource.stubs.s3.*; +import org.dozer.DozerBeanMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Component("SQLResourceBackend") +public class SQLResourceService implements IResourceService { + + private static final Logger logger = LoggerFactory.getLogger(SQLResourceService.class); + private DozerBeanMapper mapper = new DozerBeanMapper(); + @Autowired + private GenericResourceRepository resourceRepository; + + @Autowired + private S3StorageRepository s3StorageRepository; + + @Autowired + private GCSStorageRepository gcsStorageRepository; + + @Autowired + private LocalStorageRepository localStorageRepository; + + @Autowired + private StorageSecretRepository resourceSecretRepository; + + @Autowired + private ResolveStorageRepository resolveStorageRepository; + + @Autowired + ResourceStorageMapper resourceStorageMapper = new ResourceStorageMapper(); + + @Override + public void init() { + logger.info("Initializing database resource backend"); + } + + @Override + public void destroy() { + logger.info("Destroying database resource backend"); + } + + @Override + public GenericResource createGenericResource(GenericResourceCreateRequest request) throws Exception { + + StorageType storageType = null; + switch (request.getResource().getStorage().getStorageCase().getNumber()) { + case StorageWrapper.S3STORAGE_FIELD_NUMBER: + storageType = StorageType.S3; + break; + case StorageWrapper.LOCALSTORAGE_FIELD_NUMBER: + storageType = StorageType.LOCAL; + break; + case StorageWrapper.GCSSTORAGE_FIELD_NUMBER: + storageType = StorageType.GCS; + break; + case StorageWrapper.FTPSTORAGE_FIELD_NUMBER: + storageType = StorageType.FTP; + break; + default: + break; + } + + + GenericResourceEntity resourceEntity = new GenericResourceEntity(); + String storageId = ""; + StorageWrapper wrapper = null; + if (storageType == StorageType.S3) { + S3Storage storage = createS3Storage(request.getResource().getStorage().getS3Storage()); + if (storage != null) { + storageId = storage.getStorageId(); + wrapper = StorageWrapper.newBuilder().setS3Storage(storage).build(); + } + + } else if (storageType == StorageType.GCS) { + GCSStorage storage = createGCSStorage(request.getResource().getStorage().getGcsStorage()); + if (storage != null) { + storageId = storage.getStorageId(); + wrapper = StorageWrapper.newBuilder().setGcsStorage(storage).build(); + } + + } else { + throw new StorageNotSupportException( "Storage type not supported/implemented"); + } + + if (wrapper == null) { + throw new InvalidDataException("Storage not created."); + } + + resourceEntity.setResourceId(UUID.randomUUID().toString()); + resourceStorageMapper.mapGenericStorageModelToEntity(request.getResource(), resourceEntity); + resourceEntity.setStorageId(storageId); + GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); + + GenericResource.Builder responseBuilder = GenericResource.newBuilder(); + resourceStorageMapper.mapGenericStorageEntityToModel(savedDataProductEntity, wrapper, responseBuilder); + return responseBuilder.build(); + } + + @Override + public GenericResource getGenericResource(GenericResourceGetRequest request) throws Exception { + + Optional savedGenericResourceEntityList = Optional.empty(); + if (!request.getReplicaId().isBlank()) { + savedGenericResourceEntityList = resourceRepository.findByReplicaId(request.getReplicaId()); + } else if (!request.getResourceId().isBlank()) { + savedGenericResourceEntityList = resourceRepository.findByResourceId(request.getResourceId()); + } + if (savedGenericResourceEntityList.isEmpty()) { + return null; + } + GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(); + StorageWrapper wrapper = null; + if (StorageType.S3.name().equals(resourceEntity.getStorageType().name())) { + Optional storage = Optional.empty(); + storage = s3StorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + S3Storage s3 = mapper.map(storage.get(), S3Storage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); + } + } else if (StorageType.GCS.name().equals(resourceEntity.getStorageType().name())) { + Optional storage = Optional.empty(); + storage = gcsStorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + GCSStorage gcs = mapper.map(storage.get(), GCSStorage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setGcsStorage(gcs).build(); + } + } + + GenericResource.Builder responseBuilder = GenericResource.newBuilder(); + resourceStorageMapper.mapGenericStorageEntityToModel(resourceEntity, wrapper, responseBuilder); + return responseBuilder.build(); + } + + @Override + public GenericResource updateGenericResource(GenericResourceUpdateRequest request) throws Exception { + + Optional savedGenericResourceEntityList = Optional.empty(); + if (!request.getResourceId().isBlank()) { + + savedGenericResourceEntityList = resourceRepository.findByResourceId(request.getResourceId()); + + } else if (request.getGenericResource().hasStorage()) { + + savedGenericResourceEntityList = resourceRepository.findByResourceId(request.getGenericResource().getResourceId()); + + } + if (savedGenericResourceEntityList.isEmpty()) { + return null; + } + GenericResourceEntity resourceEntity = savedGenericResourceEntityList.get(); + StorageWrapper wrapper = null; + if (StorageType.S3.name().equals(resourceEntity.getStorageType().name() )) { + Optional storage = null; + storage = s3StorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + boolean updated = updateS3Storage(storage.get(), request.getGenericResource().getStorage().getS3Storage()); + if (updated) { + S3Storage s3 = mapper.map(storage.get(), S3Storage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setS3Storage(s3).build(); + } + } + } else if (StorageType.GCS.name().equals(resourceEntity.getStorageType().name())) { + Optional storage = null; + storage = gcsStorageRepository.findById(resourceEntity.getStorageId()); + if (storage.isPresent()) { + boolean updated = updateGCSStorage(storage.get(), request.getGenericResource().getStorage().getGcsStorage()); + if (updated) { + GCSStorage gcs = mapper.map(storage.get(), GCSStorage.newBuilder().getClass()).build(); + wrapper = StorageWrapper.newBuilder().setGcsStorage(gcs).build(); + } + } + } + resourceStorageMapper.mapGenericStorageModelToEntity(request.getGenericResource(), resourceEntity); + + GenericResourceEntity savedDataProductEntity = resourceRepository.save(resourceEntity); + + GenericResource.Builder responseBuilder = GenericResource.newBuilder(); + resourceStorageMapper.mapGenericStorageEntityToModel(savedDataProductEntity, wrapper, responseBuilder); + return responseBuilder.build(); + } + + @Override + public GenericResource deleteGenericResource(GenericResourceDeleteRequest request) throws Exception { + resourceRepository.deleteByResourceId(request.getResourceId()); + return null; + } + + @Override + public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception { + Optional resourceSecEtyOp = resourceSecretRepository.findByStorageId(request.getStorageId()); + SecretForStorage.Builder resultBuilder = SecretForStorage.newBuilder(); + if (resourceSecEtyOp.isPresent()) { + StorageSecretEntity storageSecretEntity = resourceSecEtyOp.get(); + resultBuilder.setSecretId(storageSecretEntity.getSecretId()); + resultBuilder.setStorageId(storageSecretEntity.getStorageId()); + resultBuilder.setStorageType(resourceStorageMapper.resolveStorage( storageSecretEntity.getType())); + } else { + resultBuilder.setError(Error.NOT_FOUND); + } + return resultBuilder.build(); + } + + @Override + public SecretForStorage registerSecretForStorage(SecretForStorageCreateRequest request) throws Exception { + + StorageSecretEntity resourceEntity = new StorageSecretEntity(); + resourceStorageMapper.mapStorageSecretModelToEntity(request.getSecretForStorage(), resourceEntity); + StorageSecretEntity savedDataProductEntity = resourceSecretRepository.save(resourceEntity); + SecretForStorage.Builder responseBuilder = SecretForStorage.newBuilder(); + resourceStorageMapper.mapStorageSecretEntityToModel(savedDataProductEntity, responseBuilder); + + return responseBuilder.build(); + } + + @Override + public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) throws Exception { + resourceSecretRepository.deleteByStorageId(request.getStorageId()); + return true; + } + + @Override + public StorageListResponse searchStorages(StorageSearchRequest request) throws Exception { + StorageListResponse.Builder resp = StorageListResponse.newBuilder(); + switch (request.getSearchQueryCase()) { + case STORAGE_ID: + Optional storageOp = resolveStorageRepository.getByStorageId(request.getStorageId()); + if (storageOp.isPresent()) { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(storageOp.get().getStorageId()); + entry.setStorageName(storageOp.get().getStorageName()); + entry.setStorageType(StorageType.valueOf(storageOp.get().getStorageType().name())); + resp.addStorageList(entry); + } + break; + case STORAGE_NAME: + List storages = resolveStorageRepository.getByStorageName(request.getStorageName()); + storages.forEach(st -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(st.getStorageId()); + entry.setStorageName(st.getStorageName()); + entry.setStorageType(StorageType.valueOf(st.getStorageType().name())); + resp.addStorageList(entry); + }); + break; + case STORAGE_TYPE: + storages = resolveStorageRepository.getByStorageType(ResolveStorageEntity.StorageType.valueOf(request.getStorageType().name())); + storages.forEach(st -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(st.getStorageId()); + entry.setStorageName(st.getStorageName()); + entry.setStorageType(StorageType.valueOf(st.getStorageType().name())); + resp.addStorageList(entry); + }); + break; + } + return resp.build(); + } + + @Override + public StorageListResponse listStorage(StorageListRequest request) throws Exception { + Iterable all = resolveStorageRepository.findAll(); + StorageListResponse.Builder builder = StorageListResponse.newBuilder(); + all.forEach(r -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(r.getStorageId()); + entry.setStorageType(StorageType.valueOf(r.getStorageType().name())); + entry.setStorageName(r.getStorageName()); + builder.addStorageList(entry); + }); + return builder.build(); + } + + + public S3StorageListResponse listS3Storage(S3StorageListRequest request) throws Exception { + S3StorageListResponse.Builder respBuilder = S3StorageListResponse.newBuilder(); + List all = s3StorageRepository.findAll(PageRequest.of(request.getOffset(), request.getLimit())); + all.forEach(ety -> respBuilder.addStorages(mapper.map(ety, S3Storage.newBuilder().getClass()))); + return respBuilder.build(); + } + + + public Optional getS3Storage(S3StorageGetRequest request) throws Exception { + Optional entity = s3StorageRepository.findById(request.getStorageId()); + return entity.map(e -> mapper.map(e, S3Storage.newBuilder().getClass()).build()); + } + + + public S3Storage createS3Storage(S3Storage request) throws Exception { + S3StorageEntity savedEntity = s3StorageRepository.save(mapper.map(request, S3StorageEntity.class)); + + ResolveStorageEntity storageTypeEty = new ResolveStorageEntity(); + storageTypeEty.setStorageId(savedEntity.getStorageId()); + storageTypeEty.setStorageType(ResolveStorageEntity.StorageType.S3); + storageTypeEty.setStorageName(savedEntity.getName()); + resolveStorageRepository.save(storageTypeEty); + + return mapper.map(savedEntity, S3Storage.newBuilder().getClass()).build(); + } + + + public boolean updateS3Storage(S3StorageEntity entity, S3Storage storage) throws Exception { + + entity.setEndpoint(storage.getEndpoint()); + entity.setName(storage.getName()); + entity.setBucketName(storage.getBucketName()); + entity.setRegion(storage.getRegion()); + entity.setUseTLS(storage.getUseTLS()); + s3StorageRepository.save(entity); + return true; + } + + + public boolean deleteS3Storage(S3StorageDeleteRequest request) throws Exception { + s3StorageRepository.deleteById(request.getStorageId()); + resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.S3); + return true; + } + + + public GCSStorage createGCSStorage(GCSStorage request) throws Exception { + GCSStorageEntity savedEntity = gcsStorageRepository.save(mapper.map(request, GCSStorageEntity.class)); + + ResolveStorageEntity storageTypeEty = new ResolveStorageEntity(); + storageTypeEty.setStorageId(savedEntity.getStorageId()); + storageTypeEty.setStorageType(ResolveStorageEntity.StorageType.GCS); + storageTypeEty.setStorageName(savedEntity.getName()); + resolveStorageRepository.save(storageTypeEty); + + return mapper.map(savedEntity, GCSStorage.newBuilder().getClass()).build(); + } + + + public boolean updateGCSStorage(GCSStorageEntity entity, GCSStorage storage) throws Exception { + entity.setName(storage.getName()); + entity.setBucketName(storage.getBucketName()); + gcsStorageRepository.save(entity); + return true; + } + + @Override + public StorageTypeResolveResponse resolveStorageType(StorageTypeResolveRequest request) throws Exception { + Optional resolveStorageOp = resolveStorageRepository.getByStorageId(request.getStorageId()); + StorageTypeResolveResponse.Builder responseBuilder = StorageTypeResolveResponse.newBuilder(); + + if (resolveStorageOp.isPresent()) { + ResolveStorageEntity resolveStorageEntity = resolveStorageOp.get(); + responseBuilder.setStorageId(resolveStorageEntity.getStorageId()); + responseBuilder.setStorageType(StorageType.valueOf(resolveStorageEntity.getStorageType().name())); + responseBuilder.setStorageName(resolveStorageEntity.getStorageName()); + } else { + responseBuilder.setError(Error.NOT_FOUND); + } + return responseBuilder.build(); + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java new file mode 100644 index 0000000..893b8a4 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/mapper/ResourceSecretMapper.java @@ -0,0 +1,33 @@ +package org.apache.airavata.replicacatalog.sceret.mapper; + +import org.apache.airavata.replicacatalog.catalog.stubs.DataReplicaLocation; +import org.apache.airavata.replicacatalog.catalogapi.model.DataReplicaLocationEntity; +import org.apache.airavata.replicacatalog.resource.model.GenericResourceEntity; +import org.apache.airavata.replicacatalog.resource.repository.GenericResourceRepository; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Map to/from + * {@link DataReplicaLocationEntity} + * <-> {@link DataReplicaLocation} + */ +@Component +public class ResourceSecretMapper +{ + + @Autowired + GenericResourceRepository genericResourceRepository; + + public void mapModelToEntity(SecretForStorage storage, GenericResourceEntity resourceEntity) { + + resourceEntity.setResourceId(storage.getStorageId()); + // TODO + } + + public void mapEntityToModel(GenericResourceEntity resourceEntity, SecretForStorage.Builder dataProductBuilder) { + + //TODO + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java new file mode 100644 index 0000000..8d02d06 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/FTPSecretEntity.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "FTP_SECRET") +public class FTPSecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "USER_ID") + private String userId; + + @Column(name = "PASSWORD") + private String password; + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java new file mode 100644 index 0000000..339cb64 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/GCSSecretEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + +@Entity +@Table(name = "GCS_SECRET") +public class GCSSecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator( name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "PROJECT_ID") + private String projectId; + + @Column(name = "PRIVATE_KEY", length = 3000) + private String privateKey; + + @Column(name = "CLIENT_EMAIL") + private String clientEmail; + + + public String getSecretId() + { + return secretId; + } + + public void setSecretId( String secretId ) + { + this.secretId = secretId; + } + + public String getProjectId() + { + return projectId; + } + + public void setProjectId( String projectId ) + { + this.projectId = projectId; + } + + public String getPrivateKey() + { + return privateKey; + } + + public void setPrivateKey( String privateKey ) + { + this.privateKey = privateKey; + } + + public String getClientEmail() + { + return clientEmail; + } + + public void setClientEmail( String clientEmail ) + { + this.clientEmail = clientEmail; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java new file mode 100644 index 0000000..363467c --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/model/S3SecretEntity.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.annotations.GenericGenerator; + + + +@Entity +@Table(name = "S3_SECRET") +public class S3SecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "ACCESS_KEY") + private String accessKey; + + @Column(name = "SECRET_KEY") + private String secretKey; + + @Column(name = "SESSION_TOKEN", length = 512) + private String sessionToken; + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public String getSessionToken() { + return sessionToken; + } + + public void setSessionToken(String sessionToken) { + this.sessionToken = sessionToken; + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java new file mode 100644 index 0000000..1a8088c --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/FTPSecretRepository.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.repository; + +import java.util.Optional; + +import org.apache.airavata.replicacatalog.sceret.model.FTPSecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface FTPSecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java new file mode 100644 index 0000000..6a98676 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/GCSSecretRepository.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.repository; + +import java.util.Optional; + +import org.apache.airavata.replicacatalog.sceret.model.GCSSecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface GCSSecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java new file mode 100644 index 0000000..8f60741 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/repository/S3SecretRepository.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.repository; + +import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface S3SecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java new file mode 100644 index 0000000..49aef84 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/ISecretService.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.service; + +import org.apache.airavata.replicacatalog.secret.stubs.common.*; + +import java.util.Optional; + +public interface ISecretService { + + public void init(); + public void destroy(); + + public StorageSecret getSecret(SecretGetRequest request) throws Exception; + public StorageSecret registerSecretForStorage(SecretCreateRequest request) throws Exception; + public boolean deleteSecret(SecretDeleteRequest request) throws Exception; + public SecretListResponse searchSecrets(SecretSearchRequest request) throws Exception; + public SecretListResponse listSecrets(SecretListRequest request) throws Exception; + +// public Optional getS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretGetRequest request) throws Exception; +// public org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret createS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretCreateRequest request) throws Exception; +// public boolean updateS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretUpdateRequest request) throws Exception; +// public boolean deleteS3Secret(org.apache.airavata.replicacatalog.secret.stubs.s3.S3SecretDeleteRequest request) throws Exception; +// + +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java new file mode 100644 index 0000000..0cb9967 --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/SQLSecretService.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.replicacatalog.sceret.service; + +import org.apache.airavata.replicacatalog.catalog.stubs.StorageType; +import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; +import org.apache.airavata.replicacatalog.sceret.model.GCSSecretEntity; +import org.apache.airavata.replicacatalog.sceret.model.S3SecretEntity; +import org.apache.airavata.replicacatalog.sceret.repository.GCSSecretRepository; +import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; +import org.apache.airavata.replicacatalog.secret.stubs.common.*; +import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret; +import org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecretGetRequest; +import org.apache.airavata.replicacatalog.secret.stubs.s3.*; +import org.dozer.DozerBeanMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +@Component("SQLSecretBackend") +public class SQLSecretService implements ISecretService { + + private static final Logger logger = LoggerFactory.getLogger(SQLSecretService.class); + + + @Autowired + ResourceSecretMapper secretMapper = new ResourceSecretMapper(); + + @Autowired + private S3SecretRepository s3SecretRepository; + + @Autowired + private GCSSecretRepository gcsSecretRepository; + + private DozerBeanMapper mapper = new DozerBeanMapper(); + + @Override + public void init() { + logger.info("Initializing database secret backend"); + } + + @Override + public void destroy() { + logger.info("Destroying database secret backend"); + } + + @Override + public StorageSecret getSecret(SecretGetRequest request) throws Exception { + String id = request.getSecretId(); + StorageSecret.Builder storageSecret = StorageSecret.newBuilder(); + if (StorageType.S3.name().equals(request.getStorageType().name())) { + S3SecretGetRequest getRequest = S3SecretGetRequest.newBuilder().setSecretId(id).build(); + Optional secret = getS3Secret(getRequest); + return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret(secret.get())).setSecretId(id).build(); + } else if (StorageType.GCS.name().equals(request.getStorageType().name())) { + GCSSecretGetRequest getRequest = GCSSecretGetRequest.newBuilder().setSecretId(id).build(); + Optional secret = getGCSSecret(getRequest); + return storageSecret.setSecret(SecretWrapper.newBuilder().setGcsSecret(secret.get())).setSecretId(id).build(); + } + return null; + + } + + @Override + public StorageSecret registerSecretForStorage(SecretCreateRequest request) throws Exception { + StorageSecret stoReq = request.getSecret(); + StorageSecret.Builder storageSecret = StorageSecret.newBuilder(); + if (StorageType.S3.name().equals(stoReq.getStorageType().name())) { + S3Secret secret = createS3Secret(stoReq.getSecret().getS3Secret()); + return storageSecret.setSecret(SecretWrapper.newBuilder().setS3Secret(secret)).setSecretId(secret.getSecretId()).build(); + } else if (StorageType.GCS.name().equals(stoReq.getStorageType().name())) { + GCSSecret secret = createGCSSecret(stoReq.getSecret().getGcsSecret()); + return storageSecret.setSecret(SecretWrapper.newBuilder().setGcsSecret(secret)).setSecretId(secret.getSecretId()).build(); + } + return null; + } + + @Override + public boolean deleteSecret(SecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public SecretListResponse searchSecrets(SecretSearchRequest request) throws Exception { + return null; + } + + @Override + public SecretListResponse listSecrets(SecretListRequest request) throws Exception { + return null; + } + + + public Optional getS3Secret(S3SecretGetRequest request) throws Exception { + Optional secretEty = s3SecretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(s3SecretEntity -> mapper.map(s3SecretEntity, S3Secret.newBuilder().getClass()).build()); + } + + public S3Secret createS3Secret(S3Secret request) throws Exception { + S3SecretEntity savedEntity = s3SecretRepository.save(mapper.map(request, S3SecretEntity.class)); + return mapper.map(savedEntity, S3Secret.newBuilder().getClass()).build(); + } + + public boolean updateS3Secret(S3SecretUpdateRequest request) throws Exception { + s3SecretRepository.save(mapper.map(request, S3SecretEntity.class)); + return true; + } + + public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { + s3SecretRepository.deleteById(request.getSecretId()); + return true; + } + + + public GCSSecret createGCSSecret(GCSSecret request) throws Exception { + GCSSecretEntity savedEntity = gcsSecretRepository.save(mapper.map(request, GCSSecretEntity.class)); + return mapper.map(savedEntity, GCSSecret.newBuilder().getClass()).build(); + } + + public Optional getGCSSecret(GCSSecretGetRequest request) throws Exception { + Optional secretEty = gcsSecretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(entity -> mapper.map(entity, GCSSecret.newBuilder().getClass()).build()); + } +} diff --git a/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java new file mode 100644 index 0000000..c570e4c --- /dev/null +++ b/replica-catalog-api/server/src/main/java/org/apache/airavata/replicacatalog/sceret/service/StorageSecretAPIService.java @@ -0,0 +1,55 @@ +package org.apache.airavata.replicacatalog.sceret.service; + +import java.util.UUID; + +import io.grpc.stub.StreamObserver; +import org.apache.airavata.replicacatalog.resource.stubs.common.SecretForStorage; +import org.apache.airavata.replicacatalog.sceret.mapper.ResourceSecretMapper; +import org.apache.airavata.replicacatalog.sceret.repository.S3SecretRepository; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCommonServiceGrpc; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretCreateRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.SecretGetRequest; +import org.apache.airavata.replicacatalog.secret.stubs.common.StorageSecret; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +@GRpcService +public class StorageSecretAPIService extends SecretCommonServiceGrpc.SecretCommonServiceImplBase { + + + private static final Logger logger = LoggerFactory.getLogger(StorageSecretAPIService.class); + + + @Autowired + ISecretService secretService; + + @Autowired + ResourceSecretMapper secretMapper = new ResourceSecretMapper(); + + @Override + public void registerSecret(SecretCreateRequest request, StreamObserver responseObserver) { + StorageSecret secretResult = null; + try { + secretResult = secretService.registerSecretForStorage(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + + responseObserver.onNext(secretResult); + responseObserver.onCompleted(); + } + + @Override + public void getSecret(SecretGetRequest request, StreamObserver responseObserver) { + StorageSecret secretResult = null; + try { + secretResult = secretService.getSecret(request); + } catch (Exception e) { + throw new RuntimeException(e); + } + responseObserver.onNext(secretResult); + responseObserver.onCompleted(); + } +} diff --git a/replica-catalog-api/server/src/main/resources/application.properties b/replica-catalog-api/server/src/main/resources/application.properties new file mode 100644 index 0000000..3948d56 --- /dev/null +++ b/replica-catalog-api/server/src/main/resources/application.properties @@ -0,0 +1,5 @@ +spring.datasource.url=jdbc:postgresql://localhost:5432/replica_catalog +spring.datasource.username=postgres +spring.datasource.password=example +spring.jpa.hibernate.ddl-auto=create-drop +spring.jpa.show-sql=true \ No newline at end of file diff --git a/replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java b/replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java new file mode 100644 index 0000000..e77dd6e --- /dev/null +++ b/replica-catalog-api/server/src/test/java/org/apache/airavata/replicacatalog/catalogapi/service/ReplicaCatalogAPIServiceTest.java @@ -0,0 +1,37 @@ +package org.apache.airavata.replicacatalog.catalogapi.service; + +import org.junit.jupiter.api.Test; + +class ReplicaCatalogAPIServiceTest { + + @Test + void registerReplicaLocation() { + + ReplicaCatalogAPIService apiService = new ReplicaCatalogAPIService(); +// apiService.registerReplicaLocation(); + } + + @Test + void updateReplicaLocation() { + } + + @Test + void getReplicaLocation() { + } + + @Test + void removeReplicaLocation() { + } + + @Test + void getAllReplicaLocation() { + } + + @Test + void removeAllReplicaLocation() { + } + + @Test + void testRegisterReplicaLocation() { + } +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/pom.xml b/replica-catalog-api/stubs/pom.xml new file mode 100644 index 0000000..895127c --- /dev/null +++ b/replica-catalog-api/stubs/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + org.apache.airavata + replica-catalog-api + 0.1-SNAPSHOT + + + replica-catalog-api-stubs + + + io.grpc + grpc-netty-shaded + ${grpc.version} + runtime + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + org.apache.tomcat + annotations-api + ${tomcat.annotations-api.version} + provided + + + + + + kr.motd.maven + os-maven-plugin + ${os.maven.plugin} + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf.maven.plugin} + + com.google.protobuf:protoc:${protobuf.protoc.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + + + + + compile + compile-custom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto new file mode 100644 index 0000000..4f9a992 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPI.proto @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.apache.airavata.replicacatalog.catalog.stubs"; + +message UserInfo { + /* + * This is an external identifier for the user, which identifies the user in + * whatever user management scheme data catalog is integrated with. + */ + string user_id = 1; + /* + * This is an (optional) external identifier for the tenant, which + * identifies the tenant in whatever tenant management scheme data catalog + * is integrated with. + */ + string tenant_id = 2; +} + +message GroupInfo { + /* + * This is an external identifier for the group, which identifies the group in + * whatever group management scheme data catalog is integrated with. + */ + string group_id = 1; + /* + * This is an (optional) external identifier for the tenant, which + * identifies the tenant in whatever tenant management scheme data catalog + * is integrated with. + */ + string tenant_id = 2; +} + +enum Permission { + OWNER = 0; + READ = 1; + READ_METADATA = 2; + WRITE = 3; + WRITE_METADATA = 4; + MANAGE_SHARING = 5; +} + +enum StorageType { + NONE = 0; + S3 = 1; + FTP = 2; + LOCAL = 3; + GCS = 4; + BOX = 5; + DROPBOX = 6; + AZURE = 7; + SWIFT = 8; + ODATA = 9; + SCP = 10; +} + +enum DataProductType { + FILE = 0; + COLLECTION = 1; +} + +enum ErrorCode { + INVALID_DATA = 0; + INTERNAL_ERROR = 1; + MAPPING_ERROR = 2; +} + +message Error { + ErrorCode code = 1; + string message = 2; +} + +// Entity which used to model a replica entry +message DataReplicaLocation { + string data_replica_id = 1; // Unique replica id + string data_product_uri = 2; // related data uri + string parent_replica_id = 3; // If there is parent replica (Like directory structure) + string storage_resource_id = 4; // Replica storage id (TO link the storage entry) + string replica_name = 5; // Replica user friendly name + string replica_description = 6; // Replica description + int64 creation_time = 7; // Replica creation date and time + int64 last_modified_time = 8; // Last modified time of replica + int64 valid_until_time = 9; // Validity time of the replica + StorageType storage_type = 10; // Storage type of the replica + map metadata = 11; // Replica's metadata information +} + +message DataReplicaCreateRequest { + DataReplicaLocation data_replica = 1; +} +message DataReplicaCreateResponse{ + DataReplicaLocation data_replica = 1; + Error error = 2; +} + +message DataReplicaGroupEntryCreateRequest { + ReplicaGroupEntry data_replica_group = 1; +} +message DataReplicaGroupEntryCreateResponse{ + ReplicaGroupEntry data_replica_group = 1; + Error error = 2; + +} + +message DataReplicaUpdateRequest { + DataReplicaLocation data_replica = 1; +} +message DataReplicaUpdateResponse{ + DataReplicaLocation data_replica = 1; + Error error = 2; +} +message DataReplicaGetRequest { + string data_replica_id = 1; +} +message DataReplicaGetResponse { + DataReplicaLocation data_replica = 1; + Error error = 2; +} +message DataReplicaDeleteRequest { + string data_replica_id = 1; +} +message DataReplicaDeleteResponse { + Error error = 1; +} + +message AllDataReplicaGetRequest { + string data_product_uri = 1; + int32 page_number = 2; + int32 page_size = 3; +} + +message AllDataReplicaGetResponse { + string data_product_uri = 1; + repeated ReplicaGroupEntry replica_list = 2; + Error error = 3; +} + +message ReplicaGroupEntry { + string replica_group_id = 1; + string data_replica_id = 2; + repeated ReplicaGroupEntry directories = 3; + repeated DataReplicaLocation files = 4; +} + +message ReplicaListEntry { + string data_replica_id = 1; + string replica_name = 2; + StorageType storage_type = 3; +} + +message AllDataReplicaDeleteRequest { + string data_product_uri = 1; +} + +message AllDataReplicaDeleteResponse{ + int32 replica_location_count = 1; + Error error = 2; +} + diff --git a/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto new file mode 100644 index 0000000..a95dc92 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/catalogapi/ReplicaCatalogAPIService.proto @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +syntax = "proto3"; + +option java_package = "org.apache.airavata.replicacatalog.catalog.service"; + +import "catalogapi/ReplicaCatalogAPI.proto"; + +service ReplicaCatalogAPIService { + + /* + * Register a new replica to the Replica Catalog + */ + rpc registerReplicaLocation(DataReplicaCreateRequest) returns (DataReplicaCreateResponse){} + + /* + * Update existing replica details in the ReplicaCatalog + */ + rpc updateReplicaLocation(DataReplicaUpdateRequest) returns (DataReplicaUpdateResponse){} + + /* + * Retrieving existing replica details in the ReplicaCatalog + */ + rpc getReplicaLocation(DataReplicaGetRequest) returns (DataReplicaGetResponse){} + + /* + * Removing a replica in the ReplicaCatalog + */ + rpc removeReplicaLocation(DataReplicaDeleteRequest) returns (DataReplicaDeleteResponse){} + + /* + * Retrieving all the existing replica details for a selected product data in the ReplicaCatalog + */ + rpc getAllReplicaLocation(AllDataReplicaGetRequest) returns (AllDataReplicaGetResponse){} + + /* + * Remove all the existing replicas for a selected product data in the ReplicaCatalog + */ + rpc removeAllReplicaLocation(AllDataReplicaDeleteRequest) returns (AllDataReplicaDeleteResponse){} + +} diff --git a/replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto b/replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto new file mode 100644 index 0000000..cd971c8 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/FTPStorage.proto @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.ftp; + +message FTPStorage { + string storage_id = 1; + string host = 2; + int32 port = 3; + string name = 4; +} + +message FTPStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message FTPStorageListResponse { + repeated FTPStorage storages = 1; +} + +message FTPStorageGetRequest { + string storage_id = 1; +} + +message FTPStorageCreateRequest { + string host = 1; + int32 port = 2; + string storage_id = 3; + string name = 4; +} + +message FTPStorageUpdateRequest { + string storage_id = 1; + string host = 2; + int32 port = 3; + string name = 4; +} + +message FTPStorageUpdateResponse { + string storage_id = 1; +} + +message FTPStorageDeleteRequest { + string storage_id = 1; +} + +message FTPStorageDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto b/replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto new file mode 100644 index 0000000..0296ac1 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/GCSStorage.proto @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.gcs; + +message GCSStorage { + string storage_id = 1; + string bucket_name = 2; + string name = 3; +} + +message GCSStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message GCSStorageListResponse { + repeated GCSStorage storages = 1; +} + +message GCSStorageGetRequest { + string storage_id = 1; +} + +message GCSStorageCreateRequest { + string bucket_name = 1; + string storage_id = 2; + string name = 3; +} + +message GCSStorageUpdateRequest { + string storage_id = 1; + string bucketName = 2; + string name = 3; +} + +message GCSStorageUpdateResponse { + string storage_id = 1; +} + +message GCSStorageDeleteRequest { + string storage_id = 1; +} + +message GCSStorageDeleteResponse { + bool status = 1; +} + + diff --git a/replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto b/replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto new file mode 100644 index 0000000..e48cd81 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/LocalStorage.proto @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.local; + +message LocalStorage { + string storage_id = 1; + string name = 2; + string agent_id = 3; + +} + +message LocalStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message LocalStorageListResponse { + repeated LocalStorage storages = 1; +} + +message LocalStorageGetRequest { + string storage_id = 1; +} + +message LocalStorageCreateRequest { + string agentId = 1; + string storage_id = 2; + string name = 3; +} + +message LocalStorageUpdateRequest { + string storage_id = 1; + string name = 2; + string agent_id = 3; +} + +message LocalStorageUpdateResponse { + string storage_id = 1; +} + +message LocalStorageDeleteRequest { + string storage_id = 1; +} + +message LocalStorageDeleteResponse { + bool status = 1; +} + diff --git a/replica-catalog-api/stubs/src/main/proto/resource/S3Storage.proto b/replica-catalog-api/stubs/src/main/proto/resource/S3Storage.proto new file mode 100644 index 0000000..4b3a4d2 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/S3Storage.proto @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.s3; + +message S3Storage { + string storage_id = 1; + string bucket_name = 2; + string region = 3; + string endpoint = 4; + bool useTLS = 5; + string name = 6; +} + +message S3StorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message S3StorageListResponse { + repeated S3Storage storages = 1; +} + +message S3StorageGetRequest { + string storageId = 1; +} + +message S3StorageCreateRequest { + string bucket_name = 1; + string region = 2; + string storage_id = 3; + string endpoint = 4; + bool useTLS = 5; + string name = 6; +} + +message S3StorageUpdateRequest { + string storage_id = 1; + string bucket_name = 2; + string region = 3; + string endpoint = 4; + bool useTLS = 5; + string name = 6; +} + +message S3StorageUpdateResponse { + string storage_id = 1; +} + + +message S3StorageDeleteRequest { + string storage_id = 1; +} + +message S3StorageDeleteResponse { + bool status = 1; +} + + diff --git a/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto b/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto new file mode 100644 index 0000000..2a0c929 --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/resource/StorageCommon.proto @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.resource.stubs.common; + +import "resource/S3Storage.proto"; +import "resource/FTPStorage.proto"; +import "resource/LocalStorage.proto"; +import "resource/GCSStorage.proto"; +import "catalogapi/ReplicaCatalogAPI.proto"; + + +//enum StorageType { +// S3 = 0; +// SCP = 1; +// FTP = 2; +// LOCAL = 3; +// BOX = 4; +// DROPBOX = 5; +// GCS = 6; +// AZURE = 7; +// SWIFT = 8; +// ODATA = 9; +//} + +enum Error { + NOT_FOUND = 0; + NO_PERMISSION = 1; + LIMIT_OVERFLOWED = 2; +} + +message StorageTypeResolveRequest { + string storage_id = 1; +} + +message StorageTypeResolveResponse { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; + Error error = 4; +} + + +message FileResource { + string resource_path = 1; +} + +message DirectoryResource { + string resource_path = 1; +} + +message GenericResource { + + string resource_id = 1; + string replica_id = 2; + oneof resource { + org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; + org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; + } + StorageWrapper storage = 5; + +} + +// Storage wrapper which holds the different storages like S3, GCS, Azura ... +message StorageWrapper { + oneof storage { + org.apache.airavata.replicacatalog.resource.stubs.s3.S3Storage s3Storage = 1; + org.apache.airavata.replicacatalog.resource.stubs.local.LocalStorage localStorage = 2; + org.apache.airavata.replicacatalog.resource.stubs.gcs.GCSStorage gcsStorage = 3; + org.apache.airavata.replicacatalog.resource.stubs.ftp.FTPStorage ftpStorage = 4; + } +} + +message GenericResourceGetRequest { + string replica_id = 1; + string resource_id = 2; +} + +message GenericResourceCreateRequest { + string storage_id = 1; + GenericResource resource = 2; +} + +message GenericResourceUpdateRequest { + string resource_id = 1; + string storage_id = 2; + oneof resource { + org.apache.airavata.replicacatalog.resource.stubs.common.FileResource file = 3; + org.apache.airavata.replicacatalog.resource.stubs.common.DirectoryResource directory = 4; + } + GenericResource genericResource = 5; + +} + +message GenericResourceUpdateResponse { + string resource_id = 1; +} + +message GenericResourceDeleteRequest { + string resource_id = 1; +} + +message GenericResourceDeleteResponse { + bool status = 1; +} + +// Link storage and secret +message SecretForStorage { + string storage_id = 1; + string secret_id = 2; + string user_identity = 3; + StorageType storage_type = 4; + Error error = 5; +} + +message SecretForStorageCreateRequest { + SecretForStorage secret_for_storage = 1; +} + +message SecretForStorageGetRequest { + string storage_id = 1; +} + +message SecretForStorageDeleteRequest { + string storage_id = 1; +} + +message SecretForStorageDeleteResponse { + bool status = 1; +} + +message StorageListEntry { + string storage_id = 1; + string storage_name = 2; + string replica_id = 3; + StorageType storage_type = 4; +} + +message StorageListResponse { + repeated StorageListEntry storage_list = 1; +} + +message StorageListRequest { + int32 page_number = 1; + int32 page_size = 2; +} + +message StorageSearchRequest { + oneof searchQuery { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; + } +} + + +/* + * Replica storage entry related services + */ +service StorageCommonService { + + /* Replica catalog resource storage operations */ + rpc createGenericResource (GenericResourceCreateRequest) returns (GenericResource); // Create Generic Resource and Create storage + rpc getGenericResource (GenericResourceGetRequest) returns (GenericResource); + rpc updateGenericResource (GenericResourceUpdateRequest) returns (GenericResourceUpdateResponse); + rpc deleteGenericResource (GenericResourceDeleteRequest) returns (GenericResourceDeleteResponse); + + /* Replica catalog resource storage and secret related operations */ + rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); // Get resource type + rpc registerSecretForStorage (SecretForStorageCreateRequest) returns (SecretForStorage); // Add new resource to replica catalog + rpc getSecretForStorage (SecretForStorageGetRequest) returns (SecretForStorage); // Get storage secret using storage ID + rpc deleteSecretsForStorage (SecretForStorageDeleteRequest) returns (SecretForStorageDeleteResponse); // Delete storage secret mapping + rpc searchStorages (StorageSearchRequest) returns (StorageListResponse); // Search storage secrets using storage id, name or the type + rpc listStorages (StorageListRequest) returns (StorageListResponse); // List storages secrets + +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto b/replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto new file mode 100644 index 0000000..632e51a --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/FTPCredential.proto @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.secret.stubs.ftp; + + +message FTPSecret { + string secretId = 1; + string userId = 2; + string password = 3; +} + +message FTPSecretGetRequest { + string secretId = 1; +} + +message FTPSecretCreateRequest { + string userId = 1; + string password = 2; +} + +message FTPSecretUpdateRequest { + string secretId = 1; + string userId = 2; + string password = 3; +} + +message FTPSecretUpdateResponse { + string secretId = 1; +} + +message FTPSecretDeleteRequest { + string secretId = 1; +} + +message FTPSecretDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto b/replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto new file mode 100644 index 0000000..0336ecb --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/GCSCredential.proto @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.secret.stubs.gcs; + + +// This represents GCS Service account credentials. https://cloud.google.com/iam/docs/service-accounts +message GCSSecret { + string secretId = 1; + string projectId = 2; + string privateKey = 3; + string clientEmail = 4; +} + +message GCSSecretGetRequest { + string secretId = 1; +} + +message GCSSecretCreateRequest { + string projectId = 1; + string privateKey = 2; + string clientEmail = 3; +} + +message GCSSecretUpdateRequest { + string secretId = 1; + string projectId = 2; + string privateKey = 3; + string clientEmail = 4; +} + +message GCSSecretUpdateResponse { + string secretId = 1; +} + +message GCSSecretDeleteRequest { + string secretId = 1; +} + +message GCSSecretDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/S3Credential.proto b/replica-catalog-api/stubs/src/main/proto/secret/S3Credential.proto new file mode 100644 index 0000000..b59f2fe --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/S3Credential.proto @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; + +package org.apache.airavata.replicacatalog.secret.stubs.s3; + +message S3Secret { + string secret_id = 1; + string access_key = 2; + string secret_key = 3; + string session_token = 4; +} + +message S3SecretGetRequest { + string secret_id = 1; +} + +message S3SecretCreateRequest { + string access_key = 1; + string secret_key = 2; + string session_token = 3; +} + +message S3SecretUpdateRequest { + string secret_id = 1; + string access_key = 2; + string secret_key = 3; + string session_token = 4; +} + +message S3SecretUpdateResponse { + string secret_id = 1; +} + +message S3SecretDeleteRequest { + string secret_id = 1; +} + +message S3SecretDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto b/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto new file mode 100644 index 0000000..8785fdc --- /dev/null +++ b/replica-catalog-api/stubs/src/main/proto/secret/SecretCommon.proto @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.replicacatalog.secret.stubs.common; + +import "secret/S3Credential.proto"; +import "secret/FTPCredential.proto"; +import "secret/GCSCredential.proto"; +import "catalogapi/ReplicaCatalogAPI.proto"; + +//enum StorageType { +// S3 = 0; +// SCP = 1; +// FTP = 2; +// LOCAL = 3; +// BOX = 4; +// DROPBOX = 5; +// GCS = 6; +// AZURE = 7; +// SWIFT = 8; +// ODATA = 9; +//} + +enum Error { + NOT_FOUND = 0; + NO_PERMISSION = 1; + LIMIT_OVERFLOWED = 2; +} +// +//message StorageTypeResolveRequest { +// string secret_id = 1; +//} +// +//message StorageTypeResolveResponse { +// string secret_id = 1; +// StorageType storage_type = 2; +// Error error = 3; +//} + +message StorageSecret { + string secret_id = 1; + StorageType storage_type = 2; + SecretWrapper secret = 3; + Error error = 4; +} + +message SecretWrapper { + oneof secret { + org.apache.airavata.replicacatalog.secret.stubs.s3.S3Secret s3Secret = 1; + org.apache.airavata.replicacatalog.secret.stubs.gcs.GCSSecret gcsSecret = 2; + org.apache.airavata.replicacatalog.secret.stubs.ftp.FTPSecret ftpSecret = 3; + + } +} + +message SecretCreateRequest { + StorageSecret secret = 1; +} + +message SecretGetRequest { + string secret_id = 1; + StorageType storage_type = 2; +} + +message SecretDeleteRequest { + string secret_id = 1; + StorageType storage_type = 2; +} + +message SecretDeleteResponse { + bool status = 1; +} + +message SecretListEntry { + string storage_id = 1; + string storage_name = 2; + StorageType storage_type = 3; +} + +message SecretListResponse { + repeated SecretListEntry secret_list = 1; +} + +message SecretListRequest { + int32 page_number = 1; + int32 page_size = 2; +} + +message SecretSearchRequest { + oneof searchQuery { + string secret_id = 1; + StorageType storage_type = 2; + } +} + +/* + * Replica storage secret related services + */ +service SecretCommonService { +// rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); // Get secret type + rpc registerSecret (SecretCreateRequest) returns (StorageSecret); // Add new secret to replica catalog + rpc getSecret (SecretGetRequest) returns (StorageSecret); // Get secret using secret Id (Secret Id can retrieve from the replica ID using replica storage id-secret id linking (StorageSecretEntity)) + rpc deleteSecrets (SecretDeleteRequest) returns (SecretDeleteResponse); // Delete secret suing secret ID + rpc searchSecrets (SecretSearchRequest) returns (SecretListResponse); // Search secrets using secret Id or the secret type + rpc listSecrets (SecretListRequest) returns (SecretListResponse); // List all the secrets +} \ No newline at end of file