diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml index 48cfbeba1..6109bd777 100644 --- a/.github/workflows/pxf-ci.yml +++ b/.github/workflows/pxf-ci.yml @@ -330,7 +330,6 @@ jobs: - jdbc - proxy - unused - - s3 - features - gpdb - gpdb_fdw @@ -501,7 +500,6 @@ jobs: - jdbc - proxy - unused - - s3 - features - gpdb - gpdb_fdw @@ -659,6 +657,7 @@ jobs: matrix: tc_group: - 'pxf-jdbc' + - 'pxf-s3' use_fdw: - 'false' - 'true' diff --git a/automation/Makefile b/automation/Makefile index 8a1d168cd..053135562 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -35,11 +35,6 @@ endif # lowercase the protocol PROTOCOL := $(shell echo $(PROTOCOL) | tr A-Z a-z) -ifeq "$(PROTOCOL)" "minio" - MINIO=true - PROTOCOL=s3 -endif - PXF_BASE ?= $(PXF_HOME) TEMPLATES_DIR=$(PXF_HOME)/templates @@ -193,21 +188,6 @@ ifneq "$(PROTOCOL)" "" cp $(TEMPLATES_DIR)/pxf-site.xml $(PROTOCOL_HOME)/; \ sed $(SED_OPTS) 's||pxf.fs.basePath$(BASE_PATH)|g' $(PROTOCOL_HOME)/pxf-site.xml; \ fi; \ - if [ $(PROTOCOL) = s3 ]; then \ - if [ "$(MINIO)" = "true" ]; then \ - cp $(TEMPLATES_DIR)/minio-site.xml $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \ - sed $(SED_OPTS) "s|YOUR_MINIO_URL|http://localhost:9000|" $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \ - fi; \ - mkdir -p $(PROTOCOL_HOME)-invalid; \ - cp $(TEMPLATES_DIR)/$(PROTOCOL)-site.xml $(PROTOCOL_HOME)-invalid/; \ - if [ -z "$(ACCESS_KEY_ID)" ] || [ -z "$(SECRET_ACCESS_KEY)" ]; then \ - echo "AWS Keys (ACCESS_KEY_ID, SECRET_ACCESS_KEY) not set"; \ - rm -rf $(PROTOCOL_HOME); \ - exit 1; \ - fi; \ - sed $(SED_OPTS) "s|YOUR_AWS_ACCESS_KEY_ID|$(ACCESS_KEY_ID)|" $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \ - sed $(SED_OPTS) "s|YOUR_AWS_SECRET_ACCESS_KEY|$(SECRET_ACCESS_KEY)|" $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \ - fi; \ if [ $(PROTOCOL) = abfss ]; then \ if [ -z "$(ABFSS_ACCOUNT)" ] || [ -z "$(ABFSS_CLIENT_ENDPOINT)" ] || [ -z "$(ABFSS_CLIENT_ID)" ] || [ -z "$(ABFSS_CLIENT_SECRET)" ]; then \ echo "ADL Keys (ABFSS_ACCOUNT, ABFSS_CLIENT_ID, ABFSS_CLIENT_SECRET, ABFSS_CLIENT_ENDPOINT) not set"; \ @@ -255,6 +235,7 @@ endif # Usage: # make test-tc => run all testcontainers tests (Ubuntu) # make test-tc TC_GROUP=pxf-jdbc => run only pxf-jdbc group +# make test-tc TC_GROUP=pxf-s3 => run S3 Select + CloudAccess tests (MinIO sidecar) # make test-tc DISTRO=rocky9 => run with Rocky Linux 9 base image .PHONY: test-tc test-tc: check-env symlink_pxf_jars pxf_regress diff --git a/automation/pom.xml b/automation/pom.xml index 825633df1..1be23e504 100644 --- a/automation/pom.xml +++ b/automation/pom.xml @@ -23,6 +23,7 @@ 1.6.13 1.1.10.4 1.6.4 + @@ -59,7 +60,7 @@ 2.15 true - -Xmx4096m + -Xmx4096m ${argLine.extra} 1 false @@ -535,4 +536,16 @@ 1.9.5 + + + + java9-plus + + [9,) + + + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED + + + \ No newline at end of file diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java new file mode 100644 index 000000000..a92f3f39e --- /dev/null +++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java @@ -0,0 +1,98 @@ +package org.apache.cloudberry.pxf.automation.applications; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * S3 API access wrapper used by automation tests to seed and clean fixtures + * in a MinIO bucket. Owns the AmazonS3 client; callers should call + * `shutdown()` when done (typically in afterClass before stopping the + * MinIO container). + */ +public class S3Application implements AutoCloseable { + + private final AmazonS3 s3Client; + + public S3Application(MinIOContainer minio) { + this.s3Client = buildS3Client(minio.getHostEndpoint(), minio.getAccessKey(), minio.getSecretKey()); + } + + public void createBucket(String bucket) { + if (!s3Client.doesBucketExistV2(bucket)) { + s3Client.createBucket(bucket); + } + } + + public void putObject(String bucket, String key, Path localFile) throws IOException { + s3Client.putObject(new PutObjectRequest(bucket, key, localFile.toFile())); + } + + public void deletePrefix(String bucket, String prefix) { + ListObjectsV2Request request = new ListObjectsV2Request() + .withBucketName(bucket) + .withPrefix(prefix); + ListObjectsV2Result listing; + do { + listing = s3Client.listObjectsV2(request); + List keys = new ArrayList<>(); + for (S3ObjectSummary summary : listing.getObjectSummaries()) { + keys.add(summary.getKey()); + } + if (!keys.isEmpty()) { + s3Client.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys.toArray(new String[0]))); + } + request.setContinuationToken(listing.getNextContinuationToken()); + } while (listing.isTruncated()); + } + + public void shutdown() { + s3Client.shutdown(); + } + + @Override + public void close() { + shutdown(); + } + + private static AmazonS3 buildS3Client(String endpoint, String accessKey, String secretKey) { + return AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "us-east-1")) + .withPathStyleAccessEnabled(true) + .withCredentials(new AWSStaticCredentialsProvider( + new BasicAWSCredentials(accessKey, secretKey))) + .build(); + } +} diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java new file mode 100644 index 000000000..17c6a2cf2 --- /dev/null +++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java @@ -0,0 +1,78 @@ +package org.apache.cloudberry.pxf.automation.testcontainers; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * TestContainers wrapper around MinIO for S3 / S3 Select automation tests. + * The container joins a shared Docker network with alias minio, so PXF inside the + * Cloudberry container can reach it at http://minio:9000. + * + * This class only manages the container lifecycle and exposes endpoint / + * credential accessors. S3 API access (buckets, objects) lives in + * {@link org.apache.cloudberry.pxf.automation.applications.S3Application}. + */ +public class MinIOContainer extends GenericContainer { + + private static final String DEFAULT_IMAGE = "minio/minio:RELEASE.2024-11-07T00-52-20Z"; + private static final String NETWORK_ALIAS = "minio"; + + public static final int API_PORT = 9000; + public static final int CONSOLE_PORT = 9001; + + public static final String ACCESS_KEY = "admin"; + public static final String SECRET_KEY = "password"; + public static final String DEFAULT_BUCKET = "gpdb-ud-scratch"; + + public MinIOContainer(Network network) { + super(DockerImageName.parse(DEFAULT_IMAGE)); + + withNetwork(network) + .withNetworkAliases(NETWORK_ALIAS) + .withExposedPorts(API_PORT, CONSOLE_PORT) + .withEnv("MINIO_ROOT_USER", ACCESS_KEY) + .withEnv("MINIO_ROOT_PASSWORD", SECRET_KEY) + .withEnv("MINIO_API_SELECT_PARQUET", "on") + .withCommand("server", "/data", "--console-address", ":" + CONSOLE_PORT) + .waitingFor(Wait.forHttp("/minio/health/live").forPort(API_PORT)); + } + + /** S3 API endpoint reachable from the test JVM (mapped port). */ + public String getHostEndpoint() { + return "http://localhost:" + getMappedPort(API_PORT); + } + + /** S3 API endpoint for PXF and other containers on the same Docker network. */ + public String getInternalEndpoint() { + return "http://" + NETWORK_ALIAS + ":" + API_PORT; + } + + public String getAccessKey() { + return ACCESS_KEY; + } + + public String getSecretKey() { + return SECRET_KEY; + } +} diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java index 5382c7943..798af68a7 100644 --- a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java +++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java @@ -31,8 +31,8 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; +import org.apache.cloudberry.pxf.automation.utils.AutomationUtils; -import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -137,7 +137,7 @@ private static String resolveDistro() { */ public static synchronized PXFCloudberryContainer getInstance() { if (instance == null) { - String repo = resolveProperty("pxf.test.repo.path", findRepoPath()); + String repo = resolveProperty("pxf.test.repo.path", AutomationUtils.findRepoRoot().toString()); String distro = resolveDistro(); String imageName = "pxf/cbdb-testcontainer-" + distro + ":1"; String baseImage = BASE_IMAGES.getOrDefault(distro, BASE_IMAGES.get("ubuntu")); @@ -204,21 +204,6 @@ private static String resolveProperty(String key, String fallback) { return (value != null && !value.isEmpty()) ? value : fallback; } - private static String findRepoPath() { - File dir = new File(System.getProperty("user.dir")); - for (int i = 0; i < 5; i++) { - if (new File(dir, "automation/pom.xml").exists()) { - return dir.getAbsolutePath(); - } - dir = dir.getParentFile(); - if (dir == null) - break; - } - throw new IllegalStateException( - "Cannot auto-detect cloudberry-pxf repo root. Set -Dpxf.test.repo.path=..."); - } - - public Network getSharedNetwork() { return network; } diff --git a/automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java b/automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java new file mode 100644 index 000000000..c90aca2e2 --- /dev/null +++ b/automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java @@ -0,0 +1,60 @@ +package org.apache.cloudberry.pxf.automation.utils; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.nio.file.Path; + +/** + * Shared helpers for the cloudberry-pxf automation suite. + */ +public final class AutomationUtils { + + private static final String REPO_MARKER = "automation/pom.xml"; + private static final int MAX_PARENT_HOPS = 6; + + private AutomationUtils() { + } + + /** + * Walks upward from {@code user.dir} looking for the cloudberry-pxf repo + * root (identified by the {@code automation/pom.xml} marker). Used by code + * that needs to read resources from the working tree at runtime. + * + * @throws IllegalStateException if the marker is not found within a handful + * of parent directories. Set {@code -Dpxf.test.repo.path=...} to bypass. + */ + public static Path findRepoRoot() { + File dir = new File(System.getProperty("user.dir")); + for (int i = 0; i < MAX_PARENT_HOPS; i++) { + if (new File(dir, REPO_MARKER).exists()) { + return dir.toPath().toAbsolutePath().normalize(); + } + dir = dir.getParentFile(); + if (dir == null) { + break; + } + } + throw new IllegalStateException( + "Cannot auto-detect cloudberry-pxf repo root from user.dir=" + + System.getProperty("user.dir") + + ". Set -Dpxf.test.repo.path=... to override."); + } +} diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh b/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh index 388880c66..3ff2a5134 100755 --- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh +++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh @@ -234,52 +234,13 @@ EOF EOF - # Configure S3 settings - mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3" - - for s3_site in "$PXF_BASE/servers/s3/s3-site.xml" "$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do - mkdir -p "$(dirname "$s3_site")" - cat > "$s3_site" <<'EOF' - - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.access.key - admin - - - fs.s3a.secret.key - password - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - - -EOF + # Configure pxf servers for S3 tests + local s3_servers_src="${REPO_DIR}/automation/src/main/resources/testcontainers/pxf-cbdb/servers" + for server_name in s3 s3-invalid; do + local server_dir="$PXF_BASE/servers/${server_name}" + mkdir -p "$server_dir" + cp -v "${s3_servers_src}/${server_name}/s3-site.xml" "$server_dir/s3-site.xml" done - mkdir -p /home/gpadmin/.aws/ - cat > "/home/gpadmin/.aws/credentials" <<'EOF' -[default] -aws_access_key_id = admin -aws_secret_access_key = password -EOF - } main() { diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh b/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh index 694ccac59..5babfb183 100755 --- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh +++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh @@ -58,15 +58,6 @@ export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DATA_DIRECTORY:-/home/gpadmin/wo # set cloudberry timezone utc export PGTZ=UTC -# -------------------------------------------------------------------- -# Minio defaults -# -------------------------------------------------------------------- -export AWS_ACCESS_KEY_ID=admin -export AWS_SECRET_ACCESS_KEY=password -export PROTOCOL=minio -export ACCESS_KEY_ID=admin -export SECRET_ACCESS_KEY=password - # -------------------------------------------------------------------- # PXF defaults # -------------------------------------------------------------------- diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml new file mode 100644 index 000000000..b7557be3a --- /dev/null +++ b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml @@ -0,0 +1,54 @@ + + + + + + fs.s3a.endpoint + http://minio:9000 + + + fs.s3a.access.key + invalid-access-key + + + fs.s3a.secret.key + invalid-secret-key + + + fs.s3a.path.style.access + true + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.aws.credentials.provider + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + + diff --git a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml new file mode 100644 index 000000000..b76f19bce --- /dev/null +++ b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml @@ -0,0 +1,50 @@ + + + + + + fs.s3a.endpoint + http://minio:9000 + + + fs.s3a.access.key + admin + + + fs.s3a.secret.key + password + + + fs.s3a.path.style.access + true + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.aws.credentials.provider + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + + diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java index bda4f5de0..8eabd4c28 100644 --- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java +++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java @@ -28,10 +28,14 @@ import org.apache.cloudberry.pxf.automation.utils.system.FDWUtils; import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Listeners; import reporters.CustomAutomationReport; +import java.lang.reflect.Method; + @Listeners({CustomAutomationLogger.class, CustomAutomationReport.class, FDWSkipTestAnalyzer.class}) public class AbstractTestcontainersTest { @@ -74,7 +78,7 @@ public final void doInit() throws Exception { regress = new RegressApplication(container); - // run users before class + // run user's before class beforeClass(); } finally { CustomAutomationLogger.revertStdoutStream(); @@ -87,8 +91,11 @@ public final void clean() throws Exception { if (ProtocolUtils.getPxfTestKeepData().equals("true")) { return; } + // redirect "clean" logs to log file CustomAutomationLogger.redirectStdoutStreamToFile(getClass().getSimpleName(), "clean"); try { + // run user's after class + afterClass(); if (cloudberry != null) { cloudberry.close(); } @@ -97,6 +104,56 @@ public final void clean() throws Exception { } } + + /** + * will be called before each test method start + * + * @throws Exception + */ + @BeforeMethod(alwaysRun = true) + public void runBeforeMethod() throws Exception { + // check if "beforeMethod exists and if so open log file and run it + if (checkMethodImplExists("beforeMethod")) { + // redirect "runBeforeMethod" logs to log file + CustomAutomationLogger.redirectStdoutStreamToFile(getClass().getSimpleName(), "beforeMethod"); + try { + beforeMethod(); + } catch (Throwable t) { + // in case of failure write stack trace to file stream and throw the exception + t.printStackTrace(System.out); + throw t; + } finally { + // anyways revert System.out to original stream + CustomAutomationLogger.revertStdoutStream(); + } + } + } + + /** + * will be called after each test method ended + * + * @throws Exception + */ + @AfterMethod(alwaysRun = true) + public void runAfterMethod() throws Exception { + // check if "afterMethod exists and if so open log file and run it + if (checkMethodImplExists("afterMethod")) { + // redirect "runAfterMethod" logs to log file + CustomAutomationLogger.redirectStdoutStreamToFile(getClass().getSimpleName(), "afterMethod"); + try { + afterMethod(); + } catch (Throwable t) { + // in case of failure write stack trace to file stream and throw the exception + t.printStackTrace(System.out); + throw t; + } finally { + // anyways revert System.out to original stream + CustomAutomationLogger.revertStdoutStream(); + } + } + } + + /** * clean up after the class finished * @@ -136,4 +193,25 @@ private void createTestDatabases(CloudberryApplication bootstrap) throws Excepti bootstrap.runQuery("SELECT 1"); System.out.println("[" + getClass().getSimpleName() + "] Test databases created"); } + + /** + * Check if the test writer used given method and return true if so. + * + * @param methodName to check + * @return true if method exists in declared methods + * @throws NoSuchMethodException + * @throws SecurityException + */ + private boolean checkMethodImplExists(String methodName) throws NoSuchMethodException, SecurityException { + // get all declared methods + Method[] methods = getClass().getDeclaredMethods(); + // run over methods and look for methodName + for (Method method : methods) { + + if (method.getName().equals(methodName)) { + return true; + } + } + return false; + } } diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/SmallDataFactory.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/SmallDataFactory.java new file mode 100644 index 000000000..7042bccd6 --- /dev/null +++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/SmallDataFactory.java @@ -0,0 +1,97 @@ +package org.apache.cloudberry.pxf.automation; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.commons.lang.StringUtils; +import org.apache.cloudberry.pxf.automation.structures.tables.basic.Table; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * Shared test-data fixtures for Testcontainers-based tests. + * + * Mirrors the data layout of {@link BaseFunctionality} + */ +public class SmallDataFactory { + + /** + * Create a data table with {@code numRows} rows of small data, + * with the following fields: String, int, double, long and boolean. + * + * @param uniqueName prefix applied to the {@code name} column (empty for no prefix) + * @param numRows number of rows to generate + * @return the generated {@link Table} + */ + public Table getSmallData(String uniqueName, int numRows) { + List> data = new ArrayList<>(); + + for (int i = 1; i <= numRows; i++) { + List row = new ArrayList<>(); + row.add(String.format("%s%srow_%d", uniqueName, StringUtils.isBlank(uniqueName) ? "" : "_", i)); + row.add(String.valueOf(i)); + row.add(Double.toString(i)); + row.add(Long.toString(100000000000L * i)); + row.add(String.valueOf(i % 2 == 0)); + data.add(row); + } + + Table dataTable = new Table("dataTable", null); + dataTable.setData(data); + + return dataTable; + } + + public Table getSmallData() { + return getSmallData(""); + } + + public Table getSmallData(String uniqueName) { + return getSmallData(uniqueName, 100); + } + + /** + * Serialize a table to a temporary CSV file, rows separated by newlines and + * fields separated by {@code delimiter}, with no trailing newline. The caller + * owns the returned file and is responsible for deleting it. + * + * @param table the table to serialize + * @param delimiter the field delimiter + * @return path to the temporary CSV file + */ + public Path writeTableToCsv(Table table, String delimiter) throws IOException { + Path tempFile = Files.createTempFile("tc-fixture-", ".csv"); + List> data = table.getData(); + try (BufferedWriter writer = Files.newBufferedWriter(tempFile, StandardCharsets.UTF_8)) { + for (int i = 0; i < data.size(); i++) { + writer.append(String.join(delimiter, data.get(i))); + if (i != data.size() - 1) { + writer.newLine(); + } + } + } + return tempFile; + } +} diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java index 13f0754f0..6fcdb3e34 100644 --- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java +++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java @@ -1,28 +1,44 @@ package org.apache.cloudberry.pxf.automation.features.cloud; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import annotations.WorksWithFDW; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.cloudberry.pxf.automation.components.hdfs.Hdfs; -import org.apache.cloudberry.pxf.automation.features.BaseFeature; -import org.apache.cloudberry.pxf.automation.structures.tables.basic.Table; +import org.apache.cloudberry.pxf.automation.AbstractTestcontainersTest; +import org.apache.cloudberry.pxf.automation.SmallDataFactory; +import org.apache.cloudberry.pxf.automation.applications.PXFApplication; +import org.apache.cloudberry.pxf.automation.applications.S3Application; +import org.apache.cloudberry.pxf.automation.structures.tables.pxf.ExternalTable; import org.apache.cloudberry.pxf.automation.structures.tables.utils.TableFactory; -import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils; -import org.apache.cloudberry.pxf.automation.utils.system.ProtocolEnum; +import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer; import org.testng.annotations.Test; -import java.net.URI; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.UUID; /** * Functional CloudAccess Test */ @WorksWithFDW -public class CloudAccessTest extends BaseFeature { - - private static final String PROTOCOL_S3 = "s3a://"; - private static final String S3_ENDPOINT = - System.getProperty("S3_ENDPOINT", System.getenv().getOrDefault("S3_ENDPOINT", "http://localhost:9000")); +public class CloudAccessTest extends AbstractTestcontainersTest { private static final String[] PXF_MULTISERVER_COLS = { "name text", @@ -37,55 +53,71 @@ public class CloudAccessTest extends BaseFeature { "score integer" }; - private Hdfs s3Server; - private String s3PathRead, s3PathWrite; + private static final String fileName = "data.txt"; + + private final SmallDataFactory dataFactory = new SmallDataFactory(); + + private MinIOContainer s3Server; + private S3Application s3Application; + private String s3PathRead; + private String s3PathWrite; + private String readObjectKeyPrefix; + private String writeObjectKeyPrefix; - /** - * Prepare all server configurations and components - */ @Override public void beforeClass() throws Exception { - // Initialize server objects - String random = UUID.randomUUID().toString(); - s3PathRead = String.format("gpdb-ud-scratch/tmp/pxf_automation_data_read/%s/" , random); - s3PathWrite = String.format("gpdb-ud-scratch/tmp/pxf_automation_data_write/%s/", random); + s3Server = new MinIOContainer(container.getSharedNetwork()); + s3Server.start(); + s3Application = new S3Application(s3Server); + s3Application.createBucket(MinIOContainer.DEFAULT_BUCKET); - Configuration s3Configuration = new Configuration(); - s3Configuration.set("fs.s3a.access.key", ProtocolUtils.getAccess()); - s3Configuration.set("fs.s3a.secret.key", ProtocolUtils.getSecret()); - applyS3Defaults(s3Configuration); + String random = UUID.randomUUID().toString(); + readObjectKeyPrefix = String.format("tmp/pxf_automation_data_read/%s/", random); + writeObjectKeyPrefix = String.format("tmp/pxf_automation_data_write/%s/", random); + s3PathRead = MinIOContainer.DEFAULT_BUCKET + "/" + readObjectKeyPrefix; + s3PathWrite = MinIOContainer.DEFAULT_BUCKET + "/" + writeObjectKeyPrefix; + } - FileSystem fs2 = FileSystem.get(URI.create(PROTOCOL_S3 + s3PathRead + fileName), s3Configuration); - s3Server = new Hdfs(fs2, s3Configuration, true); + @Override + public void afterClass() throws Exception { + if (s3Application != null) { + if (readObjectKeyPrefix != null) { + s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET, readObjectKeyPrefix); + } + if (writeObjectKeyPrefix != null) { + s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET, writeObjectKeyPrefix); + } + s3Application.shutdown(); + } + if (s3Server != null) { + s3Server.stop(); + } } @Override protected void beforeMethod() throws Exception { - if (ProtocolUtils.getProtocol() == ProtocolEnum.HDFS) { - return; + uploadSmallCsvFixture(MinIOContainer.DEFAULT_BUCKET, readObjectKeyPrefix + fileName); + } + + // Uploads small CSV test data (see BaseTCFunctionality#getSmallData()) to the given S3 object. + private void uploadSmallCsvFixture(String bucket, String objectKey) throws IOException { + Path tempFile = dataFactory.writeTableToCsv(dataFactory.getSmallData(), ","); + try { + System.out.println("[CloudAccessTest] Uploading " + tempFile + " -> s3://" + bucket + "/" + objectKey); + s3Application.putObject(bucket, objectKey, tempFile); + } finally { + Files.deleteIfExists(tempFile); } - super.beforeMethod(); - prepareData(); } @Override protected void afterMethod() throws Exception { - super.afterMethod(); - if (s3Server != null) { - s3Server.removeDirectory(PROTOCOL_S3 + s3PathRead); - s3Server.removeDirectory(PROTOCOL_S3 + s3PathWrite); + if (s3Application != null) { + s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET, readObjectKeyPrefix); + s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET, writeObjectKeyPrefix); } } - protected void prepareData() throws Exception { - // Prepare data in table - Table dataTable = getSmallData(); - - // Create Data for s3Server - s3Server.writeTableToFile(PROTOCOL_S3 + s3PathRead + fileName, dataTable, ","); - s3Server.createDirectory(PROTOCOL_S3 + s3PathWrite); - } - /* * The tests below are for the case where there's NO Hadoop cluster configured under "default" server * and assumes the "default" server has not configuration files. They are part of "s3" group and do not @@ -133,68 +165,66 @@ public void testCloudAccessOkWhenServerCredsNoConfigFileExists() throws Exceptio * both without and with Kerberos security, testing that cloud access works in presence of "default" server */ - @Test(groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudAccessWithHdfsFailsWhenNoServerNoCredsSpecified() throws Exception { runTestScenario("no_server_no_credentials_with_hdfs", null, false); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudAccessWithHdfsOkWhenServerNoCredsValidConfigFileExists() throws Exception { runTestScenario("server_no_credentials_valid_config_with_hdfs", "s3", false); } - @Test(groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudWriteWithHdfsOkWhenServerNoCredsValidConfigFileExists() throws Exception { runTestScenarioForWrite("server_no_credentials_valid_config_with_hdfs_write", "s3", false); } - @Test(groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudAccessWithHdfsFailsWhenServerNoCredsNoConfigFileExists() throws Exception { runTestScenario("server_no_credentials_no_config_with_hdfs", "s3-non-existent", false); } - @Test(groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudAccessWithHdfsFailsWhenNoServerCredsNoConfigFileExists() throws Exception { runTestScenario("no_server_credentials_no_config_with_hdfs", null, true); } - @Test(groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudAccessWithHdfsFailsWhenServerNoCredsInvalidConfigFileExists() throws Exception { runTestScenario("server_no_credentials_invalid_config_with_hdfs", "s3-invalid", false); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"gpdb", "security"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCloudAccessWithHdfsOkWhenServerCredsInvalidConfigFileExists() throws Exception { runTestScenario("server_credentials_invalid_config_with_hdfs", "s3-invalid", true); } private void runTestScenario(String name, String server, boolean creds) throws Exception { String tableName = "cloudaccess_" + name; - exTable = TableFactory.getPxfReadableTextTable(tableName, PXF_MULTISERVER_COLS, s3PathRead + fileName, ","); + ExternalTable exTable = TableFactory.getPxfReadableTextTable(tableName, PXF_MULTISERVER_COLS, s3PathRead + fileName, ","); exTable.setProfile("s3:text"); String serverParam = (server == null) ? null : "server=" + server; exTable.setServer(serverParam); if (creds) { - exTable.setUserParameters(new String[]{"accesskey=" + ProtocolUtils.getAccess(), "secretkey=" + ProtocolUtils.getSecret()}); + exTable.setUserParameters(new String[]{"accesskey=" + MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY}); } - gpdb.createTableAndVerify(exTable); + cloudberry.createTableAndVerify(exTable); - runSqlTest("features/cloud_access/" + name); + regress.runSqlTest("features/cloud_access/" + name); } private void runTestScenarioForWrite(String name, String server, boolean creds) throws Exception { // create writable external table to write to S3 String tableName = "cloudwrite_" + name; - exTable = TableFactory.getPxfWritableTextTable(tableName, PXF_WRITE_COLS, s3PathWrite, ","); + ExternalTable exTable = TableFactory.getPxfWritableTextTable(tableName, PXF_WRITE_COLS, s3PathWrite, ","); exTable.setProfile("s3:text"); String serverParam = (server == null) ? null : "server=" + server; exTable.setServer(serverParam); if (creds) { - exTable.setUserParameters(new String[]{"accesskey=" + ProtocolUtils.getAccess(), "secretkey=" + ProtocolUtils.getSecret()}); + exTable.setUserParameters(new String[]{"accesskey=" + MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY}); } - gpdb.createTableAndVerify(exTable); + cloudberry.createTableAndVerify(exTable); // create readable external table to read back from S3, making sure previous insert made it all the way to S3 tableName = "cloudaccess_" + name; @@ -202,19 +232,10 @@ private void runTestScenarioForWrite(String name, String server, boolean creds) exTable.setProfile("s3:text"); exTable.setServer(serverParam); if (creds) { - exTable.setUserParameters(new String[]{"accesskey=" + ProtocolUtils.getAccess(), "secretkey=" + ProtocolUtils.getSecret()}); + exTable.setUserParameters(new String[]{"accesskey=" + MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY}); } - gpdb.createTableAndVerify(exTable); - - runSqlTest("features/cloud_access/" + name); - } + cloudberry.createTableAndVerify(exTable); - private void applyS3Defaults(Configuration configuration) { - configuration.set("fs.s3a.endpoint", S3_ENDPOINT); - configuration.set("fs.s3a.path.style.access", "true"); - configuration.set("fs.s3a.connection.ssl.enabled", "false"); - configuration.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); - configuration.set("fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + regress.runSqlTest("features/cloud_access/" + name); } } diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java index 2b51bcd03..9986635cd 100644 --- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java +++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java @@ -1,27 +1,41 @@ package org.apache.cloudberry.pxf.automation.features.cloud; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.cloudberry.pxf.automation.components.hdfs.Hdfs; -import org.apache.cloudberry.pxf.automation.features.BaseFeature; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.cloudberry.pxf.automation.AbstractTestcontainersTest; +import org.apache.cloudberry.pxf.automation.applications.S3Application; import org.apache.cloudberry.pxf.automation.structures.tables.pxf.ReadableExternalTable; -import org.apache.cloudberry.pxf.automation.utils.system.ProtocolEnum; -import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils; +import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer; +import org.apache.cloudberry.pxf.automation.utils.AutomationUtils; import org.testng.annotations.Test; -import java.net.URI; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.UUID; import static org.apache.cloudberry.pxf.automation.features.tpch.LineItem.LINEITEM_SCHEMA; -/** - * Functional S3 Select Test - */ -public class S3SelectTest extends BaseFeature { - - private static final String PROTOCOL_S3 = "s3a://"; - private static final String S3_ENDPOINT = - System.getProperty("S3_ENDPOINT", System.getenv().getOrDefault("S3_ENDPOINT", "http://localhost:9000")); +/** Functional S3 Select Test */ +public class S3SelectTest extends AbstractTestcontainersTest { private static final String[] PXF_S3_SELECT_INVALID_COLS = { "invalid_orderkey BIGINT", @@ -42,8 +56,10 @@ public class S3SelectTest extends BaseFeature { "invalid_comment VARCHAR(44)" }; - private Hdfs s3Server; + private MinIOContainer s3Server; + private S3Application s3Application; private String s3Path; + private String objectKeyPrefix; private static final String sampleCsvFile = "sample.csv"; private static final String sampleGzippedCsvFile = "sample.csv.gz"; @@ -53,145 +69,149 @@ public class S3SelectTest extends BaseFeature { private static final String sampleParquetSnappyFile = "sample.snappy.parquet"; private static final String sampleParquetGzipFile = "sample.gz.parquet"; + private static final String[] FIXTURE_FILES = { + sampleCsvFile, + sampleCsvNoHeaderFile, + sampleGzippedCsvFile, + sampleBzip2CsvFile, + sampleParquetFile, + sampleParquetSnappyFile, + sampleParquetGzipFile, + }; + + private static final String FIXTURES_SUBDIR = "s3select"; + /** * Prepare all server configurations and components */ @Override public void beforeClass() throws Exception { - if (ProtocolUtils.getProtocol() == ProtocolEnum.HDFS) { - return; - } - // Initialize server objects - s3Path = String.format("gpdb-ud-scratch/tmp/pxf_automation_data/%s/s3select/", UUID.randomUUID().toString()); - Configuration s3Configuration = new Configuration(); - s3Configuration.set("fs.s3a.access.key", ProtocolUtils.getAccess()); - s3Configuration.set("fs.s3a.secret.key", ProtocolUtils.getSecret()); - applyS3Defaults(s3Configuration); + s3Server = new MinIOContainer(container.getSharedNetwork()); + s3Server.start(); + s3Application = new S3Application(s3Server); + s3Application.createBucket(MinIOContainer.DEFAULT_BUCKET); + + String uuid = UUID.randomUUID().toString(); + objectKeyPrefix = "tmp/pxf_automation_data/" + uuid + "/s3select/"; + s3Path = MinIOContainer.DEFAULT_BUCKET + "/" + objectKeyPrefix; - FileSystem fs2 = FileSystem.get(URI.create(PROTOCOL_S3 + s3Path + fileName), s3Configuration); - s3Server = new Hdfs(fs2, s3Configuration, true); + uploadFixtures(MinIOContainer.DEFAULT_BUCKET, objectKeyPrefix); + // Server 's3' is pre-baked into the container image by entrypoint.sh. } @Override - protected void afterClass() throws Exception { - super.afterClass(); + public void afterClass() throws Exception { + if (s3Application != null && objectKeyPrefix != null) { + s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET, objectKeyPrefix); + s3Application.shutdown(); + } if (s3Server != null) { - s3Server.removeDirectory(PROTOCOL_S3 + s3Path); + s3Server.stop(); + } + } + + // Uploads committed S3 Select fixture files from src/test/resources/data/s3select/ into MinIO. + private void uploadFixtures(String bucket, String objectKeyPrefix) throws IOException { + Path fixturesDir = resolveFixturesDirectory(); + for (String filename : FIXTURE_FILES) { + Path localFile = fixturesDir.resolve(filename); + if (!Files.isRegularFile(localFile)) { + throw new IOException("Missing S3 Select fixture: " + localFile); + } + String key = objectKeyPrefix + filename; + System.out.println("[S3SelectTest] Uploading " + localFile + " -> s3://" + bucket + "/" + key); + s3Application.putObject(bucket, key, localFile); } } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + private static Path resolveFixturesDirectory() throws IOException { + Path relative = Paths.get("src/test/resources/data", FIXTURES_SUBDIR); + if (Files.isDirectory(relative)) { + return relative.toAbsolutePath().normalize(); + } + Path fromRepo = AutomationUtils.findRepoRoot() + .resolve("automation/src/test/resources/data") + .resolve(FIXTURES_SUBDIR); + if (Files.isDirectory(fromRepo)) { + return fromRepo; + } + throw new IOException("Cannot find s3select fixtures directory (tried " + + relative.toAbsolutePath() + " and " + fromRepo + ")"); + } + + @Test(groups = {"testcontainers", "pxf-s3"}) public void testPlainCsvWithHeaders() throws Exception { String[] userParameters = {"FILE_HEADER=IGNORE", "S3_SELECT=ON"}; - runTestScenario("csv", "s3", "csv", s3Path, - localDataResourcesFolder + "/s3select/", sampleCsvFile, - "|", userParameters); + runTestScenario("csv", "s3", "csv", sampleCsvFile, "|", userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testPlainCsvWithHeadersUsingHeaderInfo() throws Exception { String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON"}; - runTestScenario("csv_use_headers", "s3", "csv", s3Path, - localDataResourcesFolder + "/s3select/", sampleCsvFile, - "|", userParameters); + runTestScenario("csv_use_headers", "s3", "csv", sampleCsvFile, "|", userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testCsvWithHeadersUsingHeaderInfoWithWrongColumnNames() throws Exception { String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON"}; - runTestScenario("errors/", "csv_use_headers_with_wrong_col_names", "s3", "csv", s3Path, - localDataResourcesFolder + "/s3select/", sampleCsvFile, "/" + s3Path + sampleCsvFile, + runTestScenario("errors/", "csv_use_headers_with_wrong_col_names", "s3", "csv", + sampleCsvFile, "/" + s3Path + sampleCsvFile, "|", userParameters, PXF_S3_SELECT_INVALID_COLS); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testPlainCsvWithNoHeaders() throws Exception { String[] userParameters = {"FILE_HEADER=NONE", "S3_SELECT=ON"}; - runTestScenario("csv_noheaders", "s3", "csv", s3Path, - localDataResourcesFolder + "/s3select/", sampleCsvNoHeaderFile, - "|", userParameters); + runTestScenario("csv_noheaders", "s3", "csv", sampleCsvNoHeaderFile, "|", userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testGzipCsvWithHeadersUsingHeaderInfo() throws Exception { String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON", "COMPRESSION_CODEC=gzip"}; - runTestScenario("gzip_csv_use_headers", "s3", "csv", s3Path, - localDataResourcesFolder + "/s3select/", sampleGzippedCsvFile, - "|", userParameters); + runTestScenario("gzip_csv_use_headers", "s3", "csv", sampleGzippedCsvFile, "|", userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testBzip2CsvWithHeadersUsingHeaderInfo() throws Exception { String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON", "COMPRESSION_CODEC=bzip2"}; - runTestScenario("bzip2_csv_use_headers", "s3", "csv", s3Path, - localDataResourcesFolder + "/s3select/", sampleBzip2CsvFile, - "|", userParameters); + runTestScenario("bzip2_csv_use_headers", "s3", "csv", sampleBzip2CsvFile, "|", userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testParquet() throws Exception { String[] userParameters = {"S3_SELECT=ON"}; - runTestScenario("parquet", "s3", "parquet", s3Path, - localDataResourcesFolder + "/s3select/", sampleParquetFile, - null, userParameters); + runTestScenario("parquet", "s3", "parquet", sampleParquetFile, null, userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testParquetWildcardLocation() throws Exception { String[] userParameters = {"S3_SELECT=ON"}; - runTestScenario("", "parquet", "s3", "parquet", s3Path, - localDataResourcesFolder + "/s3select/", sampleParquetFile, "/" + s3Path + "*e.parquet", + runTestScenario("", "parquet", "s3", "parquet", sampleParquetFile, "/" + s3Path + "*e.parquet", null, userParameters, LINEITEM_SCHEMA); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testSnappyParquet() throws Exception { String[] userParameters = {"S3_SELECT=ON"}; - runTestScenario("parquet_snappy", "s3", "parquet", s3Path, - localDataResourcesFolder + "/s3select/", sampleParquetSnappyFile, - null, userParameters); + runTestScenario("parquet_snappy", "s3", "parquet", sampleParquetSnappyFile, null, userParameters); } - // TODO: pxf_regress shows diff for this test. Should be fixed. - @Test(enabled = false, groups = {"s3"}) + @Test(groups = {"testcontainers", "pxf-s3"}) public void testGzipParquet() throws Exception { String[] userParameters = {"S3_SELECT=ON"}; - runTestScenario("parquet_gzip", "s3", "parquet", s3Path, - localDataResourcesFolder + "/s3select/", sampleParquetGzipFile, - null, userParameters); + runTestScenario("parquet_gzip", "s3", "parquet", sampleParquetGzipFile, null, userParameters); } private void runTestScenario( String name, String server, String format, - String s3Path, - String srcPath, String filename, String delimiter, String[] userParameters) throws Exception { - - runTestScenario("", - name, - server, - format, - s3Path, - srcPath, - filename, - "/" + s3Path + filename, - delimiter, - userParameters, - LINEITEM_SCHEMA); + runTestScenario("", name, server, format, filename, "/" + s3Path + filename, + delimiter, userParameters, LINEITEM_SCHEMA); } private void runTestScenario( @@ -199,40 +219,27 @@ private void runTestScenario( String name, String server, String format, - String s3Path, - String srcPath, String filename, String locationPath, String delimiter, String[] userParameters, String[] fields) throws Exception { - String tableName = "s3select_" + name; - String serverParam = (server == null) ? null : "server=" + server; - - s3Server.copyFromLocal(srcPath + filename, PROTOCOL_S3 + s3Path + filename); - - exTable = new ReadableExternalTable(tableName, fields, locationPath, "CSV"); + ReadableExternalTable exTable = new ReadableExternalTable(tableName, fields, locationPath, "CSV"); exTable.setProfile("s3:" + format); - exTable.setServer(serverParam); + exTable.setServer("server=" + server); + exTable.setHost(pxfHost); + exTable.setPort(pxfPort); - if (delimiter != null) + if (delimiter != null) { exTable.setDelimiter(delimiter); - if (userParameters != null) + } + if (userParameters != null) { exTable.setUserParameters(userParameters); + } - gpdb.createTableAndVerify(exTable); - - runSqlTest(String.format("features/s3_select/%s%s", qualifier, name)); - } - - private void applyS3Defaults(Configuration configuration) { - configuration.set("fs.s3a.endpoint", S3_ENDPOINT); - configuration.set("fs.s3a.path.style.access", "true"); - configuration.set("fs.s3a.connection.ssl.enabled", "false"); - configuration.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); - configuration.set("fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + cloudberry.createTableAndVerify(exTable); + regress.runSqlTest(String.format("features/s3_select/%s%s", qualifier, name)); } } diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh index a08bff167..7a4fe638d 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh @@ -332,52 +332,6 @@ EOF -EOF - - # Configure S3 settings - mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3" - - for s3_site in "$PXF_BASE/servers/s3/s3-site.xml" "$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do - mkdir -p "$(dirname "$s3_site")" - cat > "$s3_site" <<'EOF' - - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.access.key - admin - - - fs.s3a.secret.key - password - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - - -EOF - done - mkdir -p /home/gpadmin/.aws/ - cat > "/home/gpadmin/.aws/credentials" <<'EOF' -[default] -aws_access_key_id = admin -aws_secret_access_key = password EOF } @@ -556,11 +510,6 @@ start_hive_services() { done } -deploy_minio() { - log "deploying MinIO" - bash "${COMMON_SCRIPTS}/start_minio.bash" -} - main() { detect_java_paths setup_locale_and_packages @@ -570,7 +519,6 @@ main() { build_pxf configure_pxf prepare_hadoop_stack - deploy_minio health_check log "entrypoint finished; environment ready for tests" } diff --git a/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh index e25f7bb23..1551a7b50 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh @@ -61,15 +61,6 @@ export COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DATA_DIRECTORY:-/home/gpadmin/wo # set cloudberry timezone utc export PGTZ=UTC -# -------------------------------------------------------------------- -# Minio defaults -# -------------------------------------------------------------------- -export AWS_ACCESS_KEY_ID=admin -export AWS_SECRET_ACCESS_KEY=password -export PROTOCOL=minio -export ACCESS_KEY_ID=admin -export SECRET_ACCESS_KEY=password - # -------------------------------------------------------------------- # PXF defaults # -------------------------------------------------------------------- diff --git a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh index 40bd2bd09..5e78d40ec 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh @@ -166,15 +166,6 @@ ensure_hive_ready() { return 1 } -ensure_minio_bucket() { - local mc_bin="/home/gpadmin/workspace/mc" - if [ -x "${mc_bin}" ]; then - ${mc_bin} alias set local http://localhost:9000 admin password >/dev/null 2>&1 || true - ${mc_bin} mb local/gpdb-ud-scratch --ignore-existing >/dev/null 2>&1 || true - ${mc_bin} policy set download local/gpdb-ud-scratch >/dev/null 2>&1 || true - fi -} - set_xml_property() { local file="$1" name="$2" value="$3" if [ ! -f "${file}" ]; then @@ -201,196 +192,6 @@ ensure_yarn_vmem_settings() { set_xml_property "${yarn_site}" "yarn.nodemanager.vmem-pmem-ratio" "4.0" } -ensure_hadoop_s3a_config() { - local core_site="${HADOOP_CONF_DIR}/core-site.xml" - if [ -f "${core_site}" ] && ! grep -q "fs.s3a.endpoint" "${core_site}"; then - perl -0777 -pe ' -s## - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.access.key - '"${AWS_ACCESS_KEY_ID}"' - - - fs.s3a.secret.key - '"${AWS_SECRET_ACCESS_KEY}"' - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - -#' -i "${core_site}" - fi -} - -# Configure dedicated PXF server "s3" pointing to local MinIO; -# used by tests that explicitly set server=s3 -configure_pxf_s3_server() { - local server_dir="${PXF_BASE}/servers/s3" - mkdir -p "${server_dir}" - cat > "${server_dir}/s3-site.xml" < - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - - - fs.s3a.access.key - ${AWS_ACCESS_KEY_ID} - - - fs.s3a.secret.key - ${AWS_SECRET_ACCESS_KEY} - - -EOF - cat > "${server_dir}/core-site.xml" < - - - fs.defaultFS - s3a:// - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - - - fs.s3a.access.key - ${AWS_ACCESS_KEY_ID} - - - fs.s3a.secret.key - ${AWS_SECRET_ACCESS_KEY} - - -EOF -} - -# Configure default PXF server to point to local MinIO with explicit creds; -# used by tests that do NOT pass a server=name parameter (default server path) -configure_pxf_default_s3_server() { - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-admin} - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-password} - local default_s3_site="${PXF_BASE}/servers/default/s3-site.xml" - if [ -f "${default_s3_site}" ]; then - cat > "${default_s3_site}" < - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.access.key - ${AWS_ACCESS_KEY_ID} - - - fs.s3a.secret.key - ${AWS_SECRET_ACCESS_KEY} - - -EOF - cat > "${PXF_BASE}/servers/default/core-site.xml" < - - - fs.defaultFS - s3a:// - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - - - fs.s3a.access.key - ${AWS_ACCESS_KEY_ID} - - - fs.s3a.secret.key - ${AWS_SECRET_ACCESS_KEY} - - -EOF - # hide HDFS/Hive configs so default server is treated as S3-only - for f in hdfs-site.xml mapred-site.xml yarn-site.xml hive-site.xml hbase-site.xml; do - [ -f "${PXF_BASE}/servers/default/${f}" ] && rm -f "${PXF_BASE}/servers/default/${f}" - done - "${PXF_HOME}/bin/pxf" restart >/dev/null - fi -} - # Ensure proxy tests can login as testuser from localhost. ensure_testuser_pg_hba() { local pg_hba="/home/gpadmin/workspace/cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/pg_hba.conf" @@ -429,7 +230,6 @@ ensure_testuser_pg_hba() { } base_test(){ - # keep PROTOCOL empty so tests use HDFS; we'll set minio only for s3 later export PROTOCOL= # ensure gpdb connections target localhost over IPv4 for proxy tests export PGHOST=127.0.0.1 @@ -484,16 +284,6 @@ base_test(){ make GROUP="unused" || true save_test_reports "unused" echo "[run_tests] GROUP=unused finished" - - ensure_minio_bucket - ensure_hadoop_s3a_config - configure_pxf_s3_server - configure_pxf_default_s3_server - export PROTOCOL=s3 - export HADOOP_OPTIONAL_TOOLS=hadoop-aws - make GROUP="s3" || true - save_test_reports "s3" - echo "[run_tests] GROUP=s3 finished" } # Restore default PXF server to local HDFS/Hive/HBase configuration @@ -571,14 +361,7 @@ feature_test(){ cleanup_hive_state cleanup_hbase_state - # Prepare MinIO/S3 and restore default server to local HDFS/Hive/HBase - ensure_minio_bucket - ensure_hadoop_s3a_config - configure_pxf_s3_server configure_pxf_default_hdfs_server - # Only set default server to MinIO when explicitly running S3 groups; keeping - # it HDFS-backed avoids hijacking Hive/HDFS tests with fs.defaultFS=s3a:// - #configure_pxf_default_s3_server export PROTOCOL= make GROUP="features" || true @@ -758,7 +541,7 @@ generate_test_summary() { local group=$(basename "$group_dir") # Skip if it's not a test group directory - [[ "$group" =~ ^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|s3|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2|fdw|gpdb_fdw)$ ]] || continue + [[ "$group" =~ ^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2|fdw|gpdb_fdw)$ ]] || continue echo "Processing $group test reports from $group_dir" @@ -916,16 +699,6 @@ run_single_group() { make GROUP="hbase" save_test_reports "hbase" ;; - s3) - ensure_minio_bucket - ensure_hadoop_s3a_config - configure_pxf_s3_server - configure_pxf_default_s3_server - export PROTOCOL=s3 - export HADOOP_OPTIONAL_TOOLS=hadoop-aws - make GROUP="s3" - save_test_reports "s3" - ;; features) feature_test ;; @@ -956,7 +729,7 @@ run_single_group() { ;; *) echo "Unknown test group: $group" - echo "Available groups: cli, external-table, fdw, server, sanity, smoke, hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, s3, features, gpdb, gpdb_fdw, load, performance, bench, pxf_extension" + echo "Available groups: cli, external-table, fdw, server, sanity, smoke, hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, features, gpdb, gpdb_fdw, load, performance, bench, pxf_extension" exit 1 ;; esac diff --git a/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash b/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash deleted file mode 100755 index 896c6d7f9..000000000 --- a/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# -------------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed -# with this work for additional information regarding copyright -# ownership. The ASF licenses this file to You under the Apache -# License, Version 2.0 (the "License"); you may not use this file -# except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. -# -# -------------------------------------------------------------------- - -set -e - -WORKSPACE_DIR=${WORKSPACE_DIR:-/home/gpadmin/workspace} -MINIO_BIN=${WORKSPACE_DIR}/minio -MC_BIN=${WORKSPACE_DIR}/mc -MINIO_DATA_DIR=${MINIO_DATA_DIR:-${WORKSPACE_DIR}/minio-data} -MINIO_PORT=${MINIO_PORT:-9000} -MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT:-9001} - -export MINIO_ROOT_USER=${MINIO_ROOT_USER:-admin} -export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-password} -export MINIO_API_SELECT_PARQUET=${MINIO_API_SELECT_PARQUET:-on} - -echo "MinIO credentials: rootUser=${MINIO_ROOT_USER} rootPassword=${MINIO_ROOT_PASSWORD}" - -mkdir -p ${MINIO_DATA_DIR} - -echo "Starting MinIO server on port ${MINIO_PORT}..." -${MINIO_BIN} server ${MINIO_DATA_DIR} \ - --address ":${MINIO_PORT}" \ - --console-address ":${MINIO_CONSOLE_PORT}" & - -MINIO_PID=$! -echo "MinIO started with PID: ${MINIO_PID}" - -sleep 3 - -echo "Creating test bucket 'gpdb-ud-scratch'..." -${MC_BIN} alias set local http://localhost:${MINIO_PORT} ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} -${MC_BIN} mb local/gpdb-ud-scratch --ignore-existing - -export PROTOCOL=minio -export ACCESS_KEY_ID=${MINIO_ROOT_USER} -export SECRET_ACCESS_KEY=${MINIO_ROOT_PASSWORD} - -echo "MinIO is ready!" -echo " Console: http://localhost:${MINIO_CONSOLE_PORT}" -echo " API: http://localhost:${MINIO_PORT}" diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh b/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh index f15ee35c9..defc33284 100755 --- a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh +++ b/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh @@ -310,65 +310,6 @@ setup_ssl_material() { sudo chown gpadmin:gpadmin "${SSL_KEYSTORE}" "${SSL_TRUSTSTORE}" } -deploy_minio() { - log "deploying MinIO (for S3 tests)" - bash "${PXF_SCRIPTS}/start_minio.bash" -} - -configure_pxf_s3() { - log "configuring S3 server definitions for PXF" - local servers_base=${PXF_BASE:-/home/gpadmin/pxf-base} - local pxf_conf=/usr/local/pxf/conf - local s3_sites=( - "${servers_base}/servers/s3/s3-site.xml" - "${servers_base}/servers/default/s3-site.xml" - "${pxf_conf}/servers/s3/s3-site.xml" - ) - for s3_site in "${s3_sites[@]}"; do - mkdir -p "$(dirname "${s3_site}")" - cat > "${s3_site}" <<'EOF' - - - - fs.s3a.endpoint - http://localhost:9000 - - - fs.s3a.access.key - admin - - - fs.s3a.secret.key - password - - - fs.s3a.path.style.access - true - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider - - -EOF - done - - mkdir -p /home/gpadmin/.aws/ - cat > "/home/gpadmin/.aws/credentials" <<'EOF' -[default] -aws_access_key_id = admin -aws_secret_access_key = password -EOF -} - prepare_sut() { # Generate SUT pointing to container FQDN and overwrite build outputs to avoid localhost. local host_fqdn_local=$1 @@ -456,7 +397,7 @@ prepare_hadoop_conf_for_tests() { local conf_dir=${HADOOP_CONF_DIR:-/home/gpadmin/workspace/singlecluster/hadoop/etc/hadoop} local target_base=${REPO_ROOT}/automation/target mkdir -p "${target_base}/test-classes" "${target_base}/classes" - for f in core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml ssl-client.xml ssl-server.xml s3-site.xml; do + for f in core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml ssl-client.xml ssl-server.xml; do if [ -f "${conf_dir}/${f}" ]; then cp "${conf_dir}/${f}" "${target_base}/test-classes/${f}" cp "${conf_dir}/${f}" "${target_base}/classes/${f}" @@ -468,12 +409,6 @@ prepare_hadoop_conf_for_tests() { cp "${hbase_site}" "${target_base}/test-classes/hbase-site.xml" cp "${hbase_site}" "${target_base}/classes/hbase-site.xml" fi - # Add PXF server S3 configs to the classpath for automation tests. - local pxf_s3="${PXF_BASE:-/home/gpadmin/pxf-base}/servers/s3/s3-site.xml" - if [ -f "${pxf_s3}" ]; then - cp "${pxf_s3}" "${target_base}/test-classes/s3-site.xml" - cp "${pxf_s3}" "${target_base}/classes/s3-site.xml" - fi } configure_hadoop() { @@ -507,12 +442,6 @@ configure_hadoop() { hadoop.proxyuser.gpadmin.groups* hadoop.proxyuser.porter.hosts* hadoop.proxyuser.porter.groups* - fs.s3a.endpointhttp://localhost:9000 - fs.s3a.path.style.accesstrue - fs.s3a.connection.ssl.enabledfalse - fs.s3a.access.key${AWS_ACCESS_KEY_ID:-admin} - fs.s3a.secret.key${AWS_SECRET_ACCESS_KEY:-password} - fs.s3a.aws.credentials.providerorg.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider EOF @@ -1102,14 +1031,11 @@ init_test_env() { export KRB5CCNAME=${KRB5CCNAME:-/tmp/krb5cc_pxf_automation} export PXF_TEST_KEEP_DATA=${PXF_TEST_KEEP_DATA:-true} unset HADOOP_USER_NAME - local s3_opts="-Dfs.s3a.endpoint=http://localhost:9000 -Dfs.s3a.path.style.access=true -Dfs.s3a.connection.ssl.enabled=false -Dfs.s3a.access.key=${AWS_ACCESS_KEY_ID:-admin} -Dfs.s3a.secret.key=${AWS_SECRET_ACCESS_KEY:-password}" export HDFS_URI="hdfs://${HOST_FQDN_LOCAL}:8020" - export HADOOP_OPTS="-Dfs.defaultFS=${HDFS_URI} -Dhadoop.security.authentication=kerberos ${s3_opts}" + export HADOOP_OPTS="-Dfs.defaultFS=${HDFS_URI} -Dhadoop.security.authentication=kerberos" export HADOOP_CLIENT_OPTS="${HADOOP_OPTS}" - export MAVEN_OPTS="-Dfs.defaultFS=${HDFS_URI} -Dhadoop.security.authentication=kerberos ${s3_opts} -Dpxf.host=${PXF_HOST} -Dpxf.port=${PXF_PORT}" + export MAVEN_OPTS="-Dfs.defaultFS=${HDFS_URI} -Dhadoop.security.authentication=kerberos -Dpxf.host=${PXF_HOST} -Dpxf.port=${PXF_PORT}" export PGOPTIONS="${PGOPTIONS:---client-min-messages=error}" - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-admin} - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-password} export EXCLUDE_GROUPS_LOCAL=${EXCLUDED_GROUPS:-multiClusterSecurity} DEFAULT_MAVEN_TEST_OPTS="-Dpxf.host=${PXF_HOST} -Dpxf.port=${PXF_PORT} -DPXF_SINGLE_NODE=true -DexcludedGroups=${EXCLUDE_GROUPS_LOCAL}" } @@ -1278,8 +1204,6 @@ main() { configure_hbase configure_pxf configure_pxf_servers - configure_pxf_s3 - deploy_minio configure_pg_hba start_hdfs_secure start_hive_secure diff --git a/ci/singlecluster/Dockerfile b/ci/singlecluster/Dockerfile index 931d5d3f8..aa30c2ae4 100644 --- a/ci/singlecluster/Dockerfile +++ b/ci/singlecluster/Dockerfile @@ -104,20 +104,6 @@ RUN set -e; \ wget -O go.tgz -q "https://go.dev/dl/go${GO_VERSION}.linux-${go_arch}.tar.gz" && \ sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go.tgz && rm go.tgz -# Install MinIO and mc -RUN set -e; \ - arch="${TARGETARCH:-$(uname -m)}"; \ - case "$arch" in \ - amd64|x86_64) minio_arch="amd64" ;; \ - arm64|aarch64) minio_arch="arm64" ;; \ - *) echo "Unsupported architecture: ${arch}"; exit 1 ;; \ - esac; \ - mkdir -p /home/gpadmin/workspace && \ - wget -O /home/gpadmin/workspace/minio "https://dl.min.io/server/minio/release/linux-${minio_arch}/minio" && \ - wget -O /home/gpadmin/workspace/mc "https://dl.min.io/client/mc/release/linux-${minio_arch}/mc" && \ - chmod +x /home/gpadmin/workspace/minio /home/gpadmin/workspace/mc - - COPY ./templates $GPHD_ROOT COPY ./conf $GPHD_ROOT/conf COPY ./bin $GPHD_ROOT/bin \ No newline at end of file