messageSizeDistribution;
+
+ /**
+ * Returns true if this workload uses a size distribution instead of fixed size.
+ *
+ * @return true if messageSizeDistribution is configured, false otherwise
+ */
+ public boolean usesDistribution() {
+ return messageSizeDistribution != null && !messageSizeDistribution.isEmpty();
+ }
+
public boolean useRandomizedPayloads;
public double randomBytesRatio;
public int randomizedPayloadPoolSize;
@@ -48,13 +62,25 @@ public class Workload {
public int producerRate;
/**
- * If the consumer backlog is > 0, the generator will accumulate messages until the requested amount of storage is
- * retained and then it will start the consumers to drain it.
+ * If the consumer backlog is > 0, the generator will accumulate messages until the requested
+ * amount of storage is retained and then it will start the consumers to drain it.
*
- * The testDurationMinutes will be overruled to allow the test to complete when the consumer has drained all the
- * backlog and it's on par with the producer
+ * The testDurationMinutes will be overruled to allow the test to complete when the consumer
+ * has drained all the backlog and it's on par with the producer
*/
public long consumerBacklogSizeGB = 0;
+ /**
+ * The ratio of the backlog that can remain and yet the backlog still be considered empty, and
+ * thus the workload can complete at the end of the configured duration. In some systems it is not
+ * feasible for the backlog to be drained fully and thus the workload will run indefinitely. In
+ * such circumstances, one may be content to achieve a partial drain such as 99% of the backlog.
+ * The value should be on somewhere between 0.0 and 1.0, where 1.0 indicates that the backlog
+ * should be fully drained, and 0.0 indicates a best effort, where the workload will complete
+ * after the specified time irrespective of how much of the backlog has been drained.
+ */
+ public double backlogDrainRatio = 1.0;
public int testDurationMinutes;
+
+ public int warmupDurationMinutes = 1;
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java
index 592bfa973..86a2b1a73 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java
@@ -1,42 +1,26 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark;
-import io.openmessaging.benchmark.utils.RandomGenerator;
-import java.io.IOException;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.util.concurrent.TimeUnit.MINUTES;
import io.netty.util.concurrent.DefaultThreadFactory;
import io.openmessaging.benchmark.utils.PaddingDecimalFormat;
+import io.openmessaging.benchmark.utils.RandomGenerator;
import io.openmessaging.benchmark.utils.Timer;
import io.openmessaging.benchmark.utils.payload.FilePayloadReader;
+import io.openmessaging.benchmark.utils.payload.MessageSizeDistribution;
import io.openmessaging.benchmark.utils.payload.PayloadReader;
import io.openmessaging.benchmark.worker.Worker;
import io.openmessaging.benchmark.worker.commands.ConsumerAssignment;
@@ -46,6 +30,18 @@
import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment;
import io.openmessaging.benchmark.worker.commands.TopicSubscription;
import io.openmessaging.benchmark.worker.commands.TopicsInfo;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.ArrayUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class WorkloadGenerator implements AutoCloseable {
@@ -53,8 +49,8 @@ public class WorkloadGenerator implements AutoCloseable {
private final Workload workload;
private final Worker worker;
- private final ExecutorService executor = Executors
- .newCachedThreadPool(new DefaultThreadFactory("messaging-benchmark"));
+ private final ExecutorService executor =
+ Executors.newCachedThreadPool(new DefaultThreadFactory("messaging-benchmark"));
private volatile boolean runCompleted = false;
private volatile boolean needToWaitForBacklogDraining = false;
@@ -67,13 +63,15 @@ public WorkloadGenerator(String driverName, Workload workload, Worker worker) {
this.worker = worker;
if (workload.consumerBacklogSizeGB > 0 && workload.producerRate == 0) {
- throw new IllegalArgumentException("Cannot probe producer sustainable rate when building backlog");
+ throw new IllegalArgumentException(
+ "Cannot probe producer sustainable rate when building backlog");
}
}
public TestResult run() throws Exception {
Timer timer = new Timer();
- List topics = worker.createTopics(new TopicsInfo(workload.topics, workload.partitionsPerTopic));
+ List topics =
+ worker.createTopics(new TopicsInfo(workload.topics, workload.partitionsPerTopic));
log.info("Created {} topics in {} ms", topics.size(), timer.elapsedMillis());
createConsumers(topics);
@@ -87,60 +85,86 @@ public TestResult run() throws Exception {
// Producer rate is 0 and we need to discover the sustainable rate
targetPublishRate = 10000;
- executor.execute(() -> {
- // Run background controller to adjust rate
- try {
- findMaximumSustainableRate(targetPublishRate);
- } catch (IOException e) {
- log.warn("Failure in finding max sustainable rate", e);
- }
- });
+ executor.execute(
+ () -> {
+ // Run background controller to adjust rate
+ try {
+ findMaximumSustainableRate(targetPublishRate);
+ } catch (IOException e) {
+ log.warn("Failure in finding max sustainable rate", e);
+ }
+ });
}
- final PayloadReader payloadReader = new FilePayloadReader(workload.messageSize);
-
ProducerWorkAssignment producerWorkAssignment = new ProducerWorkAssignment();
producerWorkAssignment.keyDistributorType = workload.keyDistributor;
producerWorkAssignment.publishRate = targetPublishRate;
producerWorkAssignment.payloadData = new ArrayList<>();
- if(workload.useRandomizedPayloads) {
- // create messages that are part random and part zeros
- // better for testing effects of compression
+ if (workload.usesDistribution()) {
+ // Distribution mode: create one payload per bucket with weighted selection at runtime
+ MessageSizeDistribution dist = new MessageSizeDistribution(workload.messageSizeDistribution);
+ List sizes = dist.getBucketMaxSizes();
+ Random r = new Random();
+
+ log.info(
+ "Creating {} payloads for size distribution (max sizes: {}, weighted avg: {} bytes)",
+ sizes.size(),
+ sizes,
+ dist.getAvgSize());
+
+ for (int size : sizes) {
+ byte[] payload = new byte[size];
+ if (workload.useRandomizedPayloads) {
+ int randomBytes = (int) (size * workload.randomBytesRatio);
+ r.nextBytes(payload);
+ // Zero out non-random portion for compressibility testing
+ for (int j = randomBytes; j < size; j++) {
+ payload[j] = 0;
+ }
+ }
+ producerWorkAssignment.payloadData.add(payload);
+ }
+ producerWorkAssignment.payloadWeights = dist.getWeights();
+
+ } else if (workload.useRandomizedPayloads) {
+ // Existing fixed-size randomized payload logic
Random r = new Random();
- int randomBytes = (int)(workload.messageSize * workload.randomBytesRatio);
+ int randomBytes = (int) (workload.messageSize * workload.randomBytesRatio);
int zerodBytes = workload.messageSize - randomBytes;
- for(int i = 0; i 0) {
+ log.info("----- Starting warm-up traffic ({}m) ------", workload.warmupDurationMinutes);
+ printAndCollectStats(workload.warmupDurationMinutes, TimeUnit.MINUTES);
+ }
if (workload.consumerBacklogSizeGB > 0) {
- executor.execute(() -> {
- try {
- buildAndDrainBacklog(topics);
- } catch (IOException e) {
- e.printStackTrace();
- }
- });
+ executor.execute(
+ () -> {
+ try {
+ buildAndDrainBacklog(workload.testDurationMinutes);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ });
}
worker.resetStats();
- log.info("----- Starting benchmark traffic ------");
+ log.info("----- Starting benchmark traffic ({}m)------", workload.testDurationMinutes);
TestResult result = printAndCollectStats(workload.testDurationMinutes, TimeUnit.MINUTES);
runCompleted = true;
@@ -151,7 +175,8 @@ public TestResult run() throws Exception {
private void ensureTopicsAreReady() throws IOException {
log.info("Waiting for consumers to be ready");
- // This is work around the fact that there's no way to have a consumer ready in Kafka without first publishing
+ // This is work around the fact that there's no way to have a consumer ready in Kafka without
+ // first publishing
// some message on the topic, which will then trigger the partitions assignment to the consumers
int expectedMessages = workload.topics * workload.subscriptionsPerTopic;
@@ -159,12 +184,18 @@ private void ensureTopicsAreReady() throws IOException {
// In this case we just publish 1 message and then wait for consumers to receive the data
worker.probeProducers();
- while (true) {
+ long start = System.currentTimeMillis();
+ long end = start + 60 * 1000;
+ while (System.currentTimeMillis() < end) {
CountersStats stats = worker.getCountersStats();
-
+
+ log.info(
+ "Waiting for topics to be ready -- Sent: {}, Received: {}",
+ stats.messagesSent,
+ stats.messagesReceived);
if (stats.messagesReceived < expectedMessages) {
try {
- Thread.sleep(100);
+ Thread.sleep(2_000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -173,26 +204,26 @@ private void ensureTopicsAreReady() throws IOException {
}
}
- log.info("All consumers are ready");
+ if (System.currentTimeMillis() >= end) {
+ throw new RuntimeException("Timed out waiting for consumers to be ready");
+ } else {
+ log.info("All consumers are ready");
+ }
}
/**
- * Adjust the publish rate to a level that is sustainable, meaning that we can consume all the messages that are
- * being produced
+ * Adjust the publish rate to a level that is sustainable, meaning that we can consume all the
+ * messages that are being produced.
+ *
+ * @param currentRate
*/
private void findMaximumSustainableRate(double currentRate) throws IOException {
- double maxRate = Double.MAX_VALUE; // Discovered max sustainable rate
- double minRate = 0.1;
-
CountersStats stats = worker.getCountersStats();
- long localTotalMessagesSentCounter = stats.messagesSent;
- long localTotalMessagesReceivedCounter = stats.messagesReceived;
-
int controlPeriodMillis = 3000;
long lastControlTimestamp = System.nanoTime();
- int successfulPeriods = 0;
+ RateController rateController = new RateController();
while (!runCompleted) {
// Check every few seconds and adjust the rate
@@ -205,84 +236,13 @@ private void findMaximumSustainableRate(double currentRate) throws IOException {
// Consider multiple copies when using multiple subscriptions
stats = worker.getCountersStats();
long currentTime = System.nanoTime();
- long totalMessagesSent = stats.messagesSent;
- long totalMessagesReceived = stats.messagesReceived;
- long messagesPublishedInPeriod = totalMessagesSent - localTotalMessagesSentCounter;
- long messagesReceivedInPeriod = totalMessagesReceived - localTotalMessagesReceivedCounter;
- double publishRateInLastPeriod = messagesPublishedInPeriod / (double) (currentTime - lastControlTimestamp)
- * TimeUnit.SECONDS.toNanos(1);
- double receiveRateInLastPeriod = messagesReceivedInPeriod / (double) (currentTime - lastControlTimestamp)
- * TimeUnit.SECONDS.toNanos(1);
-
- if (log.isDebugEnabled()) {
- log.debug(
- "total-send: {} -- total-received: {} -- int-sent: {} -- int-received: {} -- sent-rate: {} -- received-rate: {}",
- totalMessagesSent, totalMessagesReceived, messagesPublishedInPeriod, messagesReceivedInPeriod,
- publishRateInLastPeriod, receiveRateInLastPeriod);
- }
+ long periodNanos = currentTime - lastControlTimestamp;
- localTotalMessagesSentCounter = totalMessagesSent;
- localTotalMessagesReceivedCounter = totalMessagesReceived;
lastControlTimestamp = currentTime;
- if (log.isDebugEnabled()) {
- log.debug("Current rate: {} -- Publish rate {} -- Consume Rate: {} -- min-rate: {} -- max-rate: {}",
- dec.format(currentRate), dec.format(publishRateInLastPeriod),
- dec.format(receiveRateInLastPeriod), dec.format(minRate), dec.format(maxRate));
- }
-
- if (publishRateInLastPeriod < currentRate * 0.95) {
- // Producer is not able to publish as fast as requested
- maxRate = currentRate * 1.1;
- currentRate = minRate + (currentRate - minRate) / 2;
-
- log.debug("Publishers are not meeting requested rate. reducing to {}", currentRate);
- } else if (receiveRateInLastPeriod < publishRateInLastPeriod * 0.98) {
- // If the consumers are building backlog, we should slow down publish rate
- maxRate = currentRate;
- currentRate = minRate + (currentRate - minRate) / 2;
- log.debug("Consumers are not meeting requested rate. reducing to {}", currentRate);
-
- // Slows the publishes to let the consumer time to absorb the backlog
- worker.adjustPublishRate(minRate / 10);
- while (true) {
- stats = worker.getCountersStats();
- long backlog = workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived;
- if (backlog < 1000) {
- break;
- }
-
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- return;
- }
- }
-
- log.debug("Resuming load at reduced rate");
- worker.adjustPublishRate(currentRate);
-
- try {
- // Wait some more time for the publish rate to catch up
- Thread.sleep(500);
- } catch (InterruptedException e) {
- return;
- }
-
- stats = worker.getCountersStats();
- localTotalMessagesSentCounter = stats.messagesSent;
- localTotalMessagesReceivedCounter = stats.messagesReceived;
-
- } else if (currentRate < maxRate) {
- minRate = currentRate;
- currentRate = Math.min(currentRate * 2, maxRate);
- log.debug("No bottleneck found, increasing the rate to {}", currentRate);
- } else if (++successfulPeriods > 3) {
- minRate = currentRate * 0.95;
- maxRate = currentRate * 1.05;
- successfulPeriods = 0;
- }
-
+ currentRate =
+ rateController.nextRate(
+ currentRate, periodNanos, stats.messagesSent, stats.messagesReceived);
worker.adjustPublishRate(currentRate);
}
}
@@ -296,12 +256,13 @@ public void close() throws Exception {
private void createConsumers(List topics) throws IOException {
ConsumerAssignment consumerAssignment = new ConsumerAssignment();
- for(String topic: topics){
- for(int i = 0; i < workload.subscriptionsPerTopic; i++){
- String subscriptionName = String.format("sub-%03d-%s", i, RandomGenerator.getRandomString());
+ for (String topic : topics) {
+ for (int i = 0; i < workload.subscriptionsPerTopic; i++) {
+ String subscriptionName =
+ String.format("sub-%03d-%s", i, RandomGenerator.getRandomString());
for (int j = 0; j < workload.consumerPerSubscription; j++) {
- consumerAssignment.topicsSubscriptions
- .add(new TopicSubscription(topic, subscriptionName));
+ consumerAssignment.topicsSubscriptions.add(
+ new TopicSubscription(topic, subscriptionName));
}
}
}
@@ -311,7 +272,10 @@ private void createConsumers(List topics) throws IOException {
Timer timer = new Timer();
worker.createConsumers(consumerAssignment);
- log.info("Created {} consumers in {} ms", consumerAssignment.topicsSubscriptions.size(), timer.elapsedMillis());
+ log.info(
+ "Created {} consumers in {} ms",
+ consumerAssignment.topicsSubscriptions.size(),
+ timer.elapsedMillis());
}
private void createProducers(List topics) throws IOException {
@@ -319,7 +283,7 @@ private void createProducers(List topics) throws IOException {
// Add the topic multiple times, one for each producer
for (int i = 0; i < workload.producersPerTopic; i++) {
- topics.forEach(fullListOfTopics::add);
+ fullListOfTopics.addAll(topics);
}
Collections.shuffle(fullListOfTopics);
@@ -330,18 +294,26 @@ private void createProducers(List topics) throws IOException {
log.info("Created {} producers in {} ms", fullListOfTopics.size(), timer.elapsedMillis());
}
- private void buildAndDrainBacklog(List topics) throws IOException {
+ private void buildAndDrainBacklog(int testDurationMinutes) throws IOException {
+ Timer timer = new Timer();
log.info("Stopping all consumers to build backlog");
worker.pauseConsumers();
this.needToWaitForBacklogDraining = true;
+ // Use average size when distribution is configured, otherwise fixed messageSize
+ int effectiveMessageSize =
+ workload.usesDistribution()
+ ? new MessageSizeDistribution(workload.messageSizeDistribution).getAvgSize()
+ : workload.messageSize;
+
long requestedBacklogSize = workload.consumerBacklogSizeGB * 1024 * 1024 * 1024;
while (true) {
CountersStats stats = worker.getCountersStats();
- long currentBacklogSize = (workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived)
- * workload.messageSize;
+ long currentBacklogSize =
+ (workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived)
+ * effectiveMessageSize;
if (currentBacklogSize >= requestedBacklogSize) {
break;
@@ -354,17 +326,29 @@ private void buildAndDrainBacklog(List topics) throws IOException {
}
}
+ log.info("--- Completed backlog build in {} s ---", timer.elapsedSeconds());
+ timer = new Timer();
log.info("--- Start draining backlog ---");
worker.resumeConsumers();
- final long minBacklog = 1000;
+ long backlogMessageCapacity = requestedBacklogSize / effectiveMessageSize;
+ long backlogEmptyLevel = (long) ((1.0 - workload.backlogDrainRatio) * backlogMessageCapacity);
+ final long minBacklog = Math.max(1000L, backlogEmptyLevel);
while (true) {
CountersStats stats = worker.getCountersStats();
- long currentBacklog = workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived;
+ long currentBacklog =
+ workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived;
if (currentBacklog <= minBacklog) {
- log.info("--- Completed backlog draining ---");
+ log.info("--- Completed backlog draining in {} s ---", timer.elapsedSeconds());
+
+ try {
+ Thread.sleep(MINUTES.toMillis(testDurationMinutes));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
needToWaitForBacklogDraining = false;
return;
}
@@ -377,6 +361,7 @@ private void buildAndDrainBacklog(List topics) throws IOException {
}
}
+ @SuppressWarnings({"checkstyle:LineLength", "checkstyle:MethodLength"})
private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throws IOException {
long startTime = System.nanoTime();
@@ -390,7 +375,11 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw
result.driver = driverName;
result.topics = workload.topics;
result.partitions = workload.partitionsPerTopic;
- result.messageSize = workload.messageSize;
+ // Use average size when distribution is configured
+ result.messageSize =
+ workload.usesDistribution()
+ ? new MessageSizeDistribution(workload.messageSizeDistribution).getAvgSize()
+ : workload.messageSize;
result.producersPerTopic = workload.producersPerTopic;
result.consumersPerTopic = workload.consumerPerSubscription;
@@ -408,25 +397,38 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw
double publishRate = stats.messagesSent / elapsed;
double publishThroughput = stats.bytesSent / elapsed / 1024 / 1024;
+ double errorRate = stats.messageSendErrors / elapsed;
double consumeRate = stats.messagesReceived / elapsed;
double consumeThroughput = stats.bytesReceived / elapsed / 1024 / 1024;
- long currentBacklog = workload.subscriptionsPerTopic * stats.totalMessagesSent
- - stats.totalMessagesReceived;
+ long currentBacklog =
+ Math.max(
+ 0L,
+ workload.subscriptionsPerTopic * stats.totalMessagesSent
+ - stats.totalMessagesReceived);
log.info(
- "Pub rate {} msg/s / {} MB/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}",
- rateFormat.format(publishRate), throughputFormat.format(publishThroughput),
- rateFormat.format(consumeRate), throughputFormat.format(consumeThroughput),
+ "Pub rate {} msg/s / {} MB/s | Pub err {} err/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {} | Pub Delay Latency (us) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}",
+ rateFormat.format(publishRate),
+ throughputFormat.format(publishThroughput),
+ rateFormat.format(errorRate),
+ rateFormat.format(consumeRate),
+ throughputFormat.format(consumeThroughput),
dec.format(currentBacklog / 1000.0), //
dec.format(microsToMillis(stats.publishLatency.getMean())),
dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(50))),
dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(99))),
dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(99.9))),
- throughputFormat.format(microsToMillis(stats.publishLatency.getMaxValue())));
+ throughputFormat.format(microsToMillis(stats.publishLatency.getMaxValue())),
+ dec.format(stats.publishDelayLatency.getMean()),
+ dec.format(stats.publishDelayLatency.getValueAtPercentile(50)),
+ dec.format(stats.publishDelayLatency.getValueAtPercentile(99)),
+ dec.format(stats.publishDelayLatency.getValueAtPercentile(99.9)),
+ throughputFormat.format(stats.publishDelayLatency.getMaxValue()));
result.publishRate.add(publishRate);
+ result.publishErrorRate.add(errorRate);
result.consumeRate.add(consumeRate);
result.backlog.add(currentBacklog);
result.publishLatencyAvg.add(microsToMillis(stats.publishLatency.getMean()));
@@ -434,58 +436,119 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw
result.publishLatency75pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(75)));
result.publishLatency95pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(95)));
result.publishLatency99pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99)));
- result.publishLatency999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.9)));
- result.publishLatency9999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.99)));
+ result.publishLatency999pct.add(
+ microsToMillis(stats.publishLatency.getValueAtPercentile(99.9)));
+ result.publishLatency9999pct.add(
+ microsToMillis(stats.publishLatency.getValueAtPercentile(99.99)));
result.publishLatencyMax.add(microsToMillis(stats.publishLatency.getMaxValue()));
+ result.publishDelayLatencyAvg.add(stats.publishDelayLatency.getMean());
+ result.publishDelayLatency50pct.add(stats.publishDelayLatency.getValueAtPercentile(50));
+ result.publishDelayLatency75pct.add(stats.publishDelayLatency.getValueAtPercentile(75));
+ result.publishDelayLatency95pct.add(stats.publishDelayLatency.getValueAtPercentile(95));
+ result.publishDelayLatency99pct.add(stats.publishDelayLatency.getValueAtPercentile(99));
+ result.publishDelayLatency999pct.add(stats.publishDelayLatency.getValueAtPercentile(99.9));
+ result.publishDelayLatency9999pct.add(stats.publishDelayLatency.getValueAtPercentile(99.99));
+ result.publishDelayLatencyMax.add(stats.publishDelayLatency.getMaxValue());
+
result.endToEndLatencyAvg.add(microsToMillis(stats.endToEndLatency.getMean()));
- result.endToEndLatency50pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(50)));
- result.endToEndLatency75pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(75)));
- result.endToEndLatency95pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(95)));
- result.endToEndLatency99pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99)));
- result.endToEndLatency999pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.9)));
- result.endToEndLatency9999pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.99)));
+ result.endToEndLatency50pct.add(
+ microsToMillis(stats.endToEndLatency.getValueAtPercentile(50)));
+ result.endToEndLatency75pct.add(
+ microsToMillis(stats.endToEndLatency.getValueAtPercentile(75)));
+ result.endToEndLatency95pct.add(
+ microsToMillis(stats.endToEndLatency.getValueAtPercentile(95)));
+ result.endToEndLatency99pct.add(
+ microsToMillis(stats.endToEndLatency.getValueAtPercentile(99)));
+ result.endToEndLatency999pct.add(
+ microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.9)));
+ result.endToEndLatency9999pct.add(
+ microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.99)));
result.endToEndLatencyMax.add(microsToMillis(stats.endToEndLatency.getMaxValue()));
if (now >= testEndTime && !needToWaitForBacklogDraining) {
CumulativeLatencies agg = worker.getCumulativeLatencies();
log.info(
- "----- Aggregated Pub Latency (ms) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {}",
+ "----- Aggregated Pub Latency (ms) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {} | Pub Delay (us) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {}",
dec.format(agg.publishLatency.getMean() / 1000.0),
dec.format(agg.publishLatency.getValueAtPercentile(50) / 1000.0),
dec.format(agg.publishLatency.getValueAtPercentile(95) / 1000.0),
dec.format(agg.publishLatency.getValueAtPercentile(99) / 1000.0),
dec.format(agg.publishLatency.getValueAtPercentile(99.9) / 1000.0),
dec.format(agg.publishLatency.getValueAtPercentile(99.99) / 1000.0),
- throughputFormat.format(agg.publishLatency.getMaxValue() / 1000.0));
+ throughputFormat.format(agg.publishLatency.getMaxValue() / 1000.0),
+ dec.format(agg.publishDelayLatency.getMean()),
+ dec.format(agg.publishDelayLatency.getValueAtPercentile(50)),
+ dec.format(agg.publishDelayLatency.getValueAtPercentile(95)),
+ dec.format(agg.publishDelayLatency.getValueAtPercentile(99)),
+ dec.format(agg.publishDelayLatency.getValueAtPercentile(99.9)),
+ dec.format(agg.publishDelayLatency.getValueAtPercentile(99.99)),
+ throughputFormat.format(agg.publishDelayLatency.getMaxValue()));
result.aggregatedPublishLatencyAvg = agg.publishLatency.getMean() / 1000.0;
result.aggregatedPublishLatency50pct = agg.publishLatency.getValueAtPercentile(50) / 1000.0;
result.aggregatedPublishLatency75pct = agg.publishLatency.getValueAtPercentile(75) / 1000.0;
result.aggregatedPublishLatency95pct = agg.publishLatency.getValueAtPercentile(95) / 1000.0;
result.aggregatedPublishLatency99pct = agg.publishLatency.getValueAtPercentile(99) / 1000.0;
- result.aggregatedPublishLatency999pct = agg.publishLatency.getValueAtPercentile(99.9) / 1000.0;
- result.aggregatedPublishLatency9999pct = agg.publishLatency.getValueAtPercentile(99.99) / 1000.0;
+ result.aggregatedPublishLatency999pct =
+ agg.publishLatency.getValueAtPercentile(99.9) / 1000.0;
+ result.aggregatedPublishLatency9999pct =
+ agg.publishLatency.getValueAtPercentile(99.99) / 1000.0;
result.aggregatedPublishLatencyMax = agg.publishLatency.getMaxValue() / 1000.0;
- result.aggregatedEndToEndLatencyAvg = agg.endToEndLatency.getMean() / 1000.0;
- result.aggregatedEndToEndLatency50pct = agg.endToEndLatency.getValueAtPercentile(50) / 1000.0;
- result.aggregatedEndToEndLatency75pct = agg.endToEndLatency.getValueAtPercentile(75) / 1000.0;
- result.aggregatedEndToEndLatency95pct = agg.endToEndLatency.getValueAtPercentile(95) / 1000.0;
- result.aggregatedEndToEndLatency99pct = agg.endToEndLatency.getValueAtPercentile(99) / 1000.0;
- result.aggregatedEndToEndLatency999pct = agg.endToEndLatency.getValueAtPercentile(99.9) / 1000.0;
- result.aggregatedEndToEndLatency9999pct = agg.endToEndLatency.getValueAtPercentile(99.99) / 1000.0;
- result.aggregatedEndToEndLatencyMax = agg.endToEndLatency.getMaxValue() / 1000.0;
-
- agg.publishLatency.percentiles(100).forEach(value -> {
- result.aggregatedPublishLatencyQuantiles.put(value.getPercentile(),
- value.getValueIteratedTo() / 1000.0);
- });
-
- agg.endToEndLatency.percentiles(100).forEach(value -> {
- result.aggregatedEndToEndLatencyQuantiles.put(value.getPercentile(),
- microsToMillis(value.getValueIteratedTo()));
- });
+ result.aggregatedPublishDelayLatencyAvg = agg.publishDelayLatency.getMean();
+ result.aggregatedPublishDelayLatency50pct =
+ agg.publishDelayLatency.getValueAtPercentile(50);
+ result.aggregatedPublishDelayLatency75pct =
+ agg.publishDelayLatency.getValueAtPercentile(75);
+ result.aggregatedPublishDelayLatency95pct =
+ agg.publishDelayLatency.getValueAtPercentile(95);
+ result.aggregatedPublishDelayLatency99pct =
+ agg.publishDelayLatency.getValueAtPercentile(99);
+ result.aggregatedPublishDelayLatency999pct =
+ agg.publishDelayLatency.getValueAtPercentile(99.9);
+ result.aggregatedPublishDelayLatency9999pct =
+ agg.publishDelayLatency.getValueAtPercentile(99.99);
+ result.aggregatedPublishDelayLatencyMax = agg.publishDelayLatency.getMaxValue();
+
+ result.aggregatedEndToEndLatencyAvg = agg.endToEndLatency.getMean() / 1000.0;
+ result.aggregatedEndToEndLatency50pct =
+ agg.endToEndLatency.getValueAtPercentile(50) / 1000.0;
+ result.aggregatedEndToEndLatency75pct =
+ agg.endToEndLatency.getValueAtPercentile(75) / 1000.0;
+ result.aggregatedEndToEndLatency95pct =
+ agg.endToEndLatency.getValueAtPercentile(95) / 1000.0;
+ result.aggregatedEndToEndLatency99pct =
+ agg.endToEndLatency.getValueAtPercentile(99) / 1000.0;
+ result.aggregatedEndToEndLatency999pct =
+ agg.endToEndLatency.getValueAtPercentile(99.9) / 1000.0;
+ result.aggregatedEndToEndLatency9999pct =
+ agg.endToEndLatency.getValueAtPercentile(99.99) / 1000.0;
+ result.aggregatedEndToEndLatencyMax = agg.endToEndLatency.getMaxValue() / 1000.0;
+
+ agg.publishLatency
+ .percentiles(100)
+ .forEach(
+ value -> {
+ result.aggregatedPublishLatencyQuantiles.put(
+ value.getPercentile(), value.getValueIteratedTo() / 1000.0);
+ });
+
+ agg.publishDelayLatency
+ .percentiles(100)
+ .forEach(
+ value -> {
+ result.aggregatedPublishDelayLatencyQuantiles.put(
+ value.getPercentile(), value.getValueIteratedTo());
+ });
+
+ agg.endToEndLatency
+ .percentiles(100)
+ .forEach(
+ value -> {
+ result.aggregatedEndToEndLatencyQuantiles.put(
+ value.getPercentile(), microsToMillis(value.getValueIteratedTo()));
+ });
break;
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java
new file mode 100644
index 000000000..366b2ca12
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+
+import java.util.Optional;
+import java.util.function.Function;
+
+public final class Env {
+ private Env() {}
+
+ public static long getLong(String key, long defaultValue) {
+ return get(key, Long::parseLong, defaultValue);
+ }
+
+ public static double getDouble(String key, double defaultValue) {
+ return get(key, Double::parseDouble, defaultValue);
+ }
+
+ public static T get(String key, Function function, T defaultValue) {
+ return Optional.ofNullable(System.getenv(key)).map(function).orElse(defaultValue);
+ }
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java
index 37b679e37..01a1ae51e 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java
@@ -1,35 +1,31 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils;
+
import java.util.ArrayList;
import java.util.List;
public class ListPartition {
/**
- * partition a list to specified size
+ * partition a list to specified size.
*
* @param originList
* @param size
* @param
- * @return
+ * @return the partitioned list
*/
public static List> partitionList(List originList, int size) {
@@ -60,5 +56,4 @@ public static List> partitionList(List originList, int size) {
}
return resultList;
}
-
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java
index 5fef088d1..a598ddb2a 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java
@@ -1,24 +1,19 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils;
+
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.FieldPosition;
@@ -28,8 +23,11 @@ public class PaddingDecimalFormat extends DecimalFormat {
private int minimumLength;
/**
- * Creates a PaddingDecimalFormat using the given pattern and minimum minimumLength and the symbols for the default
- * locale.
+ * Creates a PaddingDecimalFormat using the given pattern and minimum minimumLength and the
+ * symbols for the default locale.
+ *
+ * @param pattern
+ * @param minLength
*/
public PaddingDecimalFormat(String pattern, int minLength) {
super(pattern);
@@ -38,6 +36,10 @@ public PaddingDecimalFormat(String pattern, int minLength) {
/**
* Creates a PaddingDecimalFormat using the given pattern, symbols and minimum minimumLength.
+ *
+ * @param pattern
+ * @param symbols
+ * @param minLength
*/
public PaddingDecimalFormat(String pattern, DecimalFormatSymbols symbols, int minLength) {
super(pattern, symbols);
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java
index 563377f19..13c4f501c 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils;
+
import com.google.common.io.BaseEncoding;
import java.util.Random;
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java
index 533ca92e6..bb3ac28c0 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java
@@ -1,34 +1,45 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils;
+
import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
public class Timer {
private final long startTime;
+ private final Supplier nanoClock;
+
+ Timer(Supplier nanoClock) {
+ this.nanoClock = nanoClock;
+ startTime = this.nanoClock.get();
+ }
public Timer() {
- startTime = System.nanoTime();
+ this(System::nanoTime);
}
public double elapsedMillis() {
- long now = System.nanoTime();
- return (now - startTime) / (double) TimeUnit.MILLISECONDS.toNanos(1);
+ return elapsed(TimeUnit.MILLISECONDS);
+ }
+
+ public double elapsedSeconds() {
+ return elapsed(TimeUnit.SECONDS);
+ }
+
+ private double elapsed(TimeUnit unit) {
+ long now = nanoClock.get();
+ return (now - startTime) / (double) unit.toNanos(1);
}
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java
new file mode 100644
index 000000000..1fc69b15c
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.locks.LockSupport;
+import java.util.function.Supplier;
+
+/**
+ * Provides a next operation time for rate limited operation streams.
+ * The rate limiter is thread safe and can be shared by all threads.
+ */
+public final class UniformRateLimiter {
+
+ private static final AtomicLongFieldUpdater V_TIME_UPDATER =
+ AtomicLongFieldUpdater.newUpdater(UniformRateLimiter.class, "virtualTime");
+ private static final AtomicLongFieldUpdater START_UPDATER =
+ AtomicLongFieldUpdater.newUpdater(UniformRateLimiter.class, "start");
+ private static final double ONE_SEC_IN_NS = SECONDS.toNanos(1);
+ private volatile long start = Long.MIN_VALUE;
+ private volatile long virtualTime;
+ private final double opsPerSec;
+ private final long intervalNs;
+ private final Supplier nanoClock;
+
+ UniformRateLimiter(final double opsPerSec, Supplier nanoClock) {
+ if (Double.isNaN(opsPerSec) || Double.isInfinite(opsPerSec)) {
+ throw new IllegalArgumentException("opsPerSec cannot be Nan or Infinite");
+ }
+ if (opsPerSec <= 0) {
+ throw new IllegalArgumentException("opsPerSec must be greater then 0");
+ }
+ this.opsPerSec = opsPerSec;
+ intervalNs = Math.round(ONE_SEC_IN_NS / opsPerSec);
+ this.nanoClock = nanoClock;
+ }
+
+ public UniformRateLimiter(final double opsPerSec) {
+ this(opsPerSec, System::nanoTime);
+ }
+
+ public double getOpsPerSec() {
+ return opsPerSec;
+ }
+
+ public long getIntervalNs() {
+ return intervalNs;
+ }
+
+ public long acquire() {
+ final long currOpIndex = V_TIME_UPDATER.getAndIncrement(this);
+ long start = this.start;
+ if (start == Long.MIN_VALUE) {
+ start = nanoClock.get();
+ if (!START_UPDATER.compareAndSet(this, Long.MIN_VALUE, start)) {
+ start = this.start;
+ assert start != Long.MIN_VALUE;
+ }
+ }
+ return start + currOpIndex * intervalNs;
+ }
+
+ public static void uninterruptibleSleepNs(final long intendedTime) {
+ long sleepNs;
+ while ((sleepNs = (intendedTime - System.nanoTime())) > 0) {
+ LockSupport.parkNanos(sleepNs);
+ }
+ }
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java
index a56fc9621..14576c985 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java
@@ -1,25 +1,33 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.distributor;
-import com.google.common.io.BaseEncoding;
+import com.google.common.io.BaseEncoding;
import java.util.Random;
public abstract class KeyDistributor {
@@ -28,11 +36,11 @@ public abstract class KeyDistributor {
private static final int KEY_BYTE_SIZE = 7;
private static final String[] randomKeys = new String[UNIQUE_COUNT];
- private static final Random random = new Random();
static {
// Generate a number of random keys to be used when publishing
byte[] buffer = new byte[KEY_BYTE_SIZE];
+ Random random = new Random();
for (int i = 0; i < randomKeys.length; i++) {
random.nextBytes(buffer);
randomKeys[i] = BaseEncoding.base64Url().omitPadding().encode(buffer);
@@ -61,8 +69,9 @@ public static KeyDistributor build(KeyDistributorType keyType) {
case RANDOM_NANO:
keyDistributor = new RandomNano();
break;
+ default:
+ throw new IllegalStateException("Unexpected KeyDistributorType: " + keyType);
}
return keyDistributor;
}
-
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java
index db953a3c0..3cb1cf1c0 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java
@@ -1,39 +1,42 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.distributor;
+
import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;
public enum KeyDistributorType {
+ /** Key distributor that returns null keys to have default publish semantics. */
@JsonEnumDefaultValue
- /**
- * Key distributor that returns null keys to have default publish semantics
- */
NO_KEY,
- /**
- * Genarate a finite number of "keys" and cycle through them in round-robin fashion
- */
+ /** Genarate a finite number of "keys" and cycle through them in round-robin fashion. */
KEY_ROUND_ROBIN,
- /**
- * Random distribution based on System.nanoTime()
- */
+ /** Random distribution based on System.nanoTime(). */
RANDOM_NANO,
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java
index 749e3e864..bb600e2ca 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.distributor;
+
import javax.annotation.concurrent.NotThreadSafe;
@NotThreadSafe
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java
index d9c8d29ac..593f0e5d4 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.distributor;
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java
index 5cc1c8f61..cedbf1b1a 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.distributor;
+
import javax.annotation.concurrent.ThreadSafe;
@ThreadSafe
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java
index c9a02c35d..2fe60e052 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java
@@ -1,29 +1,24 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.payload;
+import static java.nio.file.Files.readAllBytes;
+
import java.io.File;
import java.io.IOException;
import java.text.MessageFormat;
-import static java.nio.file.Files.readAllBytes;
-
public class FilePayloadReader implements PayloadReader {
private final int expectedLength;
@@ -46,8 +41,10 @@ public byte[] load(String resourceName) {
private void checkPayloadLength(byte[] payload) {
if (expectedLength != payload.length) {
- throw new PayloadException(MessageFormat.format("Payload length mismatch. Actual is: {0}, but expected: {1} ",
- payload.length, expectedLength));
+ throw new PayloadException(
+ MessageFormat.format(
+ "Payload length mismatch. Actual is: {0}, but expected: {1} ",
+ payload.length, expectedLength));
}
}
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/MessageSizeDistribution.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/MessageSizeDistribution.java
new file mode 100644
index 000000000..56355e4b6
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/MessageSizeDistribution.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils.payload;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Parses and represents a message size distribution from workload config.
+ * Creates one payload size per bucket and provides weights for runtime selection.
+ *
+ * Example configuration:
+ *
+ * messageSizeDistribution:
+ * "0-256": 234
+ * "256-1024": 456
+ * "1024-4096": 678
+ *
+ */
+public class MessageSizeDistribution {
+
+ private final List buckets;
+ private final int totalWeight;
+
+ /**
+ * Represents a single size bucket with min/max range and weight.
+ */
+ public static class Bucket {
+ public final int minSize;
+ public final int maxSize;
+ public final int weight;
+
+ Bucket(int minSize, int maxSize, int weight) {
+ this.minSize = minSize;
+ this.maxSize = maxSize;
+ this.weight = weight;
+ }
+
+ /**
+ * Returns midpoint of range as the representative size for this bucket.
+ *
+ * @return the representative size (midpoint of min and max)
+ */
+ public int getRepresentativeSize() {
+ return (minSize + maxSize) / 2;
+ }
+ }
+
+ public MessageSizeDistribution(Map config) {
+ if (config == null || config.isEmpty()) {
+ throw new IllegalArgumentException("Distribution config cannot be null or empty");
+ }
+
+ List parsed = new ArrayList<>();
+ int total = 0;
+
+ for (Map.Entry e : config.entrySet()) {
+ int[] range = parseRange(e.getKey());
+ int weight = e.getValue();
+ if (weight < 0) {
+ throw new IllegalArgumentException("Weight cannot be negative: " + weight);
+ }
+ parsed.add(new Bucket(range[0], range[1], weight));
+ total += weight;
+ }
+
+ this.buckets = parsed;
+ this.totalWeight = total;
+
+ if (totalWeight <= 0) {
+ throw new IllegalArgumentException("Distribution weights must sum to a positive value");
+ }
+ }
+
+ private int[] parseRange(String range) {
+ if (range == null || range.isEmpty()) {
+ throw new IllegalArgumentException("Range cannot be null or empty");
+ }
+
+ // Find the last hyphen that separates min and max (to handle negative numbers if any)
+ int lastHyphen = range.lastIndexOf('-');
+ if (lastHyphen <= 0) {
+ throw new IllegalArgumentException("Invalid range format (expected 'min-max'): " + range);
+ }
+
+ String minPart = range.substring(0, lastHyphen);
+ String maxPart = range.substring(lastHyphen + 1);
+
+ int minSize = parseSize(minPart);
+ int maxSize = parseSize(maxPart);
+
+ if (minSize < 0) {
+ throw new IllegalArgumentException("Min size cannot be negative: " + minSize);
+ }
+ if (maxSize < minSize) {
+ throw new IllegalArgumentException(
+ "Max size must be >= min size: " + minSize + " > " + maxSize);
+ }
+
+ return new int[] {minSize, maxSize};
+ }
+
+ private int parseSize(String s) {
+ s = s.trim().toUpperCase();
+ int mult = 1;
+
+ if (s.endsWith("KB")) {
+ mult = 1024;
+ s = s.substring(0, s.length() - 2);
+ } else if (s.endsWith("MB")) {
+ mult = 1024 * 1024;
+ s = s.substring(0, s.length() - 2);
+ } else if (s.endsWith("B")) {
+ s = s.substring(0, s.length() - 1);
+ }
+
+ try {
+ return Integer.parseInt(s.trim()) * mult;
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid size format: " + s, e);
+ }
+ }
+
+ /**
+ * Returns list of representative sizes, one per bucket (for payload generation).
+ *
+ * @return list of representative sizes
+ */
+ public List getBucketSizes() {
+ List sizes = new ArrayList<>();
+ for (Bucket b : buckets) {
+ sizes.add(b.getRepresentativeSize());
+ }
+ return sizes;
+ }
+
+ /**
+ * Returns list of max sizes, one per bucket (for payload generation).
+ * Using max sizes ensures the system is tested with the largest messages in each bucket range.
+ *
+ * @return list of max sizes per bucket
+ */
+ public List getBucketMaxSizes() {
+ List sizes = new ArrayList<>();
+ for (Bucket b : buckets) {
+ sizes.add(b.maxSize);
+ }
+ return sizes;
+ }
+
+ /**
+ * Returns weights array matching bucket order (for runtime selection).
+ *
+ * @return array of weights for each bucket
+ */
+ public int[] getWeights() {
+ int[] weights = new int[buckets.size()];
+ for (int i = 0; i < buckets.size(); i++) {
+ weights[i] = buckets.get(i).weight;
+ }
+ return weights;
+ }
+
+ /**
+ * Returns cumulative weights array for O(log n) binary search selection.
+ *
+ * @return array of cumulative weights
+ */
+ public int[] getCumulativeWeights() {
+ int[] cumulative = new int[buckets.size()];
+ int sum = 0;
+ for (int i = 0; i < buckets.size(); i++) {
+ sum += buckets.get(i).weight;
+ cumulative[i] = sum;
+ }
+ return cumulative;
+ }
+
+ public int getTotalWeight() {
+ return totalWeight;
+ }
+
+ public int getMaxSize() {
+ return buckets.stream().mapToInt(b -> b.maxSize).max().orElse(0);
+ }
+
+ /**
+ * Returns weighted average size across all buckets.
+ *
+ * @return the weighted average size
+ */
+ public int getAvgSize() {
+ long sum = 0;
+ for (Bucket b : buckets) {
+ sum += (long) b.getRepresentativeSize() * b.weight;
+ }
+ return (int) (sum / totalWeight);
+ }
+
+ public int getBucketCount() {
+ return buckets.size();
+ }
+
+ public List getBuckets() {
+ return buckets;
+ }
+}
+
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java
index 6abfca696..a3887ac31 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.payload;
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java
index cf00c9d38..97d82aa80 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.utils.payload;
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java
index 3587ba81b..8723384cf 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java
@@ -1,53 +1,52 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker;
-import org.apache.bookkeeper.stats.Stats;
-import org.apache.bookkeeper.stats.StatsProvider;
-import org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider;
-import org.apache.commons.configuration.CompositeConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
-
import io.javalin.Javalin;
+import org.apache.bookkeeper.stats.Stats;
+import org.apache.bookkeeper.stats.StatsProvider;
+import org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-/**
- * A benchmark worker that listen for tasks to perform
- */
+/** A benchmark worker that listen for tasks to perform. */
public class BenchmarkWorker {
static class Arguments {
- @Parameter(names = { "-h", "--help" }, description = "Help message", help = true)
+ @Parameter(
+ names = {"-h", "--help"},
+ description = "Help message",
+ help = true)
boolean help;
- @Parameter(names = { "-p", "--port" }, description = "HTTP port to listen on")
+ @Parameter(
+ names = {"-p", "--port"},
+ description = "HTTP port to listen on")
public int httpPort = 8080;
- @Parameter(names = { "-sp", "--stats-port" }, description = "Stats port to listen on")
+ @Parameter(
+ names = {"-sp", "--stats-port"},
+ description = "Stats port to listen on")
public int statsPort = 8081;
}
@@ -76,9 +75,8 @@ public static void main(String[] args) throws Exception {
StatsProvider provider = Stats.get();
provider.start(conf);
- Runtime.getRuntime().addShutdownHook(new Thread(
- () -> provider.stop(),
- "benchmark-worker-shutdown-thread"));
+ Runtime.getRuntime()
+ .addShutdownHook(new Thread(() -> provider.stop(), "benchmark-worker-shutdown-thread"));
// Dump configuration variables
log.info("Starting benchmark with config: {}", writer.writeValueAsString(arguments));
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java
index c56119a1d..c41a0da51 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java
@@ -1,52 +1,25 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker;
-import static java.util.stream.Collectors.toList;
-import static org.asynchttpclient.Dsl.asyncHttpClient;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.zip.DataFormatException;
-
-import org.HdrHistogram.Histogram;
-import org.apache.pulsar.common.util.FutureUtil;
-import org.asynchttpclient.AsyncHttpClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.util.Collections.unmodifiableList;
+import static java.util.stream.Collectors.joining;
import com.beust.jcommander.internal.Maps;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-
-import io.netty.buffer.ByteBufUtil;
-import io.netty.buffer.Unpooled;
import io.openmessaging.benchmark.utils.ListPartition;
import io.openmessaging.benchmark.worker.commands.ConsumerAssignment;
import io.openmessaging.benchmark.worker.commands.CountersStats;
@@ -55,281 +28,266 @@
import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment;
import io.openmessaging.benchmark.worker.commands.TopicSubscription;
import io.openmessaging.benchmark.worker.commands.TopicsInfo;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DistributedWorkersEnsemble implements Worker {
-
- private final List workers;
- private final List producerWorkers;
- private final List consumerWorkers;
-
- private final AsyncHttpClient httpClient = asyncHttpClient();
+ private final Thread shutdownHook = new Thread(this::stopAll);
+ private final List workers;
+ private final List producerWorkers;
+ private final List consumerWorkers;
+ private final Worker leader;
private int numberOfUsedProducerWorkers;
- public DistributedWorkersEnsemble(List workers, boolean extraConsumerWorkers) {
+ public DistributedWorkersEnsemble(List workers, boolean extraConsumerWorkers) {
Preconditions.checkArgument(workers.size() > 1);
+ this.workers = unmodifiableList(workers);
+ leader = workers.get(0);
+ int numberOfProducerWorkers = getNumberOfProducerWorkers(workers, extraConsumerWorkers);
+ List> partitions =
+ Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers);
+ this.producerWorkers = partitions.get(1);
+ this.consumerWorkers = partitions.get(0);
+
+ log.info(
+ "Workers list - producers: [{}]",
+ producerWorkers.stream().map(Worker::id).collect(joining(",")));
+ log.info(
+ "Workers list - consumers: {}",
+ consumerWorkers.stream().map(Worker::id).collect(joining(",")));
+
+ Runtime.getRuntime().addShutdownHook(shutdownHook);
+ }
- this.workers = workers;
-
- // For driver-jms extra consumers are required.
- // If there is an odd number of workers then allocate the extra to consumption.
- int numberOfProducerWorkers = extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2;
- List> partitions = Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers);
- this.producerWorkers = partitions.get(1);
- this.consumerWorkers = partitions.get(0);
-
- log.info("Workers list - producers: {}", producerWorkers);
- log.info("Workers list - consumers: {}", consumerWorkers);
+ /*
+ * For driver-jms extra consumers are required. If there is an odd number of workers then allocate the extra
+ * to consumption.
+ */
+ @VisibleForTesting
+ static int getNumberOfProducerWorkers(List workers, boolean extraConsumerWorkers) {
+ return extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2;
}
@Override
public void initializeDriver(File configurationFile) throws IOException {
- byte[] confFileContent = Files.readAllBytes(Paths.get(configurationFile.toString()));
- sendPost(workers, "/initialize-driver", confFileContent);
+ workers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.initializeDriver(configurationFile);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
@SuppressWarnings("unchecked")
public List createTopics(TopicsInfo topicsInfo) throws IOException {
- // Create all topics from a single worker node
- return (List) post(workers.get(0), "/create-topics", writer.writeValueAsBytes(topicsInfo), List.class)
- .join();
+ return leader.createTopics(topicsInfo);
}
@Override
public void createProducers(List topics) {
- List> topicsPerProducer = ListPartition.partitionList(topics,
- producerWorkers.size());
- Map> topicsPerProducerMap = Maps.newHashMap();
+ List> topicsPerProducer =
+ ListPartition.partitionList(topics, producerWorkers.size());
+ Map> topicsPerProducerMap = Maps.newHashMap();
int i = 0;
for (List assignedTopics : topicsPerProducer) {
topicsPerProducerMap.put(producerWorkers.get(i++), assignedTopics);
}
// Number of actually used workers might be less than available workers
- numberOfUsedProducerWorkers = i;
-
- List> futures = topicsPerProducerMap.keySet().stream().map(producer -> {
- try {
- return sendPost(producer, "/create-producers",
- writer.writeValueAsBytes(topicsPerProducerMap.get(producer)));
- } catch (Exception e) {
- CompletableFuture future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
- }
- }).collect(toList());
-
- FutureUtil.waitForAll(futures).join();
+ numberOfUsedProducerWorkers =
+ (int) topicsPerProducerMap.values().stream().filter(t -> !t.isEmpty()).count();
+ log.debug(
+ "Producing worker count: {} of {}", numberOfUsedProducerWorkers, producerWorkers.size());
+ topicsPerProducerMap.entrySet().parallelStream()
+ .forEach(
+ e -> {
+ try {
+ e.getKey().createProducers(e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ });
}
@Override
public void startLoad(ProducerWorkAssignment producerWorkAssignment) throws IOException {
// Reduce the publish rate across all the brokers
- producerWorkAssignment.publishRate /= numberOfUsedProducerWorkers;
- sendPost(producerWorkers, "/start-load", writer.writeValueAsBytes(producerWorkAssignment));
+ double newRate = producerWorkAssignment.publishRate / numberOfUsedProducerWorkers;
+ log.debug("Setting worker assigned publish rate to {} msgs/sec", newRate);
+ // Reduce the publish rate across all the brokers
+ producerWorkers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.startLoad(producerWorkAssignment.withPublishRate(newRate));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
public void probeProducers() throws IOException {
- sendPost(producerWorkers, "/probe-producers", new byte[0]);
+ producerWorkers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.probeProducers();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
public void adjustPublishRate(double publishRate) throws IOException {
- // Reduce the publish rate across all the brokers
- publishRate /= numberOfUsedProducerWorkers;
- sendPost(producerWorkers, "/adjust-publish-rate", writer.writeValueAsBytes(publishRate));
+ double newRate = publishRate / numberOfUsedProducerWorkers;
+ log.debug("Adjusting producer publish rate to {} msgs/sec", newRate);
+ producerWorkers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.adjustPublishRate(newRate);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
public void stopAll() {
- sendPost(workers, "/stop-all", new byte[0]);
+ workers.parallelStream().forEach(Worker::stopAll);
+ }
+
+ @Override
+ public String id() {
+ return "Ensemble[" + workers.stream().map(Worker::id).collect(joining(",")) + "]";
}
@Override
public void pauseConsumers() throws IOException {
- sendPost(consumerWorkers, "/pause-consumers", new byte[0]);
+ consumerWorkers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.pauseConsumers();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
public void resumeConsumers() throws IOException {
- sendPost(consumerWorkers, "/resume-consumers", new byte[0]);
+ consumerWorkers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.resumeConsumers();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
public void createConsumers(ConsumerAssignment overallConsumerAssignment) {
- List> subscriptionsPerConsumer = ListPartition.partitionList(
- overallConsumerAssignment.topicsSubscriptions,
- consumerWorkers.size());
- Map topicsPerWorkerMap = Maps.newHashMap();
+ List> subscriptionsPerConsumer =
+ ListPartition.partitionList(
+ overallConsumerAssignment.topicsSubscriptions, consumerWorkers.size());
+ Map topicsPerWorkerMap = Maps.newHashMap();
int i = 0;
for (List tsl : subscriptionsPerConsumer) {
- ConsumerAssignment individualAssignement = new ConsumerAssignment();
- individualAssignement.topicsSubscriptions = tsl;
- topicsPerWorkerMap.put(consumerWorkers.get(i++), individualAssignement);
+ ConsumerAssignment individualAssignment = new ConsumerAssignment();
+ individualAssignment.topicsSubscriptions = tsl;
+ topicsPerWorkerMap.put(consumerWorkers.get(i++), individualAssignment);
}
-
- List> futures = topicsPerWorkerMap.keySet().stream().map(consumer -> {
- try {
- return sendPost(consumer, "/create-consumers",
- writer.writeValueAsBytes(topicsPerWorkerMap.get(consumer)));
- } catch (Exception e) {
- CompletableFuture future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
- }
- }).collect(toList());
-
- FutureUtil.waitForAll(futures).join();
+ topicsPerWorkerMap.entrySet().parallelStream()
+ .forEach(
+ e -> {
+ try {
+ e.getKey().createConsumers(e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ });
}
@Override
public PeriodStats getPeriodStats() {
- List individualStats = get(workers, "/period-stats", PeriodStats.class);
- PeriodStats stats = new PeriodStats();
- individualStats.forEach(is -> {
- stats.messagesSent += is.messagesSent;
- stats.bytesSent += is.bytesSent;
- stats.messagesReceived += is.messagesReceived;
- stats.bytesReceived += is.bytesReceived;
- stats.totalMessagesSent += is.totalMessagesSent;
- stats.totalMessagesReceived += is.totalMessagesReceived;
-
- try {
- stats.publishLatency.add(Histogram.decodeFromCompressedByteBuffer(
- ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30)));
-
- stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer(
- ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12)));
- } catch (ArrayIndexOutOfBoundsException | DataFormatException e) {
- throw new RuntimeException(e);
- }
- });
-
- return stats;
+ return workers.parallelStream()
+ .map(
+ w -> {
+ try {
+ return w.getPeriodStats();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .reduce(new PeriodStats(), PeriodStats::plus);
}
@Override
public CumulativeLatencies getCumulativeLatencies() {
- List individualStats = get(workers, "/cumulative-latencies", CumulativeLatencies.class);
-
- CumulativeLatencies stats = new CumulativeLatencies();
- individualStats.forEach(is -> {
- try {
- stats.publishLatency.add(Histogram.decodeFromCompressedByteBuffer(
- ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30)));
- } catch (Exception e) {
- log.error("Failed to decode publish latency: {}",
- ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishLatencyBytes)));
- throw new RuntimeException(e);
- }
-
- try {
- stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer(
- ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12)));
- } catch (Exception e) {
- log.error("Failed to decode end-to-end latency: {}",
- ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.endToEndLatencyBytes)));
- throw new RuntimeException(e);
- }
- });
-
- return stats;
-
+ return workers.parallelStream()
+ .map(
+ w -> {
+ try {
+ return w.getCumulativeLatencies();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .reduce(new CumulativeLatencies(), CumulativeLatencies::plus);
}
@Override
public CountersStats getCountersStats() throws IOException {
- List individualStats = get(workers, "/counters-stats", CountersStats.class);
-
- CountersStats stats = new CountersStats();
- individualStats.forEach(is -> {
- stats.messagesSent += is.messagesSent;
- stats.messagesReceived += is.messagesReceived;
- });
-
- return stats;
+ return workers.parallelStream()
+ .map(
+ w -> {
+ try {
+ return w.getCountersStats();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .reduce(new CountersStats(), CountersStats::plus);
}
@Override
public void resetStats() throws IOException {
- sendPost(workers, "/reset-stats", new byte[0]);
- }
-
- /**
- * Send a request to multiple hosts and wait for all responses
- */
- private void sendPost(List hosts, String path, byte[] body) {
- FutureUtil.waitForAll(hosts.stream().map(w -> sendPost(w, path, body)).collect(toList())).join();
- }
-
- private CompletableFuture sendPost(String host, String path, byte[] body) {
- return httpClient.preparePost(host + path).setBody(body).execute().toCompletableFuture().thenApply(x -> {
- if (x.getStatusCode() != 200) {
- log.error("Failed to do HTTP post request to {}{} -- code: {}", host, path, x.getStatusCode());
- }
- Preconditions.checkArgument(x.getStatusCode() == 200);
- return (Void) null;
- });
- }
-
- private List get(List hosts, String path, Class clazz) {
- List> futures = hosts.stream().map(w -> get(w, path, clazz)).collect(toList());
-
- CompletableFuture> resultFuture = new CompletableFuture<>();
- FutureUtil.waitForAll(futures).thenRun(() -> {
- resultFuture.complete(futures.stream().map(CompletableFuture::join).collect(toList()));
- }).exceptionally(ex -> {
- resultFuture.completeExceptionally(ex);
- return null;
- });
-
- return resultFuture.join();
- }
-
- private CompletableFuture get(String host, String path, Class clazz) {
- return httpClient.prepareGet(host + path).execute().toCompletableFuture().thenApply(response -> {
- try {
- if (response.getStatusCode() != 200) {
- log.error("Failed to do HTTP get request to {}{} -- code: {}", host, path, response.getStatusCode());
- }
- Preconditions.checkArgument(response.getStatusCode() == 200);
- return mapper.readValue(response.getResponseBody(), clazz);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- }
-
- private CompletableFuture post(String host, String path, byte[] body, Class clazz) {
- return httpClient.preparePost(host + path).setBody(body).execute().toCompletableFuture().thenApply(response -> {
- try {
- if (response.getStatusCode() != 200) {
- log.error("Failed to do HTTP post request to {}{} -- code: {}", host, path, response.getStatusCode());
- }
- Preconditions.checkArgument(response.getStatusCode() == 200);
- return mapper.readValue(response.getResponseBody(), clazz);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
+ workers.parallelStream()
+ .forEach(
+ w -> {
+ try {
+ w.resetStats();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@Override
public void close() throws Exception {
- httpClient.close();
- }
-
- private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter();
-
- private static final ObjectMapper mapper = new ObjectMapper()
- .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- static {
- mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE);
+ Runtime.getRuntime().removeShutdownHook(shutdownHook);
+ for (Worker w : workers) {
+ try {
+ w.close();
+ } catch (Exception ignored) {
+ log.trace("Ignored error while closing worker {}", w, ignored);
+ }
+ }
}
private static final Logger log = LoggerFactory.getLogger(DistributedWorkersEnsemble.class);
-
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java
new file mode 100644
index 000000000..513b04178
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker;
+
+import static io.openmessaging.benchmark.worker.WorkerHandler.ADJUST_PUBLISH_RATE;
+import static io.openmessaging.benchmark.worker.WorkerHandler.COUNTERS_STATS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.CREATE_CONSUMERS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.CREATE_PRODUCERS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.CREATE_TOPICS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.CUMULATIVE_LATENCIES;
+import static io.openmessaging.benchmark.worker.WorkerHandler.INITIALIZE_DRIVER;
+import static io.openmessaging.benchmark.worker.WorkerHandler.PAUSE_CONSUMERS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.PERIOD_STATS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.PROBE_PRODUCERS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.RESET_STATS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.RESUME_CONSUMERS;
+import static io.openmessaging.benchmark.worker.WorkerHandler.START_LOAD;
+import static io.openmessaging.benchmark.worker.WorkerHandler.STOP_ALL;
+import static org.asynchttpclient.Dsl.asyncHttpClient;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.common.base.Preconditions;
+import io.openmessaging.benchmark.worker.commands.ConsumerAssignment;
+import io.openmessaging.benchmark.worker.commands.CountersStats;
+import io.openmessaging.benchmark.worker.commands.CumulativeLatencies;
+import io.openmessaging.benchmark.worker.commands.PeriodStats;
+import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment;
+import io.openmessaging.benchmark.worker.commands.TopicsInfo;
+import io.openmessaging.benchmark.worker.jackson.ObjectMappers;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import org.asynchttpclient.AsyncHttpClient;
+import org.asynchttpclient.Dsl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpWorkerClient implements Worker {
+
+ private static final byte[] EMPTY_BODY = new byte[0];
+ private static final int HTTP_OK = 200;
+
+ private final AsyncHttpClient httpClient;
+ private final String host;
+
+ public HttpWorkerClient(String host) {
+ this(asyncHttpClient(Dsl.config().setReadTimeout(600000).setRequestTimeout(600000)), host);
+ }
+
+ HttpWorkerClient(AsyncHttpClient httpClient, String host) {
+ this.httpClient = httpClient;
+ this.host = host;
+ }
+
+ @Override
+ public void initializeDriver(File configurationFile) throws IOException {
+ byte[] confFileContent = Files.readAllBytes(Paths.get(configurationFile.toString()));
+ sendPost(INITIALIZE_DRIVER, confFileContent);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List createTopics(TopicsInfo topicsInfo) throws IOException {
+ return (List) post(CREATE_TOPICS, writer.writeValueAsBytes(topicsInfo), List.class);
+ }
+
+ @Override
+ public void createProducers(List topics) throws IOException {
+ sendPost(CREATE_PRODUCERS, writer.writeValueAsBytes(topics));
+ }
+
+ @Override
+ public void createConsumers(ConsumerAssignment consumerAssignment) throws IOException {
+ sendPost(CREATE_CONSUMERS, writer.writeValueAsBytes(consumerAssignment));
+ }
+
+ @Override
+ public void probeProducers() throws IOException {
+ sendPost(PROBE_PRODUCERS);
+ }
+
+ @Override
+ public void startLoad(ProducerWorkAssignment producerWorkAssignment) throws IOException {
+ log.debug(
+ "Setting worker assigned publish rate to {} msgs/sec", producerWorkAssignment.publishRate);
+ sendPost(START_LOAD, writer.writeValueAsBytes(producerWorkAssignment));
+ }
+
+ @Override
+ public void adjustPublishRate(double publishRate) throws IOException {
+ log.debug("Adjusting worker publish rate to {} msgs/sec", publishRate);
+ sendPost(ADJUST_PUBLISH_RATE, writer.writeValueAsBytes(publishRate));
+ }
+
+ @Override
+ public void pauseConsumers() throws IOException {
+ sendPost(PAUSE_CONSUMERS);
+ }
+
+ @Override
+ public void resumeConsumers() throws IOException {
+ sendPost(RESUME_CONSUMERS);
+ }
+
+ @Override
+ public CountersStats getCountersStats() throws IOException {
+ return get(COUNTERS_STATS, CountersStats.class);
+ }
+
+ @Override
+ public PeriodStats getPeriodStats() throws IOException {
+ return get(PERIOD_STATS, PeriodStats.class);
+ }
+
+ @Override
+ public CumulativeLatencies getCumulativeLatencies() throws IOException {
+ return get(CUMULATIVE_LATENCIES, CumulativeLatencies.class);
+ }
+
+ @Override
+ public void resetStats() throws IOException {
+ sendPost(RESET_STATS);
+ }
+
+ @Override
+ public void stopAll() {
+ sendPost(STOP_ALL);
+ }
+
+ @Override
+ public String id() {
+ return host;
+ }
+
+ @Override
+ public void close() throws Exception {
+ httpClient.close();
+ }
+
+ private void sendPost(String path) {
+ sendPost(path, EMPTY_BODY);
+ }
+
+ private void sendPost(String path, byte[] body) {
+ httpClient
+ .preparePost(host + path)
+ .setBody(body)
+ .execute()
+ .toCompletableFuture()
+ .thenApply(
+ response -> {
+ if (response.getStatusCode() != HTTP_OK) {
+ log.error(
+ "Failed to do HTTP post request to {}{} -- code: {}",
+ host,
+ path,
+ response.getStatusCode());
+ }
+ Preconditions.checkArgument(response.getStatusCode() == HTTP_OK);
+ return (Void) null;
+ })
+ .join();
+ }
+
+ private T get(String path, Class clazz) {
+ return httpClient
+ .prepareGet(host + path)
+ .execute()
+ .toCompletableFuture()
+ .thenApply(
+ response -> {
+ try {
+ if (response.getStatusCode() != HTTP_OK) {
+ log.error(
+ "Failed to do HTTP get request to {}{} -- code: {}",
+ host,
+ path,
+ response.getStatusCode());
+ }
+ Preconditions.checkArgument(response.getStatusCode() == HTTP_OK);
+ return mapper.readValue(response.getResponseBody(), clazz);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .join();
+ }
+
+ private T post(String path, byte[] body, Class clazz) {
+ return httpClient
+ .preparePost(host + path)
+ .setBody(body)
+ .execute()
+ .toCompletableFuture()
+ .thenApply(
+ response -> {
+ try {
+ if (response.getStatusCode() != HTTP_OK) {
+ log.error(
+ "Failed to do HTTP post request to {}{} -- code: {}",
+ host,
+ path,
+ response.getStatusCode());
+ }
+ Preconditions.checkArgument(response.getStatusCode() == HTTP_OK);
+ return mapper.readValue(response.getResponseBody(), clazz);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .join();
+ }
+
+ private static final ObjectMapper mapper = ObjectMappers.DEFAULT.mapper();
+ private static final ObjectWriter writer = ObjectMappers.DEFAULT.writer();
+ private static final Logger log = LoggerFactory.getLogger(HttpWorkerClient.class);
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java
index 5c5ad560b..af64369c4 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java
@@ -1,59 +1,37 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker;
import static java.util.stream.Collectors.toList;
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.LongAdder;
-import java.util.function.Function;
-
-import org.HdrHistogram.Recorder;
-import org.apache.bookkeeper.stats.Counter;
-import org.apache.bookkeeper.stats.NullStatsLogger;
-import org.apache.bookkeeper.stats.OpStatsLogger;
-import org.apache.bookkeeper.stats.StatsLogger;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.RateLimiter;
-
import io.netty.util.concurrent.DefaultThreadFactory;
import io.openmessaging.benchmark.DriverConfiguration;
import io.openmessaging.benchmark.driver.BenchmarkConsumer;
import io.openmessaging.benchmark.driver.BenchmarkDriver;
+import io.openmessaging.benchmark.driver.BenchmarkDriver.ConsumerInfo;
+import io.openmessaging.benchmark.driver.BenchmarkDriver.ProducerInfo;
+import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo;
import io.openmessaging.benchmark.driver.BenchmarkProducer;
import io.openmessaging.benchmark.driver.ConsumerCallback;
import io.openmessaging.benchmark.utils.RandomGenerator;
import io.openmessaging.benchmark.utils.Timer;
+import io.openmessaging.benchmark.utils.UniformRateLimiter;
import io.openmessaging.benchmark.utils.distributor.KeyDistributor;
import io.openmessaging.benchmark.worker.commands.ConsumerAssignment;
import io.openmessaging.benchmark.worker.commands.CountersStats;
@@ -61,45 +39,36 @@
import io.openmessaging.benchmark.worker.commands.PeriodStats;
import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment;
import io.openmessaging.benchmark.worker.commands.TopicsInfo;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class LocalWorker implements Worker, ConsumerCallback {
private BenchmarkDriver benchmarkDriver = null;
-
- private List producers = new ArrayList<>();
- private List consumers = new ArrayList<>();
-
- private final RateLimiter rateLimiter = RateLimiter.create(1.0);
-
- private final ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker"));
-
- // stats
-
- private final StatsLogger statsLogger;
-
- private final LongAdder messagesSent = new LongAdder();
- private final LongAdder bytesSent = new LongAdder();
- private final Counter messagesSentCounter;
- private final Counter bytesSentCounter;
-
- private final LongAdder messagesReceived = new LongAdder();
- private final LongAdder bytesReceived = new LongAdder();
- private final Counter messagesReceivedCounter;
- private final Counter bytesReceivedCounter;
-
- private final LongAdder totalMessagesSent = new LongAdder();
- private final LongAdder totalMessagesReceived = new LongAdder();
-
- private final Recorder publishLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5);
- private final Recorder cumulativePublishLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5);
- private final OpStatsLogger publishLatencyStats;
-
- private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5);
- private final Recorder endToEndCumulativeLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5);
- private final OpStatsLogger endToEndLatencyStats;
-
+ private final List producers = new ArrayList<>();
+ private final List consumers = new ArrayList<>();
+ private volatile MessageProducer messageProducer;
+ private final ExecutorService executor =
+ Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker"));
+ private final WorkerStats stats;
private boolean testCompleted = false;
-
private boolean consumersArePaused = false;
public LocalWorker() {
@@ -107,17 +76,8 @@ public LocalWorker() {
}
public LocalWorker(StatsLogger statsLogger) {
- this.statsLogger = statsLogger;
-
- StatsLogger producerStatsLogger = statsLogger.scope("producer");
- this.messagesSentCounter = producerStatsLogger.getCounter("messages_sent");
- this.bytesSentCounter = producerStatsLogger.getCounter("bytes_sent");
- this.publishLatencyStats = producerStatsLogger.getOpStatsLogger("produce_latency");
-
- StatsLogger consumerStatsLogger = statsLogger.scope("consumer");
- this.messagesReceivedCounter = consumerStatsLogger.getCounter("messages_recv");
- this.bytesReceivedCounter = consumerStatsLogger.getCounter("bytes_recv");
- this.endToEndLatencyStats = consumerStatsLogger.getOpStatsLogger("e2e_latency");
+ stats = new WorkerStats(statsLogger);
+ updateMessageProducer(1.0);
}
@Override
@@ -125,58 +85,78 @@ public void initializeDriver(File driverConfigFile) throws IOException {
Preconditions.checkArgument(benchmarkDriver == null);
testCompleted = false;
- DriverConfiguration driverConfiguration = mapper.readValue(driverConfigFile, DriverConfiguration.class);
+ DriverConfiguration driverConfiguration =
+ mapper.readValue(driverConfigFile, DriverConfiguration.class);
log.info("Driver: {}", writer.writeValueAsString(driverConfiguration));
try {
- benchmarkDriver = (BenchmarkDriver) Class.forName(driverConfiguration.driverClass).newInstance();
- benchmarkDriver.initialize(driverConfigFile, statsLogger);
- } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ benchmarkDriver =
+ (BenchmarkDriver) Class.forName(driverConfiguration.driverClass).newInstance();
+ benchmarkDriver.initialize(driverConfigFile, stats.getStatsLogger());
+ } catch (InstantiationException
+ | IllegalAccessException
+ | ClassNotFoundException
+ | InterruptedException e) {
throw new RuntimeException(e);
}
}
@Override
public List createTopics(TopicsInfo topicsInfo) {
- List> futures = new ArrayList<>();
-
Timer timer = new Timer();
- String topicPrefix = benchmarkDriver.getTopicNamePrefix();
+ List topicInfos =
+ IntStream.range(0, topicsInfo.numberOfTopics)
+ .mapToObj(
+ i -> new TopicInfo(generateTopicName(i), topicsInfo.numberOfPartitionsPerTopic))
+ .collect(toList());
- List topics = new ArrayList<>();
- for (int i = 0; i < topicsInfo.numberOfTopics; i++) {
- String topic = String.format("%s-%s-%04d", topicPrefix, RandomGenerator.getRandomString(), i);
- topics.add(topic);
- futures.add(benchmarkDriver.createTopic(topic, topicsInfo.numberOfPartitionsPerTopic));
- }
+ benchmarkDriver.createTopics(topicInfos).join();
- futures.forEach(CompletableFuture::join);
+ List topics = topicInfos.stream().map(TopicInfo::getTopic).collect(toList());
log.info("Created {} topics in {} ms", topics.size(), timer.elapsedMillis());
return topics;
}
+ private String generateTopicName(int i) {
+ return String.format(
+ "%s-%07d-%s", benchmarkDriver.getTopicNamePrefix(), i, RandomGenerator.getRandomString());
+ }
+
@Override
public void createProducers(List topics) {
Timer timer = new Timer();
+ AtomicInteger index = new AtomicInteger();
- List> futures = topics.stream()
- .map(topic -> benchmarkDriver.createProducer(topic)).collect(toList());
+ producers.addAll(
+ benchmarkDriver
+ .createProducers(
+ topics.stream()
+ .map(t -> new ProducerInfo(index.getAndIncrement(), t))
+ .collect(toList()))
+ .join());
- futures.forEach(f -> producers.add(f.join()));
log.info("Created {} producers in {} ms", producers.size(), timer.elapsedMillis());
}
@Override
public void createConsumers(ConsumerAssignment consumerAssignment) {
Timer timer = new Timer();
+ AtomicInteger index = new AtomicInteger();
+
+ consumers.addAll(
+ benchmarkDriver
+ .createConsumers(
+ consumerAssignment.topicsSubscriptions.stream()
+ .map(
+ c ->
+ new ConsumerInfo(
+ index.getAndIncrement(), c.topic, c.subscription, this))
+ .collect(toList()))
+ .join());
- List> futures = consumerAssignment.topicsSubscriptions.stream()
- .map(ts -> benchmarkDriver.createConsumer(ts.topic, ts.subscription, this)).collect(toList());
-
- futures.forEach(f -> consumers.add(f.join()));
log.info("Created {} consumers in {} ms", consumers.size(), timer.elapsedMillis());
}
@@ -184,104 +164,119 @@ public void createConsumers(ConsumerAssignment consumerAssignment) {
public void startLoad(ProducerWorkAssignment producerWorkAssignment) {
int processors = Runtime.getRuntime().availableProcessors();
- rateLimiter.setRate(producerWorkAssignment.publishRate);
+ updateMessageProducer(producerWorkAssignment.publishRate);
Map> processorAssignment = new TreeMap<>();
int processorIdx = 0;
for (BenchmarkProducer p : producers) {
- processorAssignment.computeIfAbsent(processorIdx, x -> new ArrayList()).add(p);
+ processorAssignment
+ .computeIfAbsent(processorIdx, x -> new ArrayList())
+ .add(p);
processorIdx = (processorIdx + 1) % processors;
}
- processorAssignment.values().forEach(producers -> submitProducersToExecutor(producers,
- KeyDistributor.build(producerWorkAssignment.keyDistributorType), producerWorkAssignment.payloadData));
+ processorAssignment
+ .values()
+ .forEach(
+ producers ->
+ submitProducersToExecutor(
+ producers,
+ KeyDistributor.build(producerWorkAssignment.keyDistributorType),
+ producerWorkAssignment.payloadData,
+ producerWorkAssignment.payloadWeights));
}
@Override
public void probeProducers() throws IOException {
- producers.forEach(producer -> producer.sendAsync(Optional.of("key"), new byte[10])
- .thenRun(() -> totalMessagesSent.increment()));
+ producers.forEach(
+ producer ->
+ producer.sendAsync(Optional.of("key"), new byte[10]).thenRun(stats::recordMessageSent));
}
- private void submitProducersToExecutor(List producers, KeyDistributor keyDistributor, List payloads) {
- executor.submit(() -> {
- int payloadCount = payloads.size();
- Random r = new Random();
- byte[] firstPayload = payloads.get(0);
-
- try {
- while (!testCompleted) {
- producers.forEach(producer -> {
- rateLimiter.acquire();
- byte[] payloadData = payloadCount == 0 ? firstPayload : payloads.get(r.nextInt(payloadCount));
- final long sendTime = System.nanoTime();
- producer.sendAsync(Optional.ofNullable(keyDistributor.next()), payloadData)
- .thenRun(() -> {
- messagesSent.increment();
- totalMessagesSent.increment();
- messagesSentCounter.inc();
- bytesSent.add(payloadData.length);
- bytesSentCounter.add(payloadData.length);
-
- long latencyMicros = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime);
- publishLatencyRecorder.recordValue(latencyMicros);
- cumulativePublishLatencyRecorder.recordValue(latencyMicros);
- publishLatencyStats.registerSuccessfulEvent(latencyMicros, TimeUnit.MICROSECONDS);
- }).exceptionally(ex -> {
- log.warn("Write error on message", ex);
- return null;
- });
- });
- }
- } catch (Throwable t) {
- log.error("Got error", t);
+ private void submitProducersToExecutor(
+ List producers,
+ KeyDistributor keyDistributor,
+ List payloads,
+ int[] weights) {
+
+ // Build cumulative weights for O(log n) weighted selection if weights are provided
+ final int[] cumulative;
+ final int totalWeight;
+ if (weights != null && weights.length > 0) {
+ cumulative = new int[weights.length];
+ int sum = 0;
+ for (int i = 0; i < weights.length; i++) {
+ sum += weights[i];
+ cumulative[i] = sum;
}
- });
+ totalWeight = sum;
+ } else {
+ cumulative = null;
+ totalWeight = payloads.size();
+ }
+
+ final int payloadCount = payloads.size();
+
+ executor.submit(
+ () -> {
+ try {
+ ThreadLocalRandom r = ThreadLocalRandom.current();
+ while (!testCompleted) {
+ producers.forEach(
+ p -> {
+ int idx;
+ if (cumulative != null) {
+ // Weighted selection via binary search
+ int target = r.nextInt(totalWeight);
+ idx = Arrays.binarySearch(cumulative, target + 1);
+ if (idx < 0) {
+ idx = -idx - 1;
+ }
+ idx = Math.min(idx, payloadCount - 1);
+ } else {
+ // Uniform selection (backward compatible)
+ idx = r.nextInt(payloadCount);
+ }
+ messageProducer.sendMessage(
+ p,
+ Optional.ofNullable(keyDistributor.next()),
+ payloads.get(idx));
+ });
+ }
+ } catch (Throwable t) {
+ log.error("Got error", t);
+ }
+ });
}
@Override
public void adjustPublishRate(double publishRate) {
- if(publishRate < 1.0) {
- rateLimiter.setRate(1.0);
+ if (publishRate < 1.0) {
+ updateMessageProducer(1.0);
return;
}
- rateLimiter.setRate(publishRate);
+ updateMessageProducer(publishRate);
+ }
+
+ private void updateMessageProducer(double publishRate) {
+ messageProducer = new MessageProducer(new UniformRateLimiter(publishRate), stats);
}
@Override
public PeriodStats getPeriodStats() {
- PeriodStats stats = new PeriodStats();
-
- stats.messagesSent = messagesSent.sumThenReset();
- stats.bytesSent = bytesSent.sumThenReset();
-
- stats.messagesReceived = messagesReceived.sumThenReset();
- stats.bytesReceived = bytesReceived.sumThenReset();
-
- stats.totalMessagesSent = totalMessagesSent.sum();
- stats.totalMessagesReceived = totalMessagesReceived.sum();
-
- stats.publishLatency = publishLatencyRecorder.getIntervalHistogram();
- stats.endToEndLatency = endToEndLatencyRecorder.getIntervalHistogram();
- return stats;
+ return stats.toPeriodStats();
}
@Override
public CumulativeLatencies getCumulativeLatencies() {
- CumulativeLatencies latencies = new CumulativeLatencies();
- latencies.publishLatency = cumulativePublishLatencyRecorder.getIntervalHistogram();
- latencies.endToEndLatency = endToEndCumulativeLatencyRecorder.getIntervalHistogram();
- return latencies;
+ return stats.toCumulativeLatencies();
}
@Override
public CountersStats getCountersStats() throws IOException {
- CountersStats stats = new CountersStats();
- stats.messagesSent = totalMessagesSent.sum();
- stats.messagesReceived = totalMessagesReceived.sum();
- return stats;
+ return stats.toCountersStats();
}
@Override
@@ -295,19 +290,9 @@ public void messageReceived(ByteBuffer data, long publishTimestamp) {
}
public void internalMessageReceived(int size, long publishTimestamp) {
- messagesReceived.increment();
- totalMessagesReceived.increment();
- messagesReceivedCounter.inc();
- bytesReceived.add(size);
- bytesReceivedCounter.add(size);
-
long now = System.currentTimeMillis();
long endToEndLatencyMicros = TimeUnit.MILLISECONDS.toMicros(now - publishTimestamp);
- if (endToEndLatencyMicros > 0) {
- endToEndCumulativeLatencyRecorder.recordValue(endToEndLatencyMicros);
- endToEndLatencyRecorder.recordValue(endToEndLatencyMicros);
- endToEndLatencyStats.registerSuccessfulEvent(endToEndLatencyMicros, TimeUnit.MICROSECONDS);
- }
+ stats.recordMessageReceived(size, endToEndLatencyMicros);
while (consumersArePaused) {
try {
@@ -332,28 +317,14 @@ public void resumeConsumers() throws IOException {
@Override
public void resetStats() throws IOException {
- publishLatencyRecorder.reset();
- cumulativePublishLatencyRecorder.reset();
- endToEndLatencyRecorder.reset();
- endToEndCumulativeLatencyRecorder.reset();
+ stats.resetLatencies();
}
@Override
- public void stopAll() throws IOException {
+ public void stopAll() {
testCompleted = true;
consumersArePaused = false;
-
- publishLatencyRecorder.reset();
- cumulativePublishLatencyRecorder.reset();
- endToEndLatencyRecorder.reset();
- endToEndCumulativeLatencyRecorder.reset();
-
- messagesSent.reset();
- bytesSent.reset();
- messagesReceived.reset();
- bytesReceived.reset();
- totalMessagesSent.reset();
- totalMessagesReceived.reset();
+ stats.reset();
try {
Thread.sleep(100);
@@ -377,6 +348,11 @@ public void stopAll() throws IOException {
}
}
+ @Override
+ public String id() {
+ return "local";
+ }
+
@Override
public void close() throws Exception {
executor.shutdown();
@@ -384,8 +360,9 @@ public void close() throws Exception {
private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter();
- private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory())
- .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ private static final ObjectMapper mapper =
+ new ObjectMapper(new YAMLFactory())
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
static {
mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE);
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java
new file mode 100644
index 000000000..bf1191ef9
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker;
+
+import static io.openmessaging.benchmark.utils.UniformRateLimiter.uninterruptibleSleepNs;
+
+import io.openmessaging.benchmark.driver.BenchmarkProducer;
+import io.openmessaging.benchmark.utils.UniformRateLimiter;
+import java.util.Optional;
+import java.util.function.Supplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MessageProducer {
+
+ private final WorkerStats stats;
+ private UniformRateLimiter rateLimiter;
+ private Supplier nanoClock;
+
+ MessageProducer(UniformRateLimiter rateLimiter, WorkerStats stats) {
+ this(System::nanoTime, rateLimiter, stats);
+ }
+
+ MessageProducer(Supplier nanoClock, UniformRateLimiter rateLimiter, WorkerStats stats) {
+ this.nanoClock = nanoClock;
+ this.rateLimiter = rateLimiter;
+ this.stats = stats;
+ }
+
+ public void sendMessage(BenchmarkProducer producer, Optional key, byte[] payload) {
+ final long intendedSendTime = rateLimiter.acquire();
+ uninterruptibleSleepNs(intendedSendTime);
+ final long sendTime = nanoClock.get();
+ producer
+ .sendAsync(key, payload)
+ .thenRun(() -> success(payload.length, intendedSendTime, sendTime))
+ .exceptionally(this::failure);
+ }
+
+ private void success(long payloadLength, long intendedSendTime, long sendTime) {
+ long nowNs = nanoClock.get();
+ stats.recordProducerSuccess(payloadLength, intendedSendTime, sendTime, nowNs);
+ }
+
+ private Void failure(Throwable t) {
+ stats.recordProducerFailure();
+ log.warn("Write error on message", t);
+ return null;
+ }
+
+ private static final Logger log = LoggerFactory.getLogger(MessageProducer.class);
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java
index d7bcb96d9..60b0f15d7 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java
@@ -1,26 +1,18 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
import io.openmessaging.benchmark.worker.commands.ConsumerAssignment;
import io.openmessaging.benchmark.worker.commands.CountersStats;
@@ -28,6 +20,9 @@
import io.openmessaging.benchmark.worker.commands.PeriodStats;
import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment;
import io.openmessaging.benchmark.worker.commands.TopicsInfo;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
public interface Worker extends AutoCloseable {
@@ -57,5 +52,7 @@ public interface Worker extends AutoCloseable {
void resetStats() throws IOException;
- void stopAll() throws IOException;
+ void stopAll();
+
+ String id();
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java
index dcf63fdda..f7142cf9f 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java
@@ -1,66 +1,70 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.List;
-import org.apache.bookkeeper.stats.StatsLogger;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.io.Files;
-
import io.javalin.Context;
import io.javalin.Javalin;
import io.openmessaging.benchmark.worker.commands.ConsumerAssignment;
-import io.openmessaging.benchmark.worker.commands.CumulativeLatencies;
-import io.openmessaging.benchmark.worker.commands.PeriodStats;
import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment;
import io.openmessaging.benchmark.worker.commands.TopicsInfo;
+import io.openmessaging.benchmark.worker.jackson.ObjectMappers;
+import java.io.File;
+import java.util.List;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("unchecked")
public class WorkerHandler {
+ public static final String INITIALIZE_DRIVER = "/initialize-driver";
+ public static final String CREATE_TOPICS = "/create-topics";
+ public static final String CREATE_PRODUCERS = "/create-producers";
+ public static final String PROBE_PRODUCERS = "/probe-producers";
+ public static final String CREATE_CONSUMERS = "/create-consumers";
+ public static final String PAUSE_CONSUMERS = "/pause-consumers";
+ public static final String RESUME_CONSUMERS = "/resume-consumers";
+ public static final String START_LOAD = "/start-load";
+ public static final String ADJUST_PUBLISH_RATE = "/adjust-publish-rate";
+ public static final String STOP_ALL = "/stop-all";
+ public static final String PERIOD_STATS = "/period-stats";
+ public static final String CUMULATIVE_LATENCIES = "/cumulative-latencies";
+ public static final String COUNTERS_STATS = "/counters-stats";
+ public static final String RESET_STATS = "/reset-stats";
private final Worker localWorker;
public WorkerHandler(Javalin app, StatsLogger statsLogger) {
this.localWorker = new LocalWorker(statsLogger);
- app.post("/initialize-driver", this::handleInitializeDriver);
- app.post("/create-topics", this::handleCreateTopics);
- app.post("/create-producers", this::handleCreateProducers);
- app.post("/probe-producers", this::handleProbeProducers);
- app.post("/create-consumers", this::handleCreateConsumers);
- app.post("/pause-consumers", this::handlePauseConsumers);
- app.post("/resume-consumers", this::handleResumeConsumers);
- app.post("/start-load", this::handleStartLoad);
- app.post("/adjust-publish-rate", this::handleAdjustPublishRate);
- app.post("/stop-all", this::handleStopAll);
- app.get("/period-stats", this::handlePeriodStats);
- app.get("/cumulative-latencies", this::handleCumulativeLatencies);
- app.get("/counters-stats", this::handleCountersStats);
- app.post("/reset-stats", this::handleResetStats);
+ app.post(INITIALIZE_DRIVER, this::handleInitializeDriver);
+ app.post(CREATE_TOPICS, this::handleCreateTopics);
+ app.post(CREATE_PRODUCERS, this::handleCreateProducers);
+ app.post(PROBE_PRODUCERS, this::handleProbeProducers);
+ app.post(CREATE_CONSUMERS, this::handleCreateConsumers);
+ app.post(PAUSE_CONSUMERS, this::handlePauseConsumers);
+ app.post(RESUME_CONSUMERS, this::handleResumeConsumers);
+ app.post(START_LOAD, this::handleStartLoad);
+ app.post(ADJUST_PUBLISH_RATE, this::handleAdjustPublishRate);
+ app.post(STOP_ALL, this::handleStopAll);
+ app.get(PERIOD_STATS, this::handlePeriodStats);
+ app.get(CUMULATIVE_LATENCIES, this::handleCumulativeLatencies);
+ app.get(COUNTERS_STATS, this::handleCountersStats);
+ app.post(RESET_STATS, this::handleResetStats);
}
private void handleInitializeDriver(Context ctx) throws Exception {
@@ -92,7 +96,8 @@ private void handleProbeProducers(Context ctx) throws Exception {
private void handleCreateConsumers(Context ctx) throws Exception {
ConsumerAssignment consumerAssignment = mapper.readValue(ctx.body(), ConsumerAssignment.class);
- log.info("Received create consumers request for topics: {}", consumerAssignment.topicsSubscriptions);
+ log.info(
+ "Received create consumers request for topics: {}", consumerAssignment.topicsSubscriptions);
localWorker.createConsumers(consumerAssignment);
}
@@ -105,9 +110,12 @@ private void handleResumeConsumers(Context ctx) throws Exception {
}
private void handleStartLoad(Context ctx) throws Exception {
- ProducerWorkAssignment producerWorkAssignment = mapper.readValue(ctx.body(), ProducerWorkAssignment.class);
+ ProducerWorkAssignment producerWorkAssignment =
+ mapper.readValue(ctx.body(), ProducerWorkAssignment.class);
- log.info("Start load publish-rate: {} msg/s -- payload-size: {}", producerWorkAssignment.publishRate,
+ log.info(
+ "Start load publish-rate: {} msg/s -- payload-size: {}",
+ producerWorkAssignment.publishRate,
producerWorkAssignment.payloadData.get(0).length);
localWorker.startLoad(producerWorkAssignment);
@@ -125,45 +133,11 @@ private void handleStopAll(Context ctx) throws Exception {
}
private void handlePeriodStats(Context ctx) throws Exception {
- PeriodStats stats = localWorker.getPeriodStats();
-
- // Serialize histograms
- synchronized (histogramSerializationBuffer) {
- histogramSerializationBuffer.clear();
- stats.publishLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer);
- stats.publishLatencyBytes = new byte[histogramSerializationBuffer.position()];
- histogramSerializationBuffer.flip();
- histogramSerializationBuffer.get(stats.publishLatencyBytes);
-
- histogramSerializationBuffer.clear();
- stats.endToEndLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer);
- stats.endToEndLatencyBytes = new byte[histogramSerializationBuffer.position()];
- histogramSerializationBuffer.flip();
- histogramSerializationBuffer.get(stats.endToEndLatencyBytes);
- }
-
- ctx.result(writer.writeValueAsString(stats));
+ ctx.result(writer.writeValueAsString(localWorker.getPeriodStats()));
}
private void handleCumulativeLatencies(Context ctx) throws Exception {
- CumulativeLatencies stats = localWorker.getCumulativeLatencies();
-
- // Serialize histograms
- synchronized (histogramSerializationBuffer) {
- histogramSerializationBuffer.clear();
- stats.publishLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer);
- stats.publishLatencyBytes = new byte[histogramSerializationBuffer.position()];
- histogramSerializationBuffer.flip();
- histogramSerializationBuffer.get(stats.publishLatencyBytes);
-
- histogramSerializationBuffer.clear();
- stats.endToEndLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer);
- stats.endToEndLatencyBytes = new byte[histogramSerializationBuffer.position()];
- histogramSerializationBuffer.flip();
- histogramSerializationBuffer.get(stats.endToEndLatencyBytes);
- }
-
- ctx.result(writer.writeValueAsString(stats));
+ ctx.result(writer.writeValueAsString(localWorker.getCumulativeLatencies()));
}
private void handleCountersStats(Context ctx) throws Exception {
@@ -175,17 +149,8 @@ private void handleResetStats(Context ctx) throws Exception {
localWorker.resetStats();
}
- private final ByteBuffer histogramSerializationBuffer = ByteBuffer.allocate(1024 * 1024);
-
private static final Logger log = LoggerFactory.getLogger(WorkerHandler.class);
- private static final ObjectMapper mapper = new ObjectMapper()
- .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- static {
- mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE);
- }
-
- private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter();
-
+ private static final ObjectMapper mapper = ObjectMappers.DEFAULT.mapper();
+ private static final ObjectWriter writer = ObjectMappers.DEFAULT.writer();
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java
new file mode 100644
index 000000000..68b9d5345
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker;
+
+
+import io.openmessaging.benchmark.worker.commands.CountersStats;
+import io.openmessaging.benchmark.worker.commands.CumulativeLatencies;
+import io.openmessaging.benchmark.worker.commands.PeriodStats;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
+import org.HdrHistogram.Recorder;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+
+public class WorkerStats {
+
+ private final StatsLogger statsLogger;
+
+ private final OpStatsLogger publishDelayLatencyStats;
+
+ private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5);
+ private final Recorder endToEndCumulativeLatencyRecorder =
+ new Recorder(TimeUnit.HOURS.toMicros(12), 5);
+ private final OpStatsLogger endToEndLatencyStats;
+
+ private final LongAdder messagesSent = new LongAdder();
+ private final LongAdder messageSendErrors = new LongAdder();
+ private final LongAdder bytesSent = new LongAdder();
+ private final Counter messageSendErrorCounter;
+ private final Counter messagesSentCounter;
+ private final Counter bytesSentCounter;
+
+ private final LongAdder messagesReceived = new LongAdder();
+ private final LongAdder bytesReceived = new LongAdder();
+ private final Counter messagesReceivedCounter;
+ private final Counter bytesReceivedCounter;
+
+ private final LongAdder totalMessagesSent = new LongAdder();
+ private final LongAdder totalMessageSendErrors = new LongAdder();
+ private final LongAdder totalMessagesReceived = new LongAdder();
+
+ private static final long highestTrackableValue = TimeUnit.SECONDS.toMicros(60);
+ private final Recorder publishLatencyRecorder = new Recorder(highestTrackableValue, 5);
+ private final Recorder cumulativePublishLatencyRecorder = new Recorder(highestTrackableValue, 5);
+ private final OpStatsLogger publishLatencyStats;
+
+ private final Recorder publishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5);
+ private final Recorder cumulativePublishDelayLatencyRecorder =
+ new Recorder(highestTrackableValue, 5);
+
+ WorkerStats(StatsLogger statsLogger) {
+ this.statsLogger = statsLogger;
+
+ StatsLogger producerStatsLogger = statsLogger.scope("producer");
+ this.messagesSentCounter = producerStatsLogger.getCounter("messages_sent");
+ this.messageSendErrorCounter = producerStatsLogger.getCounter("message_send_errors");
+ this.bytesSentCounter = producerStatsLogger.getCounter("bytes_sent");
+ this.publishDelayLatencyStats = producerStatsLogger.getOpStatsLogger("producer_delay_latency");
+ this.publishLatencyStats = producerStatsLogger.getOpStatsLogger("produce_latency");
+
+ StatsLogger consumerStatsLogger = statsLogger.scope("consumer");
+ this.messagesReceivedCounter = consumerStatsLogger.getCounter("messages_recv");
+ this.bytesReceivedCounter = consumerStatsLogger.getCounter("bytes_recv");
+ this.endToEndLatencyStats = consumerStatsLogger.getOpStatsLogger("e2e_latency");
+ }
+
+ public StatsLogger getStatsLogger() {
+ return statsLogger;
+ }
+
+ public void recordMessageSent() {
+ totalMessagesSent.increment();
+ }
+
+ public void recordMessageReceived(long payloadLength, long endToEndLatencyMicros) {
+ messagesReceived.increment();
+ totalMessagesReceived.increment();
+ messagesReceivedCounter.inc();
+ bytesReceived.add(payloadLength);
+ bytesReceivedCounter.add(payloadLength);
+
+ if (endToEndLatencyMicros > 0) {
+ endToEndCumulativeLatencyRecorder.recordValue(endToEndLatencyMicros);
+ endToEndLatencyRecorder.recordValue(endToEndLatencyMicros);
+ endToEndLatencyStats.registerSuccessfulEvent(endToEndLatencyMicros, TimeUnit.MICROSECONDS);
+ }
+ }
+
+ public PeriodStats toPeriodStats() {
+ PeriodStats stats = new PeriodStats();
+
+ stats.messagesSent = messagesSent.sumThenReset();
+ stats.messageSendErrors = messageSendErrors.sumThenReset();
+ stats.bytesSent = bytesSent.sumThenReset();
+
+ stats.messagesReceived = messagesReceived.sumThenReset();
+ stats.bytesReceived = bytesReceived.sumThenReset();
+
+ stats.totalMessagesSent = totalMessagesSent.sum();
+ stats.totalMessageSendErrors = totalMessageSendErrors.sum();
+ stats.totalMessagesReceived = totalMessagesReceived.sum();
+
+ stats.publishLatency = publishLatencyRecorder.getIntervalHistogram();
+ stats.publishDelayLatency = publishDelayLatencyRecorder.getIntervalHistogram();
+ stats.endToEndLatency = endToEndLatencyRecorder.getIntervalHistogram();
+ return stats;
+ }
+
+ public CumulativeLatencies toCumulativeLatencies() {
+ CumulativeLatencies latencies = new CumulativeLatencies();
+ latencies.publishLatency = cumulativePublishLatencyRecorder.getIntervalHistogram();
+ latencies.publishDelayLatency = cumulativePublishDelayLatencyRecorder.getIntervalHistogram();
+ latencies.endToEndLatency = endToEndCumulativeLatencyRecorder.getIntervalHistogram();
+ return latencies;
+ }
+
+ public CountersStats toCountersStats() throws IOException {
+ CountersStats stats = new CountersStats();
+ stats.messagesSent = totalMessagesSent.sum();
+ stats.messageSendErrors = totalMessageSendErrors.sum();
+ stats.messagesReceived = totalMessagesReceived.sum();
+ return stats;
+ }
+
+ public void resetLatencies() {
+ publishLatencyRecorder.reset();
+ cumulativePublishLatencyRecorder.reset();
+ publishDelayLatencyRecorder.reset();
+ cumulativePublishDelayLatencyRecorder.reset();
+ endToEndLatencyRecorder.reset();
+ endToEndCumulativeLatencyRecorder.reset();
+ }
+
+ public void reset() {
+ resetLatencies();
+
+ messagesSent.reset();
+ messageSendErrors.reset();
+ bytesSent.reset();
+ messagesReceived.reset();
+ bytesReceived.reset();
+ totalMessagesSent.reset();
+ totalMessagesReceived.reset();
+ }
+
+ public void recordProducerFailure() {
+ messageSendErrors.increment();
+ messageSendErrorCounter.inc();
+ totalMessageSendErrors.increment();
+ }
+
+ public void recordProducerSuccess(
+ long payloadLength, long intendedSendTimeNs, long sendTimeNs, long nowNs) {
+ messagesSent.increment();
+ totalMessagesSent.increment();
+ messagesSentCounter.inc();
+ bytesSent.add(payloadLength);
+ bytesSentCounter.add(payloadLength);
+
+ final long latencyMicros =
+ Math.min(highestTrackableValue, TimeUnit.NANOSECONDS.toMicros(nowNs - sendTimeNs));
+ publishLatencyRecorder.recordValue(latencyMicros);
+ cumulativePublishLatencyRecorder.recordValue(latencyMicros);
+ publishLatencyStats.registerSuccessfulEvent(latencyMicros, TimeUnit.MICROSECONDS);
+
+ final long sendDelayMicros =
+ Math.min(
+ highestTrackableValue, TimeUnit.NANOSECONDS.toMicros(sendTimeNs - intendedSendTimeNs));
+ publishDelayLatencyRecorder.recordValue(sendDelayMicros);
+ cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros);
+ publishDelayLatencyStats.registerSuccessfulEvent(sendDelayMicros, TimeUnit.MICROSECONDS);
+ }
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java
index edb7961f6..9d97642ec 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
+
import java.util.ArrayList;
import java.util.List;
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java
index 0d4c7e8ac..6ddc593e1 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java
@@ -1,24 +1,32 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
public class CountersStats {
public long messagesSent;
public long messagesReceived;
+ public long messageSendErrors;
+
+ public CountersStats plus(CountersStats toAdd) {
+ CountersStats result = new CountersStats();
+ result.messagesSent += this.messagesSent;
+ result.messagesReceived += this.messagesReceived;
+ result.messageSendErrors += this.messageSendErrors;
+
+ result.messagesSent += toAdd.messagesSent;
+ result.messagesReceived += toAdd.messagesReceived;
+ result.messageSendErrors += toAdd.messageSendErrors;
+ return result;
+ }
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java
index 7cff435ff..e9afea679 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java
@@ -1,36 +1,40 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
-import java.util.concurrent.TimeUnit;
+import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.concurrent.TimeUnit.SECONDS;
import org.HdrHistogram.Histogram;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
public class CumulativeLatencies {
- @JsonIgnore
- public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5);
- public byte[] publishLatencyBytes;
+ public Histogram publishLatency = new Histogram(SECONDS.toMicros(60), 5);
+ public Histogram publishDelayLatency = new Histogram(SECONDS.toMicros(60), 5);
+ public Histogram endToEndLatency = new Histogram(HOURS.toMicros(12), 5);
+
+ public CumulativeLatencies plus(CumulativeLatencies toAdd) {
+ CumulativeLatencies result = new CumulativeLatencies();
+
+ result.publishLatency.add(this.publishLatency);
+ result.publishDelayLatency.add(this.publishDelayLatency);
+ result.endToEndLatency.add(this.endToEndLatency);
+
+ result.publishLatency.add(toAdd.publishLatency);
+ result.publishDelayLatency.add(toAdd.publishDelayLatency);
+ result.endToEndLatency.add(toAdd.endToEndLatency);
- @JsonIgnore
- public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5);
- public byte[] endToEndLatencyBytes;
+ return result;
+ }
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java
index f5b5704ab..3f06f6925 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java
@@ -1,44 +1,66 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
-import java.util.concurrent.TimeUnit;
+import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.concurrent.TimeUnit.SECONDS;
import org.HdrHistogram.Histogram;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
public class PeriodStats {
public long messagesSent = 0;
+ public long messageSendErrors = 0;
public long bytesSent = 0;
public long messagesReceived = 0;
public long bytesReceived = 0;
public long totalMessagesSent = 0;
+ public long totalMessageSendErrors = 0;
public long totalMessagesReceived = 0;
- @JsonIgnore
- public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5);
- public byte[] publishLatencyBytes;
+ public Histogram publishLatency = new Histogram(SECONDS.toMicros(60), 5);
+ public Histogram publishDelayLatency = new Histogram(SECONDS.toMicros(60), 5);
+ public Histogram endToEndLatency = new Histogram(HOURS.toMicros(12), 5);
+
+ public PeriodStats plus(PeriodStats toAdd) {
+ PeriodStats result = new PeriodStats();
+
+ result.messagesSent += this.messagesSent;
+ result.messageSendErrors += this.messageSendErrors;
+ result.bytesSent += this.bytesSent;
+ result.messagesReceived += this.messagesReceived;
+ result.bytesReceived += this.bytesReceived;
+ result.totalMessagesSent += this.totalMessagesSent;
+ result.totalMessageSendErrors += this.totalMessageSendErrors;
+ result.totalMessagesReceived += this.totalMessagesReceived;
+ result.publishLatency.add(this.publishLatency);
+ result.publishDelayLatency.add(this.publishDelayLatency);
+ result.endToEndLatency.add(this.endToEndLatency);
+
+ result.messagesSent += toAdd.messagesSent;
+ result.messageSendErrors += toAdd.messageSendErrors;
+ result.bytesSent += toAdd.bytesSent;
+ result.messagesReceived += toAdd.messagesReceived;
+ result.bytesReceived += toAdd.bytesReceived;
+ result.totalMessagesSent += toAdd.totalMessagesSent;
+ result.totalMessageSendErrors += toAdd.totalMessageSendErrors;
+ result.totalMessagesReceived += toAdd.totalMessagesReceived;
+ result.publishLatency.add(toAdd.publishLatency);
+ result.publishDelayLatency.add(toAdd.publishDelayLatency);
+ result.endToEndLatency.add(toAdd.endToEndLatency);
- @JsonIgnore
- public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5);
- public byte[] endToEndLatencyBytes;
+ return result;
+ }
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java
index ac8041dfb..319bd7cc8 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java
@@ -1,32 +1,43 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
-import io.openmessaging.benchmark.utils.distributor.KeyDistributorType;
+import io.openmessaging.benchmark.utils.distributor.KeyDistributorType;
import java.util.List;
public class ProducerWorkAssignment {
-
+
public List payloadData;
-
+
+ /**
+ * Weights for weighted payload selection. If null, uniform random selection is used.
+ * Each weight corresponds to the payload at the same index in payloadData.
+ * Used for message size distribution feature.
+ */
+ public int[] payloadWeights;
+
public double publishRate;
public KeyDistributorType keyDistributorType;
+
+ public ProducerWorkAssignment withPublishRate(double publishRate) {
+ ProducerWorkAssignment copy = new ProducerWorkAssignment();
+ copy.keyDistributorType = this.keyDistributorType;
+ copy.payloadData = this.payloadData;
+ copy.payloadWeights = this.payloadWeights;
+ copy.publishRate = publishRate;
+ return copy;
+ }
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java
index 8dc0d7d30..bd9bf9989 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
@@ -22,12 +17,22 @@ public class TopicSubscription {
public String topic;
public String subscription;
- public TopicSubscription() {
- }
+ public TopicSubscription() {}
public TopicSubscription(String topic, String subscription) {
this.topic = topic;
this.subscription = subscription;
}
+ @Override
+ public String toString() {
+ return "TopicSubscription{"
+ + "topic='"
+ + topic
+ + '\''
+ + ", subscription='"
+ + subscription
+ + '\''
+ + '}';
+ }
}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java
index 34bc968e5..90545c725 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.worker.commands;
@@ -22,8 +17,7 @@ public class TopicsInfo {
public int numberOfTopics;
public int numberOfPartitionsPerTopic;
- public TopicsInfo() {
- }
+ public TopicsInfo() {}
public TopicsInfo(int numberOfTopics, int numberOfPartitionsPerTopic) {
this.numberOfTopics = numberOfTopics;
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java
new file mode 100644
index 000000000..6aa562bed
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.jackson;
+
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.util.ByteBufferBackedOutputStream;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import org.HdrHistogram.Histogram;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HistogramDeserializer extends StdDeserializer {
+
+ private final ThreadLocal threadBuffer =
+ ThreadLocal.withInitial(() -> ByteBuffer.allocate(8 * 1024 * 1024));
+
+ public HistogramDeserializer() {
+ super(Histogram.class);
+ }
+
+ @Override
+ public Histogram deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
+ throws IOException, JacksonException {
+ ByteBuffer buffer = threadBuffer.get();
+ buffer.clear();
+ try (OutputStream os = new ByteBufferBackedOutputStream(buffer)) {
+ jsonParser.readBinaryValue(os);
+ buffer.flip();
+ // Long.MIN_VALUE used so that Histogram will defer to the value encoded in the histogram
+ // value. This assumes that it is acceptable for the deserialized value we create to
+ // share the same parameters of the source histogram that was serialized.
+ return Histogram.decodeFromCompressedByteBuffer(buffer, Long.MIN_VALUE);
+ } catch (Exception e) {
+ log.error(
+ "Failed to decode publish delay latency: {}",
+ ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(buffer)));
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static final Logger log = LoggerFactory.getLogger(HistogramDeserializer.class);
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java
new file mode 100644
index 000000000..c1c253f16
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.jackson;
+
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.HdrHistogram.Histogram;
+
+public class HistogramSerializer extends StdSerializer {
+
+ private final ThreadLocal threadBuffer =
+ ThreadLocal.withInitial(() -> ByteBuffer.allocate(8 * 1024 * 1024));
+
+ public HistogramSerializer() {
+ super(Histogram.class);
+ }
+
+ static byte[] toByteArray(ByteBuffer buffer) {
+ byte[] encodedBuffer = new byte[buffer.remaining()];
+ buffer.get(encodedBuffer);
+ return encodedBuffer;
+ }
+
+ static ByteBuffer serializeHistogram(Histogram histo, ByteBuffer buffer) {
+ buffer.clear();
+ while (true) {
+ final int outBytes = histo.encodeIntoCompressedByteBuffer(buffer);
+ Preconditions.checkState(outBytes == buffer.position());
+ final int capacity = buffer.capacity();
+ if (outBytes < capacity) {
+ // encoding succesful
+ break;
+ }
+ // We filled the entire buffer, an indication that the buffer was not
+ // large enough, so we double the buffer and try again.
+ // See: https://github.com/HdrHistogram/HdrHistogram/issues/201
+ buffer = ByteBuffer.allocate(capacity * 2);
+ }
+ buffer.flip();
+ return buffer;
+ }
+
+ @Override
+ public void serialize(
+ Histogram histogram, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
+ throws IOException {
+ ByteBuffer buffer = threadBuffer.get();
+ ByteBuffer newBuffer = serializeHistogram(histogram, buffer);
+ if (newBuffer != buffer) {
+ threadBuffer.set(newBuffer);
+ }
+ jsonGenerator.writeBinary(toByteArray(newBuffer));
+ }
+}
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java
new file mode 100644
index 000000000..680a5dccc
--- /dev/null
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.jackson;
+
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.HdrHistogram.Histogram;
+
+public enum ObjectMappers {
+ DEFAULT;
+
+ private static final ObjectMapper mapper =
+ new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ static {
+ mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE);
+ SimpleModule module = new SimpleModule();
+ module.addSerializer(Histogram.class, new HistogramSerializer());
+ module.addDeserializer(Histogram.class, new HistogramDeserializer());
+ mapper.registerModule(module);
+ }
+
+ private static final ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
+
+ public ObjectMapper mapper() {
+ return mapper;
+ }
+
+ public ObjectWriter writer() {
+ return writer;
+ }
+}
diff --git a/benchmark-framework/src/main/resources/log4j2.yaml b/benchmark-framework/src/main/resources/log4j2.yaml
index d2d565e1e..3083a28e3 100644
--- a/benchmark-framework/src/main/resources/log4j2.yaml
+++ b/benchmark-framework/src/main/resources/log4j2.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
Configuration:
@@ -26,13 +21,22 @@ Configuration:
name: Console
target: SYSTEM_OUT
PatternLayout:
- Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level - %msg%n"
-
+ Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n"
+ RollingFile:
+ name: RollingFile
+ fileName: benchmark-worker.log
+ filePattern: benchmark-worker.log.%d{yyyy-MM-dd-hh-mm-ss}.gz
+ PatternLayout:
+ Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n"
+ Policies:
+ SizeBasedTriggeringPolicy:
+ size: 100MB
+ DefaultRollOverStrategy:
+ max: 10
Loggers:
Root:
level: info
additivity: false
AppenderRef:
- ref: Console
-
-
\ No newline at end of file
+ - ref: Console
+ - ref: RollingFile
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java
new file mode 100644
index 000000000..6f014f019
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class RateControllerTest {
+ private final RateController rateController = new RateController();
+ private double rate = 10_000;
+ private long periodNanos = SECONDS.toNanos(1);
+
+ @Test
+ void receiveBacklog() {
+ assertThat(rateController.getRampingFactor()).isEqualTo(1);
+
+ // no backlog
+ rate = rateController.nextRate(rate, periodNanos, 10_000, 10_000);
+ assertThat(rate).isEqualTo(20_000);
+ assertThat(rateController.getRampingFactor()).isEqualTo(1);
+
+ // receive backlog
+ rate = rateController.nextRate(rate, periodNanos, 20_000, 15_000);
+ assertThat(rate).isEqualTo(5_000);
+ assertThat(rateController.getRampingFactor()).isEqualTo(0.5);
+ }
+
+ @Test
+ void publishBacklog() {
+ assertThat(rateController.getRampingFactor()).isEqualTo(1);
+
+ // no backlog
+ rate = rateController.nextRate(rate, periodNanos, 10_000, 10_000);
+ assertThat(rate).isEqualTo(20_000);
+ assertThat(rateController.getRampingFactor()).isEqualTo(1);
+
+ // publish backlog
+ rate = rateController.nextRate(rate, periodNanos, 15_000, 20_000);
+ assertThat(rate).isEqualTo(5_000);
+ assertThat(rateController.getRampingFactor()).isEqualTo(0.5);
+ }
+
+ @Test
+ void rampUp() {
+ assertThat(rateController.getRampingFactor()).isEqualTo(1);
+
+ // receive backlog
+ rate = rateController.nextRate(rate, periodNanos, 10_000, 5_000);
+ assertThat(rate).isEqualTo(5_000);
+ assertThat(rateController.getRampingFactor()).isEqualTo(0.5);
+
+ // no backlog
+ rate = rateController.nextRate(rate, periodNanos, 20_000, 20_000);
+ assertThat(rate).isEqualTo(10_000);
+ assertThat(rateController.getRampingFactor()).isEqualTo(1);
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java
new file mode 100644
index 000000000..cf62b85dd
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class EnvTest {
+ private static final String ENV_KEY = "KEY";
+
+ @Test
+ void envLong() throws Exception {
+ withEnvironmentVariable(ENV_KEY, "2")
+ .execute(
+ () -> {
+ assertThat(Env.getLong(ENV_KEY, 1)).isEqualTo(2);
+ });
+ }
+
+ @Test
+ void envLongDefault() throws Exception {
+ withEnvironmentVariable(ENV_KEY, null)
+ .execute(
+ () -> {
+ assertThat(Env.getLong(ENV_KEY, 1)).isEqualTo(1);
+ });
+ }
+
+ @Test
+ void envDouble() throws Exception {
+ withEnvironmentVariable(ENV_KEY, "2.34")
+ .execute(
+ () -> {
+ assertThat(Env.getDouble(ENV_KEY, 1.23)).isEqualTo(2.34);
+ });
+ }
+
+ @Test
+ void envDoubleDefault() throws Exception {
+ withEnvironmentVariable(ENV_KEY, null)
+ .execute(
+ () -> {
+ assertThat(Env.getDouble(ENV_KEY, 1.23)).isEqualTo(1.23);
+ });
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java
new file mode 100644
index 000000000..a7472df95
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+import static java.util.Arrays.asList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+class ListPartitionTest {
+
+ @Test
+ void partitionList() {
+ List list = asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
+ List> lists = ListPartition.partitionList(list, 3);
+ assertThat(lists)
+ .satisfies(
+ s -> {
+ assertThat(s).hasSize(3);
+ assertThat(s.get(0)).isEqualTo(asList(1, 4, 7, 10));
+ assertThat(s.get(1)).isEqualTo(asList(2, 5, 8));
+ assertThat(s.get(2)).isEqualTo(asList(3, 6, 9));
+ });
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java
new file mode 100644
index 000000000..be1b9a532
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class PaddingDecimalFormatTest {
+
+ @Test
+ void format() {
+ PaddingDecimalFormat format = new PaddingDecimalFormat("0.0", 7);
+ assertThat(format.format(1L)).isEqualTo(" 1.0");
+ assertThat(format.format(1000L)).isEqualTo(" 1000.0");
+ assertThat(format.format(10000000L)).isEqualTo("10000000.0");
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java
new file mode 100644
index 000000000..87004e332
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.function.Supplier;
+import org.junit.jupiter.api.Test;
+
+class TimerTest {
+
+ @Test
+ void elapsedMillis() {
+ Supplier mockClock = mock(Supplier.class);
+ when(mockClock.get()).thenReturn(MILLISECONDS.toNanos(1), MILLISECONDS.toNanos(3));
+ Timer timer = new Timer(mockClock);
+ assertThat(timer.elapsedMillis()).isEqualTo(2.0d);
+ }
+
+ @Test
+ void elapsedSeconds() {
+ Supplier mockClock = mock(Supplier.class);
+ when(mockClock.get()).thenReturn(SECONDS.toNanos(1), SECONDS.toNanos(3));
+ Timer timer = new Timer(mockClock);
+ assertThat(timer.elapsedSeconds()).isEqualTo(2.0d);
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java
new file mode 100644
index 000000000..a9b90462e
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.function.Supplier;
+import org.junit.jupiter.api.Test;
+
+class UniformRateLimiterTest {
+
+ @Test
+ void getOpsPerSec() {
+ assertThat(new UniformRateLimiter(1000).getOpsPerSec()).isEqualTo(1000.0d);
+ }
+
+ @Test
+ void getIntervalNs() {
+ assertThat(new UniformRateLimiter(1000).getIntervalNs()).isEqualTo(SECONDS.toNanos(1) / 1000);
+ }
+
+ @Test
+ void acquireSlowSingleThread() {
+ Supplier mockClock = mock(Supplier.class);
+ when(mockClock.get()).thenReturn(SECONDS.toNanos(2));
+ UniformRateLimiter rateLimiter = new UniformRateLimiter(1000, mockClock);
+ assertThat(rateLimiter.acquire()).isEqualTo(2000000000L);
+ assertThat(rateLimiter.acquire()).isEqualTo(2001000000L);
+ assertThat(rateLimiter.acquire()).isEqualTo(2002000000L);
+ }
+
+ @Test
+ void uninterruptibleSleepNs() {
+ long start = System.nanoTime();
+ long expectedEnd = start + MILLISECONDS.toNanos(100);
+ UniformRateLimiter.uninterruptibleSleepNs(expectedEnd);
+ long end = System.nanoTime();
+ assertThat(end).isGreaterThan(expectedEnd);
+ }
+
+ @Test
+ void cinitExceptions() {
+ assertThatCode(() -> new UniformRateLimiter(Double.NaN))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatCode(() -> new UniformRateLimiter(1.0d / 0.0d))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatCode(() -> new UniformRateLimiter(-0.1)).isInstanceOf(IllegalArgumentException.class);
+ assertThatCode(() -> new UniformRateLimiter(0.0)).isInstanceOf(IllegalArgumentException.class);
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/payload/MessageSizeDistributionTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/payload/MessageSizeDistributionTest.java
new file mode 100644
index 000000000..affa77536
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/payload/MessageSizeDistributionTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.utils.payload;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+
+class MessageSizeDistributionTest {
+
+ @Test
+ void parsesBasicRanges() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-256", 100);
+ config.put("256-1024", 200);
+ config.put("1024-4096", 300);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+
+ assertThat(dist.getBucketCount()).isEqualTo(3);
+ assertThat(dist.getTotalWeight()).isEqualTo(600);
+ }
+
+ @Test
+ void parsesSizeSuffixes() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-1KB", 100);
+ config.put("1KB-1MB", 200);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+ List sizes = dist.getBucketSizes();
+
+ // Midpoint of 0-1024 = 512
+ assertThat(sizes.get(0)).isEqualTo(512);
+ // Midpoint of 1024-1048576 = 524800
+ assertThat(sizes.get(1)).isEqualTo(524800);
+ }
+
+ @Test
+ void calculatesBucketSizesAsMidpoint() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-100", 1);
+ config.put("100-200", 1);
+ config.put("200-300", 1);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+ List sizes = dist.getBucketSizes();
+
+ assertThat(sizes).containsExactly(50, 150, 250);
+ }
+
+ @Test
+ void calculatesWeightsArray() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-256", 234);
+ config.put("256-1024", 456);
+ config.put("1024-4096", 678);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+ int[] weights = dist.getWeights();
+
+ assertThat(weights).containsExactly(234, 456, 678);
+ }
+
+ @Test
+ void calculatesCumulativeWeights() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-256", 100);
+ config.put("256-1024", 200);
+ config.put("1024-4096", 300);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+ int[] cumulative = dist.getCumulativeWeights();
+
+ assertThat(cumulative).containsExactly(100, 300, 600);
+ }
+
+ @Test
+ void calculatesAverageSize() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-100", 1); // midpoint 50, weight 1
+ config.put("100-200", 1); // midpoint 150, weight 1
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+
+ // (50 * 1 + 150 * 1) / 2 = 100
+ assertThat(dist.getAvgSize()).isEqualTo(100);
+ }
+
+ @Test
+ void calculatesMaxSize() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-256", 100);
+ config.put("256-1024", 200);
+ config.put("1024-5000", 300);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+
+ assertThat(dist.getMaxSize()).isEqualTo(5000);
+ }
+
+ @Test
+ void handlesProductionDistribution() {
+ // Real production distribution from the plan
+ Map config = new LinkedHashMap<>();
+ config.put("0-256", 234);
+ config.put("256-1024", 456);
+ config.put("1024-4096", 678);
+ config.put("4096-16384", 312);
+ config.put("16384-65536", 98);
+ config.put("65536-262144", 45);
+ config.put("262144-1048576", 18);
+ config.put("1048576-5242880", 6);
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+
+ assertThat(dist.getBucketCount()).isEqualTo(8);
+ assertThat(dist.getTotalWeight()).isEqualTo(1847);
+ assertThat(dist.getMaxSize()).isEqualTo(5242880); // 5MB
+ assertThat(dist.getBucketSizes()).hasSize(8);
+ assertThat(dist.getWeights()).hasSize(8);
+ }
+
+ @Test
+ void throwsOnEmptyConfig() {
+ assertThatThrownBy(() -> new MessageSizeDistribution(new HashMap<>()))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void throwsOnNullConfig() {
+ assertThatThrownBy(() -> new MessageSizeDistribution(null))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void throwsOnInvalidRangeFormat() {
+ Map config = new HashMap<>();
+ config.put("invalid", 100);
+
+ assertThatThrownBy(() -> new MessageSizeDistribution(config))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void throwsOnNegativeWeight() {
+ Map config = new HashMap<>();
+ config.put("0-100", -1);
+
+ assertThatThrownBy(() -> new MessageSizeDistribution(config))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void throwsOnZeroTotalWeight() {
+ Map config = new HashMap<>();
+ config.put("0-100", 0);
+ config.put("100-200", 0);
+
+ assertThatThrownBy(() -> new MessageSizeDistribution(config))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void throwsOnInvalidMinMax() {
+ Map config = new HashMap<>();
+ config.put("200-100", 1); // max < min
+
+ assertThatThrownBy(() -> new MessageSizeDistribution(config))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void weightedSelectionProducesCorrectDistribution() {
+ Map config = new LinkedHashMap<>();
+ config.put("0-100", 100); // 50% weight
+ config.put("100-200", 100); // 50% weight
+
+ MessageSizeDistribution dist = new MessageSizeDistribution(config);
+ int[] cumulative = dist.getCumulativeWeights();
+ int totalWeight = dist.getTotalWeight();
+
+ // Simulate weighted selection like LocalWorker does
+ int[] counts = new int[2];
+ int iterations = 100000;
+
+ for (int i = 0; i < iterations; i++) {
+ int target = i % totalWeight; // Deterministic for testing
+ int idx = Arrays.binarySearch(cumulative, target + 1);
+ if (idx < 0) {
+ idx = -idx - 1;
+ }
+ idx = Math.min(idx, 1);
+ counts[idx]++;
+ }
+
+ // Each bucket should get approximately 50% of selections
+ double ratio0 = (double) counts[0] / iterations;
+ double ratio1 = (double) counts[1] / iterations;
+
+ assertThat(ratio0).isBetween(0.49, 0.51);
+ assertThat(ratio1).isBetween(0.49, 0.51);
+ }
+}
+
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java
new file mode 100644
index 000000000..c6f4805a1
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class DistributedWorkersEnsembleTest {
+
+ @ParameterizedTest
+ @MethodSource("producerCountExpectations")
+ void getNumberOfProducerWorkers(int workerCount, boolean extraConsumers, int expected) {
+ List workers = mock(List.class);
+ when(workers.size()).thenReturn(workerCount);
+ assertThat(DistributedWorkersEnsemble.getNumberOfProducerWorkers(workers, extraConsumers))
+ .isEqualTo(expected);
+ }
+
+ private static Stream producerCountExpectations() {
+ return Stream.of(
+ Arguments.of(2, true, 1),
+ Arguments.of(3, true, 1),
+ Arguments.of(4, true, 2),
+ Arguments.of(5, true, 2),
+ Arguments.of(6, true, 2),
+ Arguments.of(7, true, 3),
+ Arguments.of(8, true, 3),
+ Arguments.of(9, true, 3),
+ Arguments.of(2, false, 1),
+ Arguments.of(3, false, 1),
+ Arguments.of(4, false, 2),
+ Arguments.of(5, false, 2),
+ Arguments.of(6, false, 3),
+ Arguments.of(7, false, 3),
+ Arguments.of(8, false, 4),
+ Arguments.of(9, false, 4));
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java
new file mode 100644
index 000000000..699304de0
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.commands;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class CountersStatsTest {
+
+ @Test
+ void plus() {
+ CountersStats one = new CountersStats();
+ one.messagesSent = 1;
+ one.messageSendErrors = 10;
+ one.messagesReceived = 100;
+ CountersStats two = new CountersStats();
+ two.messagesSent = 2;
+ two.messageSendErrors = 20;
+ two.messagesReceived = 200;
+
+ CountersStats result = one.plus(two);
+ assertThat(result)
+ .satisfies(
+ r -> {
+ assertThat(r.messagesSent).isEqualTo(3);
+ assertThat(r.messageSendErrors).isEqualTo(30);
+ assertThat(r.messagesReceived).isEqualTo(300);
+ });
+ }
+
+ @Test
+ void zeroPlus() {
+ CountersStats zero = new CountersStats();
+ CountersStats two = new CountersStats();
+ two.messagesSent = 2;
+ two.messageSendErrors = 20;
+ two.messagesReceived = 200;
+
+ CountersStats result = zero.plus(two);
+ assertThat(result)
+ .satisfies(
+ r -> {
+ assertThat(r.messagesSent).isEqualTo(2);
+ assertThat(r.messageSendErrors).isEqualTo(20);
+ assertThat(r.messagesReceived).isEqualTo(200);
+ });
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java
new file mode 100644
index 000000000..5919739ff
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.commands;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class CumulativeLatenciesTest {
+
+ @Test
+ void zeroPlus() {
+ CumulativeLatencies one = new CumulativeLatencies();
+ CumulativeLatencies two = new CumulativeLatencies();
+ CumulativeLatencies result = one.plus(two);
+ assertThat(result)
+ .satisfies(
+ r -> {
+ assertThat(r.publishLatency).isEqualTo(two.publishLatency);
+ assertThat(r.publishDelayLatency).isEqualTo(two.publishDelayLatency);
+ assertThat(r.endToEndLatency).isEqualTo(two.endToEndLatency);
+ });
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java
new file mode 100644
index 000000000..01c3e04b5
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.commands;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class PeriodStatsTest {
+
+ @Test
+ void plus() {
+ PeriodStats one = new PeriodStats();
+ one.messagesSent = 1;
+ one.messageSendErrors = 2;
+ one.bytesSent = 3;
+ one.messagesReceived = 4;
+ one.bytesReceived = 5;
+ one.totalMessagesSent = 6;
+ one.totalMessageSendErrors = 7;
+ one.totalMessagesReceived = 8;
+ PeriodStats two = new PeriodStats();
+ two.messagesSent = 10;
+ two.messageSendErrors = 20;
+ two.bytesSent = 30;
+ two.messagesReceived = 40;
+ two.bytesReceived = 50;
+ two.totalMessagesSent = 60;
+ two.totalMessageSendErrors = 70;
+ two.totalMessagesReceived = 80;
+ PeriodStats result = one.plus(two);
+ assertThat(result)
+ .satisfies(
+ r -> {
+ assertThat(r.messagesSent).isEqualTo(11);
+ assertThat(r.messageSendErrors).isEqualTo(22);
+ assertThat(r.bytesSent).isEqualTo(33);
+ assertThat(r.messagesReceived).isEqualTo(44);
+ assertThat(r.bytesReceived).isEqualTo(55);
+ assertThat(r.totalMessagesSent).isEqualTo(66);
+ assertThat(r.totalMessageSendErrors).isEqualTo(77);
+ assertThat(r.totalMessagesReceived).isEqualTo(88);
+
+ two.publishLatency.add(one.publishLatency);
+ two.publishDelayLatency.add(one.publishDelayLatency);
+ two.endToEndLatency.add(one.endToEndLatency);
+
+ assertThat(r.publishLatency).isEqualTo(two.publishLatency);
+ assertThat(r.publishDelayLatency).isEqualTo(two.publishDelayLatency);
+ assertThat(r.endToEndLatency).isEqualTo(two.endToEndLatency);
+ });
+ }
+
+ @Test
+ void zeroPlus() {
+ PeriodStats one = new PeriodStats();
+ PeriodStats two = new PeriodStats();
+ two.messagesSent = 10;
+ two.messageSendErrors = 20;
+ two.bytesSent = 30;
+ two.messagesReceived = 40;
+ two.bytesReceived = 50;
+ two.totalMessagesSent = 60;
+ two.totalMessageSendErrors = 70;
+ two.totalMessagesReceived = 80;
+ PeriodStats result = one.plus(two);
+ assertThat(result)
+ .satisfies(
+ r -> {
+ assertThat(r.messagesSent).isEqualTo(10);
+ assertThat(r.messageSendErrors).isEqualTo(20);
+ assertThat(r.bytesSent).isEqualTo(30);
+ assertThat(r.messagesReceived).isEqualTo(40);
+ assertThat(r.bytesReceived).isEqualTo(50);
+ assertThat(r.totalMessagesSent).isEqualTo(60);
+ assertThat(r.totalMessageSendErrors).isEqualTo(70);
+ assertThat(r.totalMessagesReceived).isEqualTo(80);
+
+ assertThat(r.publishLatency).isEqualTo(two.publishLatency);
+ assertThat(r.publishDelayLatency).isEqualTo(two.publishDelayLatency);
+ assertThat(r.endToEndLatency).isEqualTo(two.endToEndLatency);
+ });
+ }
+}
diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java
new file mode 100644
index 000000000..9769eac6f
--- /dev/null
+++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.worker.jackson;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import org.HdrHistogram.Histogram;
+import org.junit.jupiter.api.Test;
+
+class HistogramSerDeTest {
+
+ @Test
+ void deserialize() throws IOException {
+ Histogram value = new Histogram(100_000, 3);
+ value.recordValue(1);
+ value.recordValue(100);
+ value.recordValue(10_000);
+
+ ObjectMapper mapper = new ObjectMapper();
+ SimpleModule module = new SimpleModule();
+ module.addSerializer(Histogram.class, new HistogramSerializer());
+ module.addDeserializer(Histogram.class, new HistogramDeserializer());
+ mapper.registerModule(module);
+
+ byte[] serialized = mapper.writeValueAsBytes(value);
+ Histogram deserialized = mapper.readValue(serialized, Histogram.class);
+
+ assertThat(deserialized).isEqualTo(value);
+ }
+
+ /**
+ * Create a random histogram and insert the given number of samples.
+ *
+ * @param samples the number of samples to record into the histogram
+ * @return a Histogram with the given number of samples
+ */
+ private Histogram randomHisto(int samples) {
+ Random r = new Random(0xBADBEEF);
+ Histogram h = new org.HdrHistogram.Histogram(5);
+ for (int i = 0; i < samples; i++) {
+ h.recordValue(r.nextInt(10000000));
+ }
+
+ return h;
+ }
+
+ byte[] serializeRandomHisto(int samples, int initialBufferSize) throws Exception {
+ ByteBuffer inbuffer = ByteBuffer.allocate(initialBufferSize);
+ Histogram inHisto = randomHisto(samples);
+ byte[] serialBytes =
+ HistogramSerializer.toByteArray(HistogramSerializer.serializeHistogram(inHisto, inbuffer));
+
+ // check roundtrip
+ Histogram outHisto =
+ Histogram.decodeFromCompressedByteBuffer(ByteBuffer.wrap(serialBytes), Long.MIN_VALUE);
+ assertThat(inHisto).isEqualTo(outHisto);
+
+ return serialBytes;
+ }
+
+ @Test
+ public void testHistogram() throws Exception {
+
+ // in the worker it's 8 MB but it takes a while to make a histogram that big
+ final int bufSize = 1002;
+
+ int samples = 300;
+
+ // we do an exponential search to fit the crossover point where we need to grow the buffer
+ while (true) {
+ byte[] serialBytes = serializeRandomHisto(samples, bufSize);
+ // System.out.println("Samples: " + samples + ", histogram size: " + serialBytes.length);
+ if (serialBytes.length >= bufSize) {
+ break;
+ }
+ samples *= 1.05;
+ }
+
+ // then walk backwards across the point linearly with increment of 1 to check the boundary
+ // carefully
+ while (true) {
+ samples--;
+ byte[] serialBytes = serializeRandomHisto(samples, bufSize);
+ // System.out.println("Samples: " + samples + ", histogram size: " + serialBytes.length);
+ if (serialBytes.length < bufSize - 10) {
+ break;
+ }
+ }
+ }
+}
diff --git a/benchmark-framework/src/test/resources/log4j2.yaml b/benchmark-framework/src/test/resources/log4j2.yaml
new file mode 100644
index 000000000..7ed3f311f
--- /dev/null
+++ b/benchmark-framework/src/test/resources/log4j2.yaml
@@ -0,0 +1,33 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Configuration:
+ status: INFO
+ name: messaging-benchmark
+
+ Appenders:
+ Console:
+ name: Console
+ target: SYSTEM_OUT
+ PatternLayout:
+ Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n"
+ Loggers:
+ Logger:
+ - name: io.openmessaging
+ level: debug
+ Root:
+ level: info
+ additivity: false
+ AppenderRef:
+ - ref: Console
diff --git a/bin/benchmark b/bin/benchmark
index c5452d973..555893496 100755
--- a/bin/benchmark
+++ b/bin/benchmark
@@ -1,21 +1,16 @@
#!/bin/bash
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
if [ -d "./lib" ]; then
@@ -32,4 +27,4 @@ fi
JVM_MEM="${HEAP_OPTS} -XX:+UseG1GC"
JVM_GC_LOG=" -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=64m -Xloggc:/dev/shm/benchmark-client-gc_%p.log"
-java -server -cp $CLASSPATH $JVM_MEM io.openmessaging.benchmark.Benchmark $*
+java -server -cp $CLASSPATH $JVM_MEM io.openmessaging.benchmark.Benchmark "$@"
diff --git a/bin/benchmark-worker b/bin/benchmark-worker
index 8447a4cc0..164f45b80 100755
--- a/bin/benchmark-worker
+++ b/bin/benchmark-worker
@@ -1,21 +1,16 @@
#!/bin/bash
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
if [ -d "./lib" ]; then
@@ -31,5 +26,12 @@ fi
JVM_MEM="${HEAP_OPTS} -XX:+UseG1GC"
JVM_GC_LOG=" -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=64m -Xloggc:/dev/shm/benchmark-client-gc_%p.log"
-
-exec java -server -cp $CLASSPATH $JVM_MEM io.openmessaging.benchmark.worker.BenchmarkWorker $*
+# Required by Netty for optimized direct byte buffer access
+JVM_OPTS="$JVM_OPTS -Dio.netty.tryReflectionSetAccessible=true"
+JVM_OPTS="$JVM_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+# Set io.netty.tryReflectionSetAccessible in Pulsar's shaded client
+JVM_OPTS="$JVM_OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true"
+# Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms
+# reflection access to java.util.zip.CRC32C
+JVM_OPTS="$JVM_OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+exec java -server -cp $CLASSPATH $JVM_MEM $JVM_OPTS io.openmessaging.benchmark.worker.BenchmarkWorker "$@"
\ No newline at end of file
diff --git a/bin/create_charts.py b/bin/create_charts.py
index 54391abd1..909051620 100755
--- a/bin/create_charts.py
+++ b/bin/create_charts.py
@@ -1,21 +1,16 @@
#!/usr/bin/env python
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -43,10 +38,18 @@ def create_charts(test_results):
y_label='Latency (ms)',
time_series=[(x['driver'], x['publishLatency99pct']) for x in results])
+ create_chart(workload, 'Publish Delay latency 99pct',
+ y_label='Latency (us)',
+ time_series=[(x['driver'], x['publishDelayLatency99pct']) for x in results])
+
create_chart(workload, 'Publish rate',
y_label='Rate (msg/s)',
time_series=[(x['driver'], x['publishRate']) for x in results])
+ create_chart(workload, 'Publish Error rate',
+ y_label='Rate (err/s)',
+ time_series=[(x['driver'], x['publishErrorRate']) for x in results])
+
create_chart(workload, 'End To End Latency 95pct',
y_label='Latency (ms)',
time_series=[(x['driver'], x['endToEndLatency95pct']) for x in results])
@@ -63,6 +66,10 @@ def create_charts(test_results):
y_label='Latency (ms)',
time_series=[(x['driver'], x['aggregatedPublishLatencyQuantiles']) for x in results])
+ create_quantile_chart(workload, 'Publish Delay Latency Quantiles',
+ y_label='Latency (us)',
+ time_series=[(x['driver'], x['aggregatedPublishDelayLatencyQuantiles']) for x in results])
+
create_quantile_chart(workload, 'End To End Latency Quantiles',
y_label='Latency (ms)',
time_series=[(x['driver'], x['aggregatedEndToEndLatencyQuantiles']) for x in results])
diff --git a/deployment/kubernetes/helm/README.md b/deployment/kubernetes/helm/README.md
index ac43ac652..f8ef3125a 100644
--- a/deployment/kubernetes/helm/README.md
+++ b/deployment/kubernetes/helm/README.md
@@ -1,11 +1,10 @@
-
Users can deploy the helm chart:
```bash
$ helm install ./benchmark --name benchmark
```
-After the chart has started, users can exec into the pod name "benchmark-driver" and run the benchmark from there.
+After the chart has started, users can exec into the pod name "benchmark-driver" and run the benchmark from there.
For example, once inside the "benchmark-driver" pod, users can execute:
diff --git a/deployment/kubernetes/helm/benchmark/Chart.yaml b/deployment/kubernetes/helm/benchmark/Chart.yaml
index a1dcacefd..85d0c9556 100644
--- a/deployment/kubernetes/helm/benchmark/Chart.yaml
+++ b/deployment/kubernetes/helm/benchmark/Chart.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
apiVersion: v1
diff --git a/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml b/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml
index bd77b7fba..250169401 100644
--- a/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml
+++ b/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
apiVersion: apps/v1
diff --git a/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml b/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml
index aa440dc3a..4310b79e8 100644
--- a/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml
+++ b/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
apiVersion: v1
@@ -40,7 +35,7 @@ spec:
- name: WORKERS
value: '{{ template "workers" .Values }}'
- name: HEAP_OPTS
- value: 'Xms4G -Xmx4G'
+ value: '-Xms4G -Xmx4G'
command: ["sh", "-c"]
args:
- >
diff --git a/deployment/kubernetes/helm/benchmark/values.yaml b/deployment/kubernetes/helm/benchmark/values.yaml
index aee1f260f..e4963a2bd 100644
--- a/deployment/kubernetes/helm/benchmark/values.yaml
+++ b/deployment/kubernetes/helm/benchmark/values.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
numWorkers: 8
diff --git a/dev/license-header.txt b/dev/license-header.txt
deleted file mode 100644
index 60b675e31..000000000
--- a/dev/license-header.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index e1af6fa52..243487eb9 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,28 +1,31 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
-FROM openjdk:8-jdk
+FROM eclipse-temurin:17
ARG BENCHMARK_TARBALL
+# Using ADD instead of COPY because ${BENCHMARK_TARBALL} is an archive that needs to be extracted
ADD ${BENCHMARK_TARBALL} /
RUN mv openmessaging-benchmark-* /benchmark
WORKDIR /benchmark
+
+# Install vim
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y vim tzdata && rm -rf /var/lib/apt/lists/*
+
+# Start a shell by default
+CMD ["/bin/bash"]
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 79c0520c9..d3b3129f9 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -1,31 +1,33 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# Build the Project
-FROM maven:latest as build
+FROM maven:3.9.9-eclipse-temurin-17 as build
COPY . /benchmark
WORKDIR /benchmark
RUN mvn install
# Create the benchmark image
-FROM openjdk:8-jdk
+FROM eclipse-temurin:17
COPY --from=build /benchmark/package/target/openmessaging-benchmark-*-SNAPSHOT-bin.tar.gz /
RUN mkdir /benchmark && tar -xzf openmessaging-benchmark-*-SNAPSHOT-bin.tar.gz -C /benchmark --strip=1
RUN rm /openmessaging-benchmark-*-SNAPSHOT-bin.tar.gz
-WORKDIR /benchmark
\ No newline at end of file
+WORKDIR /benchmark
+
+# Install vim
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y vim tzdata && rm -rf /var/lib/apt/lists/*
+
+# Start a shell by default
+CMD ["/bin/bash"]
diff --git a/docker/README.md b/docker/README.md
index 1cefa026d..df4a283f2 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,18 +1,48 @@
# OpenMessaging Benchmark Framework Docker
-You can use either of the Dockerfiles - `./docker/Dockerfile` or `./docker/Dockerfile.build` based on your needs.
+
+## Precondition
+
+You need to install [Eclipse Temurin 17](https://adoptium.net/) (or 21)
+and set the JAVA_HOME environment variable to its installation directory.
+
+## Building the image
+
+You can use one of the Dockerfiles based on your needs:
+
+- `Dockerfile` - requires local build first
+- `Dockerfile.build` - builds inside Docker (JDK 17 only)
+- `Dockerfile.builder` - **recommended** - multi-JDK support (17 and 21)
+
+### `Dockerfile.builder` (Recommended)
+
+Supports both JDK 17 and JDK 21. Skips formatting checks that can have
+JDK version incompatibilities in container environments.
+
+```bash
+# Build with JDK 17 (default)
+docker build -t omb-benchmark:jdk17 -f docker/Dockerfile.builder .
+
+# Build with JDK 21
+docker build -t omb-benchmark:jdk21 --build-arg JDK_VERSION=21 -f docker/Dockerfile.builder .
+```
### `Dockerfile`
-Uses `openjdk-8` and takes `BENCHMARK_TARBALL` as an argument.
+
+Uses `eclipse-temurin:17` and takes `BENCHMARK_TARBALL` as an argument.
While using this Dockerfile, you will need to build the project locally **first**.
-```
-#> mvn build
-#> export BENCHMARK_TARBALL=package/target/openmessaging-benchmark--SNAPSHOT-bin.tar.gz
-#> docker build --build-arg BENCHMARK_TARBALL . -f docker/Dockerfile
+
+```bash
+mvn install -DskipTests
+export BENCHMARK_TARBALL=package/target/openmessaging-benchmark--SNAPSHOT-bin.tar.gz
+docker build -t openmessaging-benchmark:latest --build-arg BENCHMARK_TARBALL . -f docker/Dockerfile
```
### `Dockerfile.build`
-Uses the latest version of `maven` in order to build the project, and then use `openjdk-8` as runtime.
-This Dockerfile has no dependency (you do not need Mavent to be installed locally).
+
+Uses Maven to build the project inside Docker, then uses `eclipse-temurin:17` as runtime.
+This Dockerfile has no local dependency (you do not need Maven installed locally).
+
+```bash
+docker build -t openmessaging-benchmark:latest . -f docker/Dockerfile.build
```
-#> docker build . -f docker/Dockerfile.build
-```
\ No newline at end of file
+
diff --git a/docker/pom.xml b/docker/pom.xml
index 3b72c5261..6f9925976 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -1,36 +1,32 @@
+
+ 4.0.0
- messaging-benchmark
io.openmessaging.benchmark
+ messaging-benchmark
0.0.1-SNAPSHOT
- 4.0.0
messaging-benchmark-docker
+ pom
Open Messaging Benchmark Docker Image
- pom
1.3.7
@@ -57,6 +53,14 @@
com.spotify
dockerfile-maven-plugin
${dockerfile-maven.version}
+
+ ${docker.organization}/openmessaging-benchmark
+ false
+ ${project.version}
+
+ target/package-${project.version}-bin.tar.gz
+
+
default
@@ -85,14 +89,6 @@
-
- ${docker.organization}/openmessaging-benchmark
- false
- ${project.version}
-
- target/package-${project.version}-bin.tar.gz
-
-
org.apache.maven.plugins
diff --git a/driver-api/pom.xml b/driver-api/pom.xml
index 1f1cc7942..cbf66b392 100644
--- a/driver-api/pom.xml
+++ b/driver-api/pom.xml
@@ -1,40 +1,55 @@
+
- 4.0.0
-
- io.openmessaging.benchmark
- messaging-benchmark
- 0.0.1-SNAPSHOT
- ..
-
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+
+ io.openmessaging.benchmark
+ messaging-benchmark
+ 0.0.1-SNAPSHOT
+
- driver-api
+ driver-api
-
-
- org.apache.bookkeeper.stats
- bookkeeper-stats-api
- ${bookkeeper.version}
-
-
-
\ No newline at end of file
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+
+
+ com.fasterxml.jackson.jaxrs
+ jackson-jaxrs-base
+
+
+ com.fasterxml.jackson.jaxrs
+ jackson-jaxrs-json-provider
+
+
+ org.apache.bookkeeper.stats
+ bookkeeper-stats-api
+ ${bookkeeper.version}
+
+
+ org.projectlombok
+ lombok
+
+
+
diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java
index 0499767d6..a8e745e88 100644
--- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java
+++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java
@@ -1,23 +1,16 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver;
-public interface BenchmarkConsumer extends AutoCloseable {
-
-}
+public interface BenchmarkConsumer extends AutoCloseable {}
diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java
index 45f20ff1a..88c5814aa 100644
--- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java
+++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java
@@ -1,72 +1,143 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver;
+import static java.util.stream.Collectors.toList;
+
import java.io.File;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.CompletableFuture;
+import lombok.Value;
import org.apache.bookkeeper.stats.StatsLogger;
-/**
- * Base driver interface
- */
+/** Base driver interface. */
public interface BenchmarkDriver extends AutoCloseable {
/**
- * Driver implementation can use this method to initialize the client libraries, with the provided configuration
- * file.
- *
- * The format of the configuration file is specific to the driver implementation.
- *
+ * Driver implementation can use this method to initialize the client libraries, with the provided
+ * configuration file.
+ *
+ *
The format of the configuration file is specific to the driver implementation.
+ *
* @param configurationFile
* @param statsLogger stats logger to collect stats from benchmark driver
* @throws IOException
*/
- void initialize(File configurationFile, StatsLogger statsLogger) throws IOException;
+ void initialize(File configurationFile, StatsLogger statsLogger)
+ throws IOException, InterruptedException;
/**
- * Get a driver specific prefix to be used in creating multiple topic names
+ * Get a driver specific prefix to be used in creating multiple topic names.
+ *
+ * @return the topic name prefix
*/
String getTopicNamePrefix();
/**
- * Create a new topic with a given number of partitions
+ * Create a new topic with a given number of partitions.
+ *
+ * @param topic
+ * @param partitions
+ * @return a future the completes when the topic is created
*/
CompletableFuture createTopic(String topic, int partitions);
/**
- * Create a producer for a given topic
+ * Create a list of new topics with the given number of partitions.
+ *
+ * @param topicInfos
+ * @return a future the completes when the topics are created
+ */
+ default CompletableFuture createTopics(List topicInfos) {
+ @SuppressWarnings("unchecked")
+ CompletableFuture[] futures =
+ topicInfos.stream()
+ .map(topicInfo -> createTopic(topicInfo.getTopic(), topicInfo.getPartitions()))
+ .toArray(CompletableFuture[]::new);
+ return CompletableFuture.allOf(futures);
+ }
+
+ /**
+ * Create a producer for a given topic.
+ *
+ * @param topic
+ * @return a producer future
*/
CompletableFuture createProducer(String topic);
+ /**
+ * Create a producers for a given topic.
+ *
+ * @param producers
+ * @return a producers future
+ */
+ default CompletableFuture> createProducers(List producers) {
+ List> futures =
+ producers.stream().map(ci -> createProducer(ci.getTopic())).collect(toList());
+ return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
+ .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(toList()));
+ }
+
/**
* Create a benchmark consumer relative to one particular topic and subscription.
- *
- * It is responsibility of the driver implementation to invoke the consumerCallback each time a message
- * is received.
- *
+ *
+ * It is responsibility of the driver implementation to invoke the consumerCallback
+ * each time a message is received.
+ *
* @param topic
* @param subscriptionName
* @param consumerCallback
- * @return
+ * @return a consumer future
*/
CompletableFuture createConsumer(
- String topic,
- String subscriptionName,
- ConsumerCallback consumerCallback);
+ String topic, String subscriptionName, ConsumerCallback consumerCallback);
+
+ /**
+ * Create a consumers for a given topic.
+ *
+ * @param consumers
+ * @return a consumers future
+ */
+ default CompletableFuture> createConsumers(List consumers) {
+ List> futures =
+ consumers.stream()
+ .map(
+ ci ->
+ createConsumer(
+ ci.getTopic(), ci.getSubscriptionName(), ci.getConsumerCallback()))
+ .collect(toList());
+ return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
+ .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(toList()));
+ }
+
+ @Value
+ class TopicInfo {
+ String topic;
+ int partitions;
+ }
+
+ @Value
+ class ProducerInfo {
+ int id;
+ String topic;
+ }
+
+ @Value
+ class ConsumerInfo {
+ int id;
+ String topic;
+ String subscriptionName;
+ ConsumerCallback consumerCallback;
+ }
}
diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java
index 5f5bf996f..1662cf1c6 100644
--- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java
+++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver;
+
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@@ -26,12 +22,9 @@ public interface BenchmarkProducer extends AutoCloseable {
/**
* Publish a message and return a callback to track the completion of the operation.
*
- * @param key
- * the key associated with this message
- * @param payload
- * the message payload
+ * @param key the key associated with this message
+ * @param payload the message payload
* @return a future that will be triggered when the message is successfully published
*/
CompletableFuture sendAsync(Optional key, byte[] payload);
-
}
diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java
index 24c55968a..b68ecf451 100644
--- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java
+++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java
@@ -1,46 +1,36 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver;
+
import java.nio.ByteBuffer;
-/**
- * Callback that the driver implementation calls when a message is received
- */
+/** Callback that the driver implementation calls when a message is received. */
public interface ConsumerCallback {
/**
- * Driver should invoke this method (or the ByteBuffer variant) once for each message received
- *
- * @param payload
- * the received message payload
- * @param publishTimestamp
- * the publish timestamp in milliseconds
+ * Driver should invoke this method (or the ByteBuffer variant) once for each message received.
+ *
+ * @param payload the received message payload
+ * @param publishTimestamp the publish timestamp in milliseconds
*/
void messageReceived(byte[] payload, long publishTimestamp);
/**
- * Driver should invoke this method (or the byte[] variant) once for each message received
+ * Driver should invoke this method (or the byte[] variant) once for each message received.
*
- * @param payload
- * the received message payload
- * @param publishTimestamp
- * the publish timestamp in milliseconds
+ * @param payload the received message payload
+ * @param publishTimestamp the publish timestamp in milliseconds
*/
void messageReceived(ByteBuffer payload, long publishTimestamp);
}
diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java
new file mode 100644
index 000000000..3bdfe5372
--- /dev/null
+++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.openmessaging.benchmark.driver;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.stream.Collectors.toMap;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.Value;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class ResourceCreator {
+ private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ private final String name;
+ private final int maxBatchSize;
+ private final long interBatchDelayMs;
+ private final Function, Map>> invokeBatchFn;
+ private final Function, CreationResult> complete;
+
+ public CompletableFuture> create(List resources) {
+ return CompletableFuture.completedFuture(createBlocking(resources));
+ }
+
+ private List createBlocking(List resources) {
+ BlockingQueue queue = new ArrayBlockingQueue<>(resources.size(), true, resources);
+ List batch = new ArrayList<>();
+ List created = new ArrayList<>();
+ AtomicInteger succeeded = new AtomicInteger();
+
+ ScheduledFuture loggingFuture =
+ executor.scheduleAtFixedRate(
+ () -> log.info("Created {}s {}/{}", name, succeeded.get(), resources.size()),
+ 10,
+ 10,
+ SECONDS);
+
+ try {
+ while (succeeded.get() < resources.size()) {
+ int batchSize = queue.drainTo(batch, maxBatchSize);
+ if (batchSize > 0) {
+ executeBatch(batch)
+ .forEach(
+ (resource, result) -> {
+ if (result.success) {
+ created.add(result.created);
+ succeeded.incrementAndGet();
+ } else {
+ //noinspection ResultOfMethodCallIgnored
+ queue.offer(resource);
+ }
+ });
+ batch.clear();
+ }
+ }
+ } finally {
+ loggingFuture.cancel(true);
+ }
+ return created;
+ }
+
+ @SneakyThrows
+ private Map> executeBatch(List batch) {
+ log.debug("Executing batch, size: {}", batch.size());
+ Thread.sleep(interBatchDelayMs);
+ return invokeBatchFn.apply(batch).entrySet().stream()
+ .collect(toMap(Map.Entry::getKey, e -> complete.apply(e.getValue())));
+ }
+
+ @Value
+ public static class CreationResult {
+ C created;
+ boolean success;
+ }
+}
diff --git a/driver-artemis/artemis.yaml b/driver-artemis/artemis.yaml
index 3529ea61f..75dc0342f 100644
--- a/driver-artemis/artemis.yaml
+++ b/driver-artemis/artemis.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: Artemis
diff --git a/driver-artemis/pom.xml b/driver-artemis/pom.xml
index 20b0f50b9..fe8eed01d 100644
--- a/driver-artemis/pom.xml
+++ b/driver-artemis/pom.xml
@@ -1,47 +1,40 @@
+
- 4.0.0
-
- io.openmessaging.benchmark
- messaging-benchmark
- 0.0.1-SNAPSHOT
- ..
-
-
- driver-artemis
-
-
-
-
- ${project.groupId}
- driver-api
- ${project.version}
-
-
-
- org.apache.activemq
- artemis-core-client
- 2.4.0
-
-
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+
+ io.openmessaging.benchmark
+ messaging-benchmark
+ 0.0.1-SNAPSHOT
+
+
+ driver-artemis
+
+
+
+ ${project.groupId}
+ driver-api
+ ${project.version}
+
+
+ org.apache.activemq
+ artemis-core-client
+ 2.23.1
+
+
diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java
index 6f6a71e64..e0b988219 100644
--- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java
+++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java
@@ -1,23 +1,21 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.artemis;
+
+import io.openmessaging.benchmark.driver.BenchmarkConsumer;
+import io.openmessaging.benchmark.driver.ConsumerCallback;
import org.apache.activemq.artemis.api.core.ActiveMQException;
import org.apache.activemq.artemis.api.core.RoutingType;
import org.apache.activemq.artemis.api.core.SimpleString;
@@ -27,31 +25,36 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import io.openmessaging.benchmark.driver.BenchmarkConsumer;
-import io.openmessaging.benchmark.driver.ConsumerCallback;
-
public class ArtemisBenchmarkConsumer implements BenchmarkConsumer {
private final ClientSession session;
private final ClientConsumer consumer;
- public ArtemisBenchmarkConsumer(String topic, String queueName, ClientSessionFactory sessionFactory, ConsumerCallback callback)
+ public ArtemisBenchmarkConsumer(
+ String topic,
+ String queueName,
+ ClientSessionFactory sessionFactory,
+ ConsumerCallback callback)
throws ActiveMQException {
session = sessionFactory.createSession();
- session.createQueue(SimpleString.toSimpleString(topic), RoutingType.MULTICAST,
- SimpleString.toSimpleString(queueName), true /* durable */);
+ session.createQueue(
+ SimpleString.toSimpleString(topic),
+ RoutingType.MULTICAST,
+ SimpleString.toSimpleString(queueName),
+ true /* durable */);
consumer = session.createConsumer(queueName);
- consumer.setMessageHandler(message -> {
- byte[] payload = new byte[message.getBodyBuffer().readableBytes()];
- message.getBodyBuffer().readBytes(payload);
- callback.messageReceived(payload, message.getTimestamp());
- try {
- message.acknowledge();
- } catch (ActiveMQException e) {
- log.warn("Failed to acknowledge message", e);
- }
- });
-
+ consumer.setMessageHandler(
+ message -> {
+ byte[] payload = new byte[message.getBodyBuffer().readableBytes()];
+ message.getBodyBuffer().readBytes(payload);
+ callback.messageReceived(payload, message.getTimestamp());
+ try {
+ message.acknowledge();
+ } catch (ActiveMQException e) {
+ log.warn("Failed to acknowledge message", e);
+ }
+ });
+
session.start();
}
diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java
index f547109bb..67db26990 100644
--- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java
+++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java
@@ -1,28 +1,31 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.artemis;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import io.openmessaging.benchmark.driver.BenchmarkConsumer;
+import io.openmessaging.benchmark.driver.BenchmarkDriver;
+import io.openmessaging.benchmark.driver.BenchmarkProducer;
+import io.openmessaging.benchmark.driver.ConsumerCallback;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ForkJoinPool;
-
import org.apache.activemq.artemis.api.core.ActiveMQException;
import org.apache.activemq.artemis.api.core.RoutingType;
import org.apache.activemq.artemis.api.core.SimpleString;
@@ -34,16 +37,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-
-import io.openmessaging.benchmark.driver.BenchmarkConsumer;
-import io.openmessaging.benchmark.driver.BenchmarkDriver;
-import io.openmessaging.benchmark.driver.BenchmarkProducer;
-import io.openmessaging.benchmark.driver.ConsumerCallback;
-
public class ArtemisBenchmarkDriver implements BenchmarkDriver {
private ArtemisConfig config;
@@ -56,8 +49,15 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I
log.info("ActiveMQ Artemis driver configuration: {}", writer.writeValueAsString(config));
try {
ServerLocator serverLocator = ActiveMQClient.createServerLocator(config.brokerAddress);
- serverLocator.setConfirmationWindowSize(1000);
-
+ // confirmation window size is in bytes, set to 1MB
+ serverLocator.setConfirmationWindowSize(1024 * 1024);
+ // use asynchronous sending of messages where SendAcknowledgementHandler reports
+ // success/failure
+ serverLocator.setBlockOnDurableSend(false);
+ serverLocator.setBlockOnNonDurableSend(false);
+ // use async acknowledgement
+ serverLocator.setBlockOnAcknowledge(false);
+
sessionFactory = serverLocator.createSessionFactory();
session = sessionFactory.createSession();
} catch (Exception e) {
@@ -74,18 +74,22 @@ public String getTopicNamePrefix() {
public CompletableFuture createTopic(String topic, int partitions) {
CompletableFuture future = new CompletableFuture<>();
if (partitions != 1) {
- future.completeExceptionally(new IllegalArgumentException("Partitions are not supported in Artemis"));
+ future.completeExceptionally(
+ new IllegalArgumentException("Partitions are not supported in Artemis"));
return future;
}
- ForkJoinPool.commonPool().submit(() -> {
- try {
- session.createAddress(SimpleString.toSimpleString(topic), RoutingType.MULTICAST, true);
- future.complete(null);
- } catch (ActiveMQException e) {
- future.completeExceptionally(e);
- }
- });
+ ForkJoinPool.commonPool()
+ .submit(
+ () -> {
+ try {
+ session.createAddress(
+ SimpleString.toSimpleString(topic), RoutingType.MULTICAST, true);
+ future.complete(null);
+ } catch (ActiveMQException e) {
+ future.completeExceptionally(e);
+ }
+ });
return future;
}
@@ -102,19 +106,22 @@ public CompletableFuture createProducer(String topic) {
}
@Override
- public CompletableFuture createConsumer(String topic, String subscriptionName,
- ConsumerCallback consumerCallback) {
+ public CompletableFuture createConsumer(
+ String topic, String subscriptionName, ConsumerCallback consumerCallback) {
CompletableFuture future = new CompletableFuture<>();
- ForkJoinPool.commonPool().submit(() -> {
- try {
- String queueName = topic + "-" + subscriptionName;
- BenchmarkConsumer consumer = new ArtemisBenchmarkConsumer(topic, queueName, sessionFactory,
- consumerCallback);
- future.complete(consumer);
- } catch (ActiveMQException e) {
- future.completeExceptionally(e);
- }
- });
+ ForkJoinPool.commonPool()
+ .submit(
+ () -> {
+ try {
+ String queueName = topic + "-" + subscriptionName;
+ BenchmarkConsumer consumer =
+ new ArtemisBenchmarkConsumer(
+ topic, queueName, sessionFactory, consumerCallback);
+ future.complete(consumer);
+ } catch (ActiveMQException e) {
+ future.completeExceptionally(e);
+ }
+ });
return future;
}
@@ -134,8 +141,9 @@ public void close() throws Exception {
log.info("ActiveMQ Artemis benchmark driver successfully shut down");
}
- private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory())
- .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ private static final ObjectMapper mapper =
+ new ObjectMapper(new YAMLFactory())
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
private static ArtemisConfig readConfig(File configurationFile) throws IOException {
return mapper.readValue(configurationFile, ArtemisConfig.class);
diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java
index 1db656d5e..0b40546fa 100644
--- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java
+++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java
@@ -1,40 +1,37 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.artemis;
+
+import io.openmessaging.benchmark.driver.BenchmarkProducer;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
-
import org.apache.activemq.artemis.api.core.ActiveMQException;
+import org.apache.activemq.artemis.api.core.Message;
import org.apache.activemq.artemis.api.core.client.ClientMessage;
import org.apache.activemq.artemis.api.core.client.ClientProducer;
import org.apache.activemq.artemis.api.core.client.ClientSession;
import org.apache.activemq.artemis.api.core.client.ClientSessionFactory;
-
-import io.openmessaging.benchmark.driver.BenchmarkProducer;
+import org.apache.activemq.artemis.api.core.client.SendAcknowledgementHandler;
public class ArtemisBenchmarkProducer implements BenchmarkProducer {
private final ClientSession session;
private final ClientProducer producer;
- public ArtemisBenchmarkProducer(String address, ClientSessionFactory sessionFactory) throws ActiveMQException {
+ public ArtemisBenchmarkProducer(String address, ClientSessionFactory sessionFactory)
+ throws ActiveMQException {
session = sessionFactory.createSession();
producer = session.createProducer(address);
session.start();
@@ -48,20 +45,29 @@ public void close() throws Exception {
@Override
public CompletableFuture sendAsync(Optional key, byte[] payload) {
- ClientMessage msg = session.createMessage(true /* durable */ );
+ ClientMessage msg = session.createMessage(true /* durable */);
msg.setTimestamp(System.currentTimeMillis());
msg.getBodyBuffer().writeBytes(payload);
CompletableFuture future = new CompletableFuture<>();
try {
- producer.send(msg, message -> {
- future.complete(null);
- });
+ producer.send(
+ msg,
+ new SendAcknowledgementHandler() {
+ @Override
+ public void sendAcknowledged(Message message) {
+ future.complete(null);
+ }
+
+ @Override
+ public void sendFailed(Message message, Exception e) {
+ future.completeExceptionally(e);
+ }
+ });
} catch (ActiveMQException e) {
future.completeExceptionally(e);
}
return future;
}
-
}
diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java
index 6aed0d5ed..418340b66 100644
--- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java
+++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.artemis;
diff --git a/driver-bookkeeper/bookkeeper.yaml b/driver-bookkeeper/bookkeeper.yaml
index 8c049101e..3a6153b7f 100644
--- a/driver-bookkeeper/bookkeeper.yaml
+++ b/driver-bookkeeper/bookkeeper.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: BookKeeper
diff --git a/driver-bookkeeper/deploy/deploy.yaml b/driver-bookkeeper/deploy/deploy.yaml
index be4df52a1..87c1af4da 100644
--- a/driver-bookkeeper/deploy/deploy.yaml
+++ b/driver-bookkeeper/deploy/deploy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
- name: Format and mount disks for BookKeeper hosts
diff --git a/driver-bookkeeper/deploy/templates/bookkeeper.conf b/driver-bookkeeper/deploy/templates/bookkeeper.conf
index ad8d9f937..e7ec1bdb6 100644
--- a/driver-bookkeeper/deploy/templates/bookkeeper.conf
+++ b/driver-bookkeeper/deploy/templates/bookkeeper.conf
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
diff --git a/driver-bookkeeper/deploy/templates/prometheus.yml b/driver-bookkeeper/deploy/templates/prometheus.yml
index 7286f762f..e32538ef1 100644
--- a/driver-bookkeeper/deploy/templates/prometheus.yml
+++ b/driver-bookkeeper/deploy/templates/prometheus.yml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# my global config
diff --git a/driver-bookkeeper/deploy/templates/pulsar_env.sh b/driver-bookkeeper/deploy/templates/pulsar_env.sh
index 1a271b119..903b7df08 100644
--- a/driver-bookkeeper/deploy/templates/pulsar_env.sh
+++ b/driver-bookkeeper/deploy/templates/pulsar_env.sh
@@ -1,21 +1,16 @@
#!/usr/bin/env bash
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# Set JAVA_HOME here to override the environment setting
diff --git a/driver-bookkeeper/deploy/templates/workers.yaml b/driver-bookkeeper/deploy/templates/workers.yaml
index c9ef4e456..1415c4807 100644
--- a/driver-bookkeeper/deploy/templates/workers.yaml
+++ b/driver-bookkeeper/deploy/templates/workers.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
diff --git a/driver-bookkeeper/deploy/templates/zoo.cfg b/driver-bookkeeper/deploy/templates/zoo.cfg
index b33cad819..dbd337067 100644
--- a/driver-bookkeeper/deploy/templates/zoo.cfg
+++ b/driver-bookkeeper/deploy/templates/zoo.cfg
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# The number of milliseconds of each tick
diff --git a/driver-bookkeeper/pom.xml b/driver-bookkeeper/pom.xml
index 08e8a9397..3bbea8691 100644
--- a/driver-bookkeeper/pom.xml
+++ b/driver-bookkeeper/pom.xml
@@ -1,47 +1,43 @@
+
- 4.0.0
-
- io.openmessaging.benchmark
- messaging-benchmark
- 0.0.1-SNAPSHOT
- ..
-
-
- driver-bookkeeper
-
-
-
-
- ${project.groupId}
- driver-api
- ${project.version}
-
-
-
- org.apache.distributedlog
- distributedlog-core-shaded
- 4.7.0
-
-
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+
+ io.openmessaging.benchmark
+ messaging-benchmark
+ 0.0.1-SNAPSHOT
+
+
+ driver-bookkeeper
+
+
+ ${project.groupId}
+ driver-api
+ ${project.version}
+
+
+ io.netty
+ netty-all
+
+
+ org.apache.distributedlog
+ distributedlog-core-shaded
+ 4.14.4
+
+
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java
index afaaedc25..db8c83a0b 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper;
@@ -23,5 +18,4 @@ public class Config {
public String dlogUri;
public String dlogConf;
-
}
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java
index e2d9ae8dc..95414ef84 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper;
+
import dlshade.com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.openmessaging.benchmark.driver.BenchmarkConsumer;
import io.openmessaging.benchmark.driver.ConsumerCallback;
@@ -49,64 +45,74 @@ private static boolean backoff(long backoffTime, TimeUnit timeUnit) {
return true;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- log.warn("Interrupted at backoff {} ms",
- timeUnit.toMillis(backoffTime), e);
+ log.warn("Interrupted at backoff {} ms", timeUnit.toMillis(backoffTime), e);
return false;
}
}
- public DlogBenchmarkConsumer(DistributedLogManager dlm,
- ConsumerCallback callback) {
+ @SuppressWarnings("checkstyle:LineLength")
+ public DlogBenchmarkConsumer(DistributedLogManager dlm, ConsumerCallback callback) {
this.dlm = dlm;
- this.executor = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("dlog-benchmark-reader-thread-%d").build());
+ this.executor =
+ Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("dlog-benchmark-reader-thread-%d").build());
- this.readerTask = executor.submit(() -> {
- LogReader reader = null;
- DLSN lastDLSN = DLSN.InitialDLSN;
- LogRecordWithDLSN record;
+ this.readerTask =
+ executor.submit(
+ () -> {
+ LogReader reader = null;
+ DLSN lastDLSN = DLSN.InitialDLSN;
+ LogRecordWithDLSN record;
- while (!closing) {
- if (null == reader) {
- try {
- reader = dlm.openLogReader(lastDLSN);
- log.info("Successfully open log reader for stream {} at {}",
- dlm.getStreamName(), lastDLSN);
- } catch (IOException e) {
- log.error("Failed to open reader of stream {} at {}",
- dlm.getStreamName(), lastDLSN, e);
- if (backoff(10, TimeUnit.SECONDS)) {
- continue;
- } else {
- break;
- }
- }
- }
+ while (!closing) {
+ if (null == reader) {
+ try {
+ reader = dlm.openLogReader(lastDLSN);
+ log.info(
+ "Successfully open log reader for stream {} at {}",
+ dlm.getStreamName(),
+ lastDLSN);
+ } catch (IOException e) {
+ log.error(
+ "Failed to open reader of stream {} at {}",
+ dlm.getStreamName(),
+ lastDLSN,
+ e);
+ if (backoff(10, TimeUnit.SECONDS)) {
+ continue;
+ } else {
+ break;
+ }
+ }
+ }
- try {
- record = reader.readNext(false);
- if (null == record) {
- try {
- Thread.sleep(1);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- continue;
- }
+ try {
+ record = reader.readNext(false);
+ if (null == record) {
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ continue;
+ }
- callback.messageReceived(record.getPayload(), record.getTransactionId());
+ callback.messageReceived(record.getPayload(), record.getTransactionId());
- lastDLSN = record.getDlsn();
- } catch (IOException e) {
- log.info("Encountered error on reading records from reading stream {}, last record = {}",
- dlm.getStreamName(), lastDLSN, e);
- Utils.closeQuietly(reader);
- reader = null;
- }
- }
+ lastDLSN = record.getDlsn();
+ } catch (IOException e) {
+ log.info(
+ "Encountered error on reading records from reading stream {}, last record = {}",
+ dlm.getStreamName(),
+ lastDLSN,
+ e);
+ Utils.closeQuietly(reader);
+ reader = null;
+ }
+ }
- Utils.closeQuietly(reader);
- });
+ Utils.closeQuietly(reader);
+ });
}
@Override
@@ -118,5 +124,4 @@ public void close() throws Exception {
dlm.close();
}
}
-
}
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java
index 2734eca62..fd60a4cdf 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper;
+
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
@@ -26,7 +22,6 @@
import io.openmessaging.benchmark.driver.BenchmarkDriver;
import io.openmessaging.benchmark.driver.BenchmarkProducer;
import io.openmessaging.benchmark.driver.ConsumerCallback;
-
import io.openmessaging.benchmark.driver.bookkeeper.stats.StatsLoggerAdaptor;
import java.io.File;
import java.io.IOException;
@@ -43,14 +38,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * Benchmark driver testing distributedlog.
- */
+/** Benchmark driver testing distributedlog. */
public class DlogBenchmarkDriver implements BenchmarkDriver {
private static final Logger log = LoggerFactory.getLogger(DlogBenchmarkProducer.class);
- private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory())
- .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ private static final ObjectMapper mapper =
+ new ObjectMapper(new YAMLFactory())
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
private Config config;
private Namespace namespace;
@@ -71,13 +65,10 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I
URI dlogUri = URI.create(config.dlogUri);
dlshade.org.apache.bookkeeper.stats.StatsLogger dlStatsLogger =
- new CachingStatsLogger(new StatsLoggerAdaptor(statsLogger.scope("dlog")));
+ new CachingStatsLogger(new StatsLoggerAdaptor(statsLogger.scope("dlog")));
- namespace = NamespaceBuilder.newBuilder()
- .conf(conf)
- .uri(dlogUri)
- .statsLogger(dlStatsLogger)
- .build();
+ namespace =
+ NamespaceBuilder.newBuilder().conf(conf).uri(dlogUri).statsLogger(dlStatsLogger).build();
log.info("Initialized distributedlog namespace at {}", dlogUri);
}
@@ -93,53 +84,53 @@ private static String getFullyQualifiedPartitionedStreamName(String topic, int p
@Override
public CompletableFuture createTopic(String topic, int partitions) {
- return CompletableFuture.runAsync(() -> {
- try {
- namespace.createLog(topic);
- if (partitions > 1) {
- for (int i = 0; i < partitions; i++) {
- namespace.createLog(getFullyQualifiedPartitionedStreamName(topic, i));
+ return CompletableFuture.runAsync(
+ () -> {
+ try {
+ namespace.createLog(topic);
+ if (partitions > 1) {
+ for (int i = 0; i < partitions; i++) {
+ namespace.createLog(getFullyQualifiedPartitionedStreamName(topic, i));
+ }
+ }
+ log.info("Successfully create topic {} with {} partitions", topic, partitions);
+ } catch (IOException ioe) {
+ log.error("Failed to create topic {} with {} partitions", topic, partitions, ioe);
+ throw new RuntimeException(ioe);
}
- }
- log.info("Successfully create topic {} with {} partitions", topic, partitions);
- } catch (IOException ioe) {
- log.error("Failed to create topic {} with {} partitions",
- topic, partitions, ioe);
- throw new RuntimeException(ioe);
- }
- });
+ });
}
@Override
public CompletableFuture createProducer(String topic) {
- return CompletableFuture.supplyAsync(() -> {
- try {
- DistributedLogManager dlm = namespace.openLog(topic);
- log.info("Open stream {} for producer", topic);
- return dlm;
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- })
- .thenCompose(dlm -> dlm.openAsyncLogWriter())
- .thenApply(writer -> new DlogBenchmarkProducer(writer));
+ return CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ DistributedLogManager dlm = namespace.openLog(topic);
+ log.info("Open stream {} for producer", topic);
+ return dlm;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ })
+ .thenCompose(dlm -> dlm.openAsyncLogWriter())
+ .thenApply(writer -> new DlogBenchmarkProducer(writer));
}
@Override
public CompletableFuture createConsumer(
- String topic,
- String subscriptionName,
- ConsumerCallback consumerCallback) {
- return CompletableFuture.supplyAsync(() -> {
- try {
- DistributedLogManager dlm = namespace.openLog(topic);
- log.info("Open stream {} for consumer", topic);
- return dlm;
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- })
- .thenApply(dlm -> new DlogBenchmarkConsumer(dlm, consumerCallback));
+ String topic, String subscriptionName, ConsumerCallback consumerCallback) {
+ return CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ DistributedLogManager dlm = namespace.openLog(topic);
+ log.info("Open stream {} for consumer", topic);
+ return dlm;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ })
+ .thenApply(dlm -> new DlogBenchmarkConsumer(dlm, consumerCallback));
}
@Override
@@ -152,5 +143,4 @@ public void close() throws Exception {
log.info("BookKeeper benchmark driver successfully shut down");
}
-
}
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java
index 1bf981043..43f7399a7 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java
@@ -1,32 +1,26 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper;
+
+import io.openmessaging.benchmark.driver.BenchmarkProducer;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
-
import org.apache.distributedlog.LogRecord;
import org.apache.distributedlog.api.AsyncLogWriter;
import org.apache.distributedlog.util.TimeSequencer;
-import io.openmessaging.benchmark.driver.BenchmarkProducer;
-
public class DlogBenchmarkProducer implements BenchmarkProducer {
private final AsyncLogWriter writer;
@@ -44,10 +38,8 @@ public void close() throws Exception {
@Override
public CompletableFuture sendAsync(Optional key, byte[] payload) {
- LogRecord record = new LogRecord(
- sequencer.nextId(), payload);
+ LogRecord record = new LogRecord(sequencer.nextId(), payload);
return writer.write(record).thenApply(dlsn -> null);
}
-
}
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java
index 3255eb5e8..95d908774 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper.stats;
+
import org.apache.bookkeeper.stats.Counter;
class CounterAdaptor implements dlshade.org.apache.bookkeeper.stats.Counter {
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java
index fe8e33105..a675a7484 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper.stats;
+
import org.apache.bookkeeper.stats.Gauge;
class GaugeAdaptor implements Gauge {
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java
index 112ff7ca6..8f3d16680 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper.stats;
+
import dlshade.org.apache.bookkeeper.stats.OpStatsData;
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.stats.OpStatsLogger;
diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java
index ad36231f4..7f59c2db7 100644
--- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java
+++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java
@@ -1,23 +1,19 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.bookkeeper.stats;
+
import dlshade.com.google.common.collect.Maps;
import java.util.concurrent.ConcurrentMap;
import org.apache.bookkeeper.stats.Gauge;
@@ -27,7 +23,8 @@ public class StatsLoggerAdaptor implements dlshade.org.apache.bookkeeper.stats.S
private final StatsLogger statsLogger;
private final ConcurrentMap gauges;
- private final ConcurrentMap statsLoggers;
+ private final ConcurrentMap
+ statsLoggers;
public StatsLoggerAdaptor(StatsLogger statsLogger) {
this.statsLogger = statsLogger;
@@ -46,14 +43,16 @@ public dlshade.org.apache.bookkeeper.stats.Counter getCounter(String name) {
}
@Override
- public void registerGauge(String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) {
+ public void registerGauge(
+ String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) {
Gauge gaugeAdaptor = new GaugeAdaptor<>(gauge);
statsLogger.registerGauge(name, gaugeAdaptor);
gauges.put(gauge, gaugeAdaptor);
}
@Override
- public void unregisterGauge(String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) {
+ public void unregisterGauge(
+ String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) {
Gauge gaugeAdaptor = gauges.remove(gauge);
if (null != gaugeAdaptor) {
statsLogger.unregisterGauge(name, gaugeAdaptor);
@@ -69,7 +68,8 @@ public dlshade.org.apache.bookkeeper.stats.StatsLogger scope(String name) {
}
@Override
- public void removeScope(String name, dlshade.org.apache.bookkeeper.stats.StatsLogger dlShadeStatsLogger) {
+ public void removeScope(
+ String name, dlshade.org.apache.bookkeeper.stats.StatsLogger dlShadeStatsLogger) {
StatsLogger scopedStatsLogger = statsLoggers.remove(dlShadeStatsLogger);
if (null != scopedStatsLogger) {
statsLogger.removeScope(name, scopedStatsLogger);
diff --git a/driver-jms/README.md b/driver-jms/README.md
index 360c6ebba..3792d8bc3 100644
--- a/driver-jms/README.md
+++ b/driver-jms/README.md
@@ -13,10 +13,12 @@ Rather than simply dropping a JMS Client Library into `/opt/benchmark/lib` the l
Follow these instructions to compile the openmessaging benchmark for Fast JMS for Apache Pulsar
- Build the openmessaging benchmark package as you would normally
+
```
mvn clean package
```
- Run the repacking script
+
```
bash driver-jms/package-pulsar.sh
```
@@ -28,6 +30,7 @@ You can now deploy to AWS from `driver-pulsar/deploy`.
Follow the [Confluent instructions][1] to create a fat jar.
- Create a directory
+
```
cd ~
mkdir kafka-jms-client
@@ -36,6 +39,7 @@ Follow the [Confluent instructions][1] to create a fat jar.
- Create the pom.xml
- Change `http://packages.confluent.io/maven/` to `https://packages.confluent.io/maven/`
- Build the fat jar
+
```
mvn clean package
```
@@ -43,10 +47,12 @@ Follow the [Confluent instructions][1] to create a fat jar.
Follow these instructions to compile the openmessaging benchmark for Confluent JMS Client
- Build the openmessaging benchmark package
+
```
mvn clean package
```
- Run the repacking script passing in the location of the fat jar. EG. `~/kafka-jms-client/target/kafka-jms-client-fat-6.2.1.jar`
+
```
bash driver-jms/package-kafka.sh /path/to/the/kafka-jms-client.jar
```
@@ -60,6 +66,5 @@ For Pulsar JMS (and likely Kafka) you will likely want to allocate additional co
- Edit your `terraform.tfvars` file to adjust `num_instances["client"]`.
- Run `bin/benchmark` with the `--extra` option to allocate more workers as consumers.
+[1]: https://docs.confluent.io/platform/current/clients/kafka-jms-client/installation.html#appendix-1
-
-[1]: https://docs.confluent.io/platform/current/clients/kafka-jms-client/installation.html#appendix-1
\ No newline at end of file
diff --git a/driver-jms/artemis-jms-transactions.yaml b/driver-jms/artemis-jms-transactions.yaml
index 2035ee38b..099ff7518 100644
--- a/driver-jms/artemis-jms-transactions.yaml
+++ b/driver-jms/artemis-jms-transactions.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: JMS
diff --git a/driver-jms/artemis-jms.yaml b/driver-jms/artemis-jms.yaml
index b229ef5b1..3e2ca8122 100644
--- a/driver-jms/artemis-jms.yaml
+++ b/driver-jms/artemis-jms.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: JMS
diff --git a/driver-jms/kafka-jms.yaml b/driver-jms/kafka-jms.yaml
index c1ddce32a..d9bebc1a2 100644
--- a/driver-jms/kafka-jms.yaml
+++ b/driver-jms/kafka-jms.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: JMS
diff --git a/driver-jms/package-kafka.sh b/driver-jms/package-kafka.sh
index 5b1d18d44..925733f5d 100755
--- a/driver-jms/package-kafka.sh
+++ b/driver-jms/package-kafka.sh
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# the one argument is the path to the confluent jms client fat jar on your local system
diff --git a/driver-jms/package-pulsar.sh b/driver-jms/package-pulsar.sh
index 7c3195c01..cfd357d74 100755
--- a/driver-jms/package-pulsar.sh
+++ b/driver-jms/package-pulsar.sh
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
cd package/target
diff --git a/driver-jms/pom.xml b/driver-jms/pom.xml
index 7789a27dc..12101a80f 100644
--- a/driver-jms/pom.xml
+++ b/driver-jms/pom.xml
@@ -1,52 +1,47 @@
+
- 4.0.0
-
- io.openmessaging.benchmark
- messaging-benchmark
- 0.0.1-SNAPSHOT
- ..
-
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+
+ io.openmessaging.benchmark
+ messaging-benchmark
+ 0.0.1-SNAPSHOT
+
- driver-jms
-
- 2.0.3
-
+ driver-jms
+
+ 2.0.3
+
-
-
- ${project.groupId}
- driver-api
- ${project.version}
-
-
- jakarta.jms
- jakarta.jms-api
- ${jms.version}
-
-
- com.google.guava
- guava
-
-
+
+
+ ${project.groupId}
+ driver-api
+ ${project.version}
+
+
+ com.google.guava
+ guava
+
+
+ jakarta.jms
+ jakarta.jms-api
+ ${jms.version}
+
+
diff --git a/driver-jms/pulsar-jms-selectors.yaml b/driver-jms/pulsar-jms-selectors.yaml
index a37059df6..5dbc1ee58 100644
--- a/driver-jms/pulsar-jms-selectors.yaml
+++ b/driver-jms/pulsar-jms-selectors.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: JMS
diff --git a/driver-jms/pulsar-jms-transactions.yaml b/driver-jms/pulsar-jms-transactions.yaml
index aa0f808fa..bc4fcda41 100644
--- a/driver-jms/pulsar-jms-transactions.yaml
+++ b/driver-jms/pulsar-jms-transactions.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: JMS
diff --git a/driver-jms/pulsar-jms.yaml b/driver-jms/pulsar-jms.yaml
index bf659d2da..6e42fbe55 100644
--- a/driver-jms/pulsar-jms.yaml
+++ b/driver-jms/pulsar-jms.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
name: JMS
diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java
index 06bd2844a..e88fe86dc 100644
--- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java
+++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java
@@ -1,37 +1,29 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.jms;
+
+import io.openmessaging.benchmark.driver.BenchmarkConsumer;
+import io.openmessaging.benchmark.driver.ConsumerCallback;
import javax.jms.BytesMessage;
import javax.jms.Connection;
-import javax.jms.JMSConsumer;
-import javax.jms.JMSContext;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.Session;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import io.openmessaging.benchmark.driver.BenchmarkConsumer;
-import io.openmessaging.benchmark.driver.ConsumerCallback;
-
public class JMSBenchmarkConsumer implements BenchmarkConsumer {
private final Connection connection;
@@ -39,33 +31,39 @@ public class JMSBenchmarkConsumer implements BenchmarkConsumer {
private final MessageConsumer consumer;
private final boolean useGetBody;
- public JMSBenchmarkConsumer(Connection connection,
+ public JMSBenchmarkConsumer(
+ Connection connection,
Session session,
- MessageConsumer consumer, ConsumerCallback callback,
- boolean useGetBody) throws Exception {
+ MessageConsumer consumer,
+ ConsumerCallback callback,
+ boolean useGetBody)
+ throws Exception {
this.connection = connection;
this.consumer = consumer;
this.session = session;
this.useGetBody = useGetBody;
- consumer.setMessageListener(message -> {
- try {
- byte[] payload = getPayload(message);
- callback.messageReceived(payload, message.getLongProperty("E2EStartMillis"));
- message.acknowledge();
- } catch (Throwable e) {
- log.warn("Failed to acknowledge message", e);
- }
- });
+ consumer.setMessageListener(
+ message -> {
+ try {
+ byte[] payload = getPayload(message);
+ callback.messageReceived(payload, message.getLongProperty("E2EStartMillis"));
+ message.acknowledge();
+ } catch (Throwable e) {
+ log.warn("Failed to acknowledge message", e);
+ }
+ });
// Kafka JMS client does not allow you to add a listener after the connection has been started
connection.start();
}
@Override
public void close() throws Exception {
- // This exception may be thrown: java.util.concurrent.ExecutionException: java.util.ConcurrentModificationException: KafkaConsumer is not safe for multi-threaded access
- // See https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/session#close--
- // and https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/connection#close--
- // It should be enough to just close the connection.
+ // This exception may be thrown: java.util.concurrent.ExecutionException:
+ // java.util.ConcurrentModificationException: KafkaConsumer is not safe for multi-threaded
+ // access
+ // See https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/session#close--
+ // and https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/connection#close--
+ // It should be enough to just close the connection.
connection.close();
}
diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java
index c7cd77b03..221c30bc3 100644
--- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java
+++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java
@@ -1,24 +1,28 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.jms;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import io.openmessaging.benchmark.driver.BenchmarkConsumer;
+import io.openmessaging.benchmark.driver.BenchmarkDriver;
+import io.openmessaging.benchmark.driver.BenchmarkProducer;
+import io.openmessaging.benchmark.driver.ConsumerCallback;
+import io.openmessaging.benchmark.driver.jms.config.JMSConfig;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
@@ -27,28 +31,16 @@
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.CompletableFuture;
-
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.MessageConsumer;
import javax.jms.Session;
import javax.jms.Topic;
-
import org.apache.bookkeeper.stats.StatsLogger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import io.openmessaging.benchmark.driver.BenchmarkConsumer;
-import io.openmessaging.benchmark.driver.BenchmarkDriver;
-import io.openmessaging.benchmark.driver.BenchmarkProducer;
-import io.openmessaging.benchmark.driver.ConsumerCallback;
-import io.openmessaging.benchmark.driver.jms.config.JMSConfig;
-
public class JMSBenchmarkDriver implements BenchmarkDriver {
private ConnectionFactory connectionFactory;
@@ -61,35 +53,46 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I
this.config = readConfig(configurationFile);
log.info("JMS driver configuration: {}", writer.writeValueAsString(config));
- if (config.delegateForAdminOperationsClassName != null && !config.delegateForAdminOperationsClassName.isEmpty()) {
- log.info("Initializing Driver for Admin operations {}", config.delegateForAdminOperationsClassName);
- try
- {
- delegateForAdminOperations = (BenchmarkDriver) Class.forName(config.delegateForAdminOperationsClassName,
- true, JMSBenchmarkDriver.class.getClassLoader())
- .getConstructor().newInstance();
+ if (config.delegateForAdminOperationsClassName != null
+ && !config.delegateForAdminOperationsClassName.isEmpty()) {
+ log.info(
+ "Initializing Driver for Admin operations {}",
+ config.delegateForAdminOperationsClassName);
+ try {
+ delegateForAdminOperations =
+ (BenchmarkDriver)
+ Class.forName(
+ config.delegateForAdminOperationsClassName,
+ true,
+ JMSBenchmarkDriver.class.getClassLoader())
+ .getConstructor()
+ .newInstance();
delegateForAdminOperations.initialize(configurationFile, statsLogger);
- }
- catch (Throwable e)
- {
- log.error("Cannot created delegate driver " + config.delegateForAdminOperationsClassName, e);
+ } catch (Throwable e) {
+ log.error(
+ "Cannot created delegate driver " + config.delegateForAdminOperationsClassName, e);
throw new IOException(e);
}
}
- try
- {
+ try {
connectionFactory = buildConnectionFactory();
connection = connectionFactory.createConnection();
connection.start();
} catch (Throwable t) {
- log.error("Cannot initialize connectionFactoryClassName = "+config.connectionFactoryClassName, t);
+ log.error(
+ "Cannot initialize connectionFactoryClassName = " + config.connectionFactoryClassName, t);
throw new IOException(t);
}
}
private ConnectionFactory buildConnectionFactory() throws Exception {
- Class clazz = (Class) Class.forName(config.connectionFactoryClassName, true, Thread.currentThread().getContextClassLoader());
+ Class clazz =
+ (Class)
+ Class.forName(
+ config.connectionFactoryClassName,
+ true,
+ Thread.currentThread().getContextClassLoader());
// constructor with a String (like DataStax Pulsar JMS)
try {
@@ -103,7 +106,8 @@ private ConnectionFactory buildConnectionFactory() throws Exception {
Constructor constructor = clazz.getConstructor(Properties.class);
Properties props = new Properties();
ObjectMapper mapper = new ObjectMapper();
- Map map = mapper.readValue(new StringReader(config.connectionFactoryConfigurationParam), Map.class);
+ Map map =
+ mapper.readValue(new StringReader(config.connectionFactoryConfigurationParam), Map.class);
props.putAll(map);
return constructor.newInstance(props);
} catch (NoSuchMethodException ignore) {
@@ -132,11 +136,14 @@ public CompletableFuture createTopic(String topic, int partitions) {
public CompletableFuture createProducer(String topic) {
try {
if (config.sendWithTransactions) {
- return CompletableFuture.completedFuture(new JMSBenchmarkTransactionProducer(connection, topic, config.use20api, config.properties));
+ return CompletableFuture.completedFuture(
+ new JMSBenchmarkTransactionProducer(
+ connection, topic, config.use20api, config.properties));
} else {
Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
Destination destination = session.createTopic(topic);
- return CompletableFuture.completedFuture(new JMSBenchmarkProducer(session, destination, config.use20api, config.properties));
+ return CompletableFuture.completedFuture(
+ new JMSBenchmarkProducer(session, destination, config.use20api, config.properties));
}
} catch (Exception err) {
CompletableFuture res = new CompletableFuture<>();
@@ -146,22 +153,28 @@ public CompletableFuture createProducer(String topic) {
}
@Override
- public CompletableFuture createConsumer(String topic, String subscriptionName,
- ConsumerCallback consumerCallback) {
+ public CompletableFuture createConsumer(
+ String topic, String subscriptionName, ConsumerCallback consumerCallback) {
try {
- String selector = config.messageSelector != null && !config.messageSelector.isEmpty() ? config.messageSelector : null;
+ String selector =
+ config.messageSelector != null && !config.messageSelector.isEmpty()
+ ? config.messageSelector
+ : null;
Connection connection = connectionFactory.createConnection();
Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
Topic destination = session.createTopic(topic);
MessageConsumer durableConsumer;
if (config.use20api) {
- durableConsumer = session.createSharedDurableConsumer(destination, subscriptionName, selector);
+ durableConsumer =
+ session.createSharedDurableConsumer(destination, subscriptionName, selector);
} else {
// in JMS 1.0 we should use session.createDurableSubscriber()
// but it is not supported in Confluent Kafka JMS client
durableConsumer = session.createConsumer(destination, selector);
}
- return CompletableFuture.completedFuture(new JMSBenchmarkConsumer(connection, session, durableConsumer, consumerCallback, config.use20api));
+ return CompletableFuture.completedFuture(
+ new JMSBenchmarkConsumer(
+ connection, session, durableConsumer, consumerCallback, config.use20api));
} catch (Exception err) {
CompletableFuture res = new CompletableFuture<>();
res.completeExceptionally(err);
@@ -184,7 +197,8 @@ public void close() throws Exception {
}
}
- private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory())
+ private static final ObjectMapper mapper =
+ new ObjectMapper(new YAMLFactory())
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
private static JMSConfig readConfig(File configurationFile) throws IOException {
diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java
index fb651a621..b6ae05a69 100644
--- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java
+++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java
@@ -1,44 +1,35 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.jms;
+
+import io.openmessaging.benchmark.driver.BenchmarkProducer;
+import io.openmessaging.benchmark.driver.jms.config.JMSConfig;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
-
import javax.jms.BytesMessage;
import javax.jms.CompletionListener;
import javax.jms.Destination;
-import javax.jms.JMSContext;
import javax.jms.JMSException;
-import javax.jms.JMSProducer;
import javax.jms.Message;
import javax.jms.MessageProducer;
import javax.jms.Session;
-
-import io.openmessaging.benchmark.driver.jms.config.JMSConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import io.openmessaging.benchmark.driver.BenchmarkProducer;
-
public class JMSBenchmarkProducer implements BenchmarkProducer {
private final Session session;
@@ -46,7 +37,13 @@ public class JMSBenchmarkProducer implements BenchmarkProducer {
private final MessageProducer producer;
private final boolean useAsyncSend;
private final List properties;
- public JMSBenchmarkProducer(Session session, Destination destination, boolean useAsyncSend, List properties) throws Exception {
+
+ public JMSBenchmarkProducer(
+ Session session,
+ Destination destination,
+ boolean useAsyncSend,
+ List properties)
+ throws Exception {
this.session = session;
this.destination = destination;
this.useAsyncSend = useAsyncSend;
@@ -62,36 +59,33 @@ public void close() throws Exception {
@Override
public CompletableFuture sendAsync(Optional key, byte[] payload) {
CompletableFuture res = new CompletableFuture<>();
- try
- {
+ try {
BytesMessage bytesMessage = session.createBytesMessage();
bytesMessage.writeBytes(payload);
- if (key.isPresent())
- {
+ if (key.isPresent()) {
// a behaviour similar to https://activemq.apache.org/message-groups
bytesMessage.setStringProperty("JMSXGroupID", key.get());
}
for (JMSConfig.AddProperty prop : properties) {
bytesMessage.setStringProperty(prop.name, prop.value);
}
- // Add a timer property for end to end
- bytesMessage.setLongProperty("E2EStartMillis",System.currentTimeMillis());
+ // Add a timer property for end to end
+ bytesMessage.setLongProperty("E2EStartMillis", System.currentTimeMillis());
if (useAsyncSend) {
- producer.send(bytesMessage, new CompletionListener()
- {
- @Override
- public void onCompletion(Message message)
- {
- res.complete(null);
- }
+ producer.send(
+ bytesMessage,
+ new CompletionListener() {
+ @Override
+ public void onCompletion(Message message) {
+ res.complete(null);
+ }
- @Override
- public void onException(Message message, Exception exception)
- {
- log.error("send completed with error", exception);
- res.completeExceptionally(exception);
- }
- });
+ @Override
+ public void onException(Message message, Exception exception) {
+ log.error("send completed with error", exception);
+ res.completeExceptionally(exception);
+ }
+ });
} else {
producer.send(bytesMessage);
res.complete(null);
@@ -101,5 +95,6 @@ public void onException(Message message, Exception exception)
}
return res;
}
+
private static final Logger log = LoggerFactory.getLogger(JMSBenchmarkProducer.class);
}
diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java
index 9e24277f6..01087a9cb 100644
--- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java
+++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java
@@ -1,34 +1,35 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.jms;
+
import io.openmessaging.benchmark.driver.BenchmarkProducer;
import io.openmessaging.benchmark.driver.jms.config.JMSConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.jms.*;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
+import javax.jms.BytesMessage;
+import javax.jms.CompletionListener;
+import javax.jms.Connection;
+import javax.jms.JMSException;
+import javax.jms.Message;
+import javax.jms.MessageProducer;
+import javax.jms.Session;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class JMSBenchmarkTransactionProducer implements BenchmarkProducer {
@@ -36,7 +37,13 @@ public class JMSBenchmarkTransactionProducer implements BenchmarkProducer {
private final boolean useAsyncSend;
private final Connection connection;
private final List properties;
- public JMSBenchmarkTransactionProducer(Connection connection, String destination, boolean useAsyncSend, List properties) throws Exception {
+
+ public JMSBenchmarkTransactionProducer(
+ Connection connection,
+ String destination,
+ boolean useAsyncSend,
+ List properties)
+ throws Exception {
this.destination = destination;
this.useAsyncSend = useAsyncSend;
this.connection = connection;
@@ -44,57 +51,54 @@ public JMSBenchmarkTransactionProducer(Connection connection, String destination
}
@Override
- public void close() {
- }
+ public void close() {}
@Override
public CompletableFuture sendAsync(Optional key, byte[] payload) {
- try
- {
+ try {
// start a new Session every time, we cannot share the same Session
// among the Producers because we want to have control over the commit operation
Session session = connection.createSession(true, Session.SESSION_TRANSACTED);
MessageProducer producer = session.createProducer(session.createTopic(destination));
BytesMessage bytesMessage = session.createBytesMessage();
bytesMessage.writeBytes(payload);
- if (key.isPresent())
- {
+ if (key.isPresent()) {
// a behaviour similar to https://activemq.apache.org/message-groups
bytesMessage.setStringProperty("JMSXGroupID", key.get());
}
for (JMSConfig.AddProperty prop : properties) {
bytesMessage.setStringProperty(prop.name, prop.value);
}
- // Add a timer property for end to end
- bytesMessage.setLongProperty("E2EStartMillis",System.currentTimeMillis());
+ // Add a timer property for end to end
+ bytesMessage.setLongProperty("E2EStartMillis", System.currentTimeMillis());
if (useAsyncSend) {
CompletableFuture res = new CompletableFuture<>();
- producer.send(bytesMessage, new CompletionListener()
- {
- @Override
- public void onCompletion(Message message)
- {
- res.complete(null);
- }
+ producer.send(
+ bytesMessage,
+ new CompletionListener() {
+ @Override
+ public void onCompletion(Message message) {
+ res.complete(null);
+ }
- @Override
- public void onException(Message message, Exception exception)
- {
- log.info("send completed with error", exception);
- res.completeExceptionally(exception);
- }
- });
- return res.whenCompleteAsync((msg, error) -> {
- if (error == null) {
- // you cannot close the producer and session inside the CompletionListener
- try {
- session.commit();
- } catch (JMSException err) {
- throw new CompletionException(err);
- }
- }
- ensureClosed(producer, session);;
- });
+ @Override
+ public void onException(Message message, Exception exception) {
+ log.info("send completed with error", exception);
+ res.completeExceptionally(exception);
+ }
+ });
+ return res.whenCompleteAsync(
+ (msg, error) -> {
+ if (error == null) {
+ // you cannot close the producer and session inside the CompletionListener
+ try {
+ session.commit();
+ } catch (JMSException err) {
+ throw new CompletionException(err);
+ }
+ }
+ ensureClosed(producer, session);
+ });
} else {
try {
@@ -112,10 +116,9 @@ public void onException(Message message, Exception exception)
res.completeExceptionally(err);
return res;
}
-
}
- private void ensureClosed(MessageProducer producer, Session session) {
+ private void ensureClosed(MessageProducer producer, Session session) {
try {
producer.close();
} catch (Throwable err) {
diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java
index 324873aca..3b936024c 100644
--- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java
+++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java
@@ -1,29 +1,24 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.jms.config;
+
import java.util.ArrayList;
import java.util.List;
-public class JMSConfig
-{
+public class JMSConfig {
public String connectionFactoryClassName = "";
public String connectionFactoryConfigurationParam = "";
diff --git a/driver-kafka/deploy/README.md b/driver-kafka/deploy/README.md
index 67f1fadbf..9cadfd6c9 100644
--- a/driver-kafka/deploy/README.md
+++ b/driver-kafka/deploy/README.md
@@ -9,8 +9,8 @@ Customize the instance types in the terraform.tfvars.
If you choose larger instances, they come with more drives. To include those in the benchmarks you must:
- - update the Ansible script to include them in the mount and filesystem tasks
- - update the server.properties to include them in the logs.dir config
+- update the Ansible script to include them in the mount and filesystem tasks
+- update the server.properties to include them in the logs.dir config
NOTE: When using d2 instances, the instance stores are not automatically generated. You must add them to the provision-kafka-aws.tf file.
@@ -34,4 +34,4 @@ For instructions on how to run a benchmark see the [Kafka instructions](http://o
When using 4 client VMs or less you may see lower throughput when using compression. Compression is performed by the producers and consumers only (when using defaults) and clients need to be spread across more VMs to see any throughput gains.
-Obviously, throughput may not be your primary goal when using compression.
\ No newline at end of file
+Obviously, throughput may not be your primary goal when using compression.
diff --git a/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml b/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml
index 8cd0ed1de..24816a5db 100644
--- a/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml
+++ b/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
- name: Format and mount disks for Kafka hosts
@@ -59,7 +54,7 @@
- set_fact:
zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}"
boostrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}"
- kafkaVersion: "2.0.0"
+ kafkaVersion: "3.6.1"
- debug:
msg: "zookeeper servers: {{ zookeeperServers }}\nboostrap servers: {{ boostrapServers }}"
- name: Download Kafka package
diff --git a/driver-kafka/deploy/hdd-deployment/deploy.yaml b/driver-kafka/deploy/hdd-deployment/deploy.yaml
index d5d05a06c..b9f632b77 100644
--- a/driver-kafka/deploy/hdd-deployment/deploy.yaml
+++ b/driver-kafka/deploy/hdd-deployment/deploy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
- name: Format and mount disks for Kafka hosts
@@ -85,7 +80,7 @@
- set_fact:
zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}"
boostrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}"
- kafkaVersion: "2.8.1"
+ kafkaVersion: "3.6.1"
- debug:
msg: "zookeeper servers: {{ zookeeperServers }}\nboostrap servers: {{ boostrapServers }}"
- name: Download Kafka package
diff --git a/driver-kafka/deploy/hdd-deployment/templates/chrony.conf b/driver-kafka/deploy/hdd-deployment/templates/chrony.conf
index 0e68c66a5..2ec5485aa 100644
--- a/driver-kafka/deploy/hdd-deployment/templates/chrony.conf
+++ b/driver-kafka/deploy/hdd-deployment/templates/chrony.conf
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# Use public servers from the pool.ntp.org project.
diff --git a/driver-kafka/deploy/hdd-deployment/templates/kafka.service b/driver-kafka/deploy/hdd-deployment/templates/kafka.service
index c5afac56a..d35ac4b8b 100644
--- a/driver-kafka/deploy/hdd-deployment/templates/kafka.service
+++ b/driver-kafka/deploy/hdd-deployment/templates/kafka.service
@@ -5,7 +5,7 @@ After=network.target
[Service]
ExecStart=/opt/kafka/bin/kafka-server-start.sh config/server.properties
Environment='KAFKA_HEAP_OPTS=-Xms6g -Xmx6g -XX:MetaspaceSize=96m'
-Environment='KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -Djava.awt.headless=true"
+Environment='KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -Djava.awt.headless=true'
WorkingDirectory=/opt/kafka
RestartSec=1s
Restart=on-failure
diff --git a/driver-kafka/deploy/hdd-deployment/templates/server.properties b/driver-kafka/deploy/hdd-deployment/templates/server.properties
index 0d565707f..9ad5c2369 100644
--- a/driver-kafka/deploy/hdd-deployment/templates/server.properties
+++ b/driver-kafka/deploy/hdd-deployment/templates/server.properties
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
diff --git a/driver-kafka/deploy/hdd-deployment/templates/workers.yaml b/driver-kafka/deploy/hdd-deployment/templates/workers.yaml
index c9ef4e456..1415c4807 100644
--- a/driver-kafka/deploy/hdd-deployment/templates/workers.yaml
+++ b/driver-kafka/deploy/hdd-deployment/templates/workers.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
diff --git a/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties b/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties
index b33cad819..dbd337067 100644
--- a/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties
+++ b/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# The number of milliseconds of each tick
diff --git a/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml b/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml
index 8cd0ed1de..24816a5db 100644
--- a/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml
+++ b/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
- name: Format and mount disks for Kafka hosts
@@ -59,7 +54,7 @@
- set_fact:
zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}"
boostrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}"
- kafkaVersion: "2.0.0"
+ kafkaVersion: "3.6.1"
- debug:
msg: "zookeeper servers: {{ zookeeperServers }}\nboostrap servers: {{ boostrapServers }}"
- name: Download Kafka package
diff --git a/driver-kafka/deploy/ssd-deployment/ansible.cfg b/driver-kafka/deploy/ssd-deployment/ansible.cfg
index d1fac48cd..066d4f2e4 100644
--- a/driver-kafka/deploy/ssd-deployment/ansible.cfg
+++ b/driver-kafka/deploy/ssd-deployment/ansible.cfg
@@ -5,4 +5,8 @@ private_key_file=~/.ssh/kafka_aws
[privilege_escalation]
become=true
become_method='sudo'
-become_user='root'
\ No newline at end of file
+become_user='root'
+
+[ssh_connection]
+ssh_args=-o ServerAliveInterval=60
+retries=10
diff --git a/driver-kafka/deploy/ssd-deployment/deploy.yaml b/driver-kafka/deploy/ssd-deployment/deploy.yaml
index eaec0d276..d02ce5dda 100644
--- a/driver-kafka/deploy/ssd-deployment/deploy.yaml
+++ b/driver-kafka/deploy/ssd-deployment/deploy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
- name: Format and mount disks for Kafka hosts
@@ -67,6 +62,29 @@
- name: Reboot the machine with all defaults
reboot:
+- name: Format and mount disks for Zookeeper hosts
+ hosts: zookeeper
+ connection: ssh
+ become: true
+ tasks:
+ - name: Format disks
+ filesystem:
+ fstype: xfs
+ dev: '{{ item }}'
+ with_items:
+ - '/dev/nvme1n1'
+ - '/dev/nvme2n1'
+ - name: Mount disks
+ mount:
+ path: "{{ item.path }}"
+ src: "{{ item.src }}"
+ fstype: xfs
+ opts: defaults,noatime,nodiscard
+ state: mounted
+ with_items:
+ - { path: "/mnt/zookeeper/logs", src: "/dev/nvme1n1" }
+ - { path: "/mnt/zookeeper/data", src: "/dev/nvme2n1" }
+
- name: Kafka setup
hosts: all
connection: ssh
@@ -84,10 +102,11 @@
- file: path=/opt/kafka state=directory
- set_fact:
zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}"
- boostrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}"
- kafkaVersion: "3.0.0"
+ bootstrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}"
+ kafkaVersion: "3.6.1"
+ tags: [client-code]
- debug:
- msg: "zookeeper servers: {{ zookeeperServers }}\nboostrap servers: {{ boostrapServers }}"
+ msg: "zookeeper servers: {{ zookeeperServers }}\nbootstrap servers: {{ bootstrapServers }}"
- name: Download Kafka package
unarchive:
src: "http://archive.apache.org/dist/kafka/{{ kafkaVersion }}/kafka_2.13-{{ kafkaVersion }}.tgz"
@@ -108,15 +127,12 @@
tasks:
- set_fact:
zid: "{{ groups['zookeeper'].index(inventory_hostname) }}"
- - file:
- path: "/opt/kafka/data/zookeeper"
- state: directory
- template:
src: "templates/zookeeper.properties"
dest: "/opt/kafka/config/zookeeper.properties"
- template:
src: templates/myid
- dest: "/opt/kafka/data/zookeeper/myid"
+ dest: "/mnt/zookeeper/data/myid"
- template:
src: "templates/zookeeper.service"
dest: "/etc/systemd/system/zookeeper.service"
@@ -143,6 +159,7 @@
state: restarted
daemon_reload: yes
name: "kafka"
+ tags: [restart-kafka]
- name: Chrony setup
hosts: client
@@ -164,23 +181,28 @@
become: true
tasks:
- file: path=/opt/benchmark state=absent
+ tags: [client-code]
- name: Copy benchmark code
unarchive:
src: ../../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz
dest: /opt
+ tags: [client-code]
- shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark
+ tags: [client-code]
- shell: tuned-adm profile latency-performance
- name: Get list of driver config files
raw: ls -1 /opt/benchmark/driver-kafka/*.yaml
register: drivers_list
+ tags: [client-code]
- name: Configure Bootstrap Servers
lineinfile:
dest: '{{ item }}'
regexp: '^ bootstrap.servers='
- line: ' bootstrap.servers={{ boostrapServers }}'
+ line: ' bootstrap.servers={{ bootstrapServers }}'
with_items: '{{ drivers_list.stdout_lines }}'
+ tags: [client-code]
- name: Get list of jms driver config files
raw: ls -1 /opt/benchmark/driver-jms/kafka*.yaml
@@ -190,14 +212,14 @@
lineinfile:
dest: '{{ item }}'
regexp: '^ bootstrap.servers='
- line: ' bootstrap.servers={{ boostrapServers }}'
+ line: ' bootstrap.servers={{ bootstrapServers }}'
with_items: '{{ jms_drivers_list.stdout_lines }}'
- name: Configure JMS Connection Factory
ansible.builtin.replace:
dest: '{{ item }}'
regexp: 'localhost\:9092'
- replace: '{{ boostrapServers }}'
+ replace: '{{ bootstrapServers }}'
with_items: '{{ jms_drivers_list.stdout_lines }}'
- name: Configure memory
@@ -205,22 +227,27 @@
dest: /opt/benchmark/bin/benchmark-worker
regexp: '^JVM_MEM='
line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"'
+ tags: [client-code]
- name: Configure memory
lineinfile:
dest: /opt/benchmark/bin/benchmark
regexp: '^JVM_MEM='
line: 'JVM_MEM="-Xmx4G"'
+ tags: [client-code]
- template:
src: "templates/workers.yaml"
dest: "/opt/benchmark/workers.yaml"
+ tags: [client-code]
- name: Install benchmark systemd service
template:
src: "templates/benchmark-worker.service"
dest: "/etc/systemd/system/benchmark-worker.service"
+ tags: [client-code]
- systemd:
state: restarted
daemon_reload: yes
name: "benchmark-worker"
+ tags: [client-code]
- name: Hosts addresses
hosts: localhost
diff --git a/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf b/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf
index e1f99472f..172c01ee8 100644
--- a/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf
+++ b/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf
@@ -45,7 +45,7 @@ resource "aws_vpc" "benchmark_vpc" {
cidr_block = "10.0.0.0/16"
tags = {
- Name = "Kafka-Benchmark-VPC-${random_id.hash.hex}"
+ Name = "Kafka_Benchmark_VPC_${random_id.hash.hex}"
}
}
@@ -116,7 +116,8 @@ resource "aws_instance" "zookeeper" {
count = "${var.num_instances["zookeeper"]}"
tags = {
- Name = "zk-${count.index}"
+ Name = "zk_${count.index}"
+ Benchmark = "Kafka"
}
}
@@ -129,7 +130,8 @@ resource "aws_instance" "kafka" {
count = "${var.num_instances["kafka"]}"
tags = {
- Name = "kafka-${count.index}"
+ Name = "kafka_${count.index}"
+ Benchmark = "Kafka"
}
}
@@ -142,10 +144,15 @@ resource "aws_instance" "client" {
count = "${var.num_instances["client"]}"
tags = {
- Name = "kafka-client-${count.index}"
+ Name = "kafka_client_${count.index}"
+ Benchmark = "Kafka"
}
}
+output "kafka_ssh_host" {
+ value = "${aws_instance.kafka.0.public_ip}"
+}
+
output "client_ssh_host" {
value = "${aws_instance.client.0.public_ip}"
}
diff --git a/driver-kafka/deploy/ssd-deployment/templates/chrony.conf b/driver-kafka/deploy/ssd-deployment/templates/chrony.conf
index 0e68c66a5..2ec5485aa 100644
--- a/driver-kafka/deploy/ssd-deployment/templates/chrony.conf
+++ b/driver-kafka/deploy/ssd-deployment/templates/chrony.conf
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# Use public servers from the pool.ntp.org project.
diff --git a/driver-kafka/deploy/ssd-deployment/templates/kafka.service b/driver-kafka/deploy/ssd-deployment/templates/kafka.service
index 39c04c1a8..9d4a975ef 100644
--- a/driver-kafka/deploy/ssd-deployment/templates/kafka.service
+++ b/driver-kafka/deploy/ssd-deployment/templates/kafka.service
@@ -5,7 +5,7 @@ After=network.target
[Service]
ExecStart=/opt/kafka/bin/kafka-server-start.sh config/server.properties
Environment='KAFKA_HEAP_OPTS=-Xms16g -Xmx16g -XX:MetaspaceSize=96m'
-Environment='KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -Djava.awt.headless=true"
+Environment='KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -Djava.awt.headless=true'
WorkingDirectory=/opt/kafka
RestartSec=1s
Restart=on-failure
diff --git a/driver-kafka/deploy/ssd-deployment/templates/server.properties b/driver-kafka/deploy/ssd-deployment/templates/server.properties
index 0d565707f..9ad5c2369 100644
--- a/driver-kafka/deploy/ssd-deployment/templates/server.properties
+++ b/driver-kafka/deploy/ssd-deployment/templates/server.properties
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
diff --git a/driver-kafka/deploy/ssd-deployment/templates/workers.yaml b/driver-kafka/deploy/ssd-deployment/templates/workers.yaml
index c9ef4e456..1415c4807 100644
--- a/driver-kafka/deploy/ssd-deployment/templates/workers.yaml
+++ b/driver-kafka/deploy/ssd-deployment/templates/workers.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
diff --git a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties
index b33cad819..4fb919bd5 100644
--- a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties
+++ b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# The number of milliseconds of each tick
@@ -26,7 +21,8 @@ initLimit=10
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
-dataDir=data/zookeeper
+dataLogDir=/mnt/zookeeper/logs
+dataDir=/mnt/zookeeper/data
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
diff --git a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service
index 3a2706efa..cb0d13d7b 100644
--- a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service
+++ b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service
@@ -3,6 +3,7 @@ Description=ZooKeeper
After=network.target
[Service]
+Environment='KAFKA_HEAP_OPTS=-Xms32g -Xmx32g'
ExecStart=/opt/kafka/bin/zookeeper-server-start.sh config/zookeeper.properties
WorkingDirectory=/opt/kafka
RestartSec=1s
diff --git a/driver-kafka/deploy/ssd-deployment/terraform.tfvars b/driver-kafka/deploy/ssd-deployment/terraform.tfvars
index ac01d4acb..f6b6d8995 100644
--- a/driver-kafka/deploy/ssd-deployment/terraform.tfvars
+++ b/driver-kafka/deploy/ssd-deployment/terraform.tfvars
@@ -1,11 +1,11 @@
public_key_path = "~/.ssh/kafka_aws.pub"
region = "us-west-2"
az = "us-west-2a"
-ami = "ami-9fa343e7" // RHEL-7.4
+ami = "ami-08970fb2e5767e3b8" // RHEL-8
instance_types = {
"kafka" = "i3en.6xlarge"
- "zookeeper" = "t2.small"
+ "zookeeper" = "i3en.2xlarge"
"client" = "m5n.8xlarge"
}
diff --git a/driver-kafka/kafka-big-batches-gzip.yaml b/driver-kafka/kafka-big-batches-gzip.yaml
index 603145388..86e48eac4 100644
--- a/driver-kafka/kafka-big-batches-gzip.yaml
+++ b/driver-kafka/kafka-big-batches-gzip.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-big-batches-lz4.yaml b/driver-kafka/kafka-big-batches-lz4.yaml
index 0c3fcfdf6..18453137d 100644
--- a/driver-kafka/kafka-big-batches-lz4.yaml
+++ b/driver-kafka/kafka-big-batches-lz4.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-big-batches-snappy.yaml b/driver-kafka/kafka-big-batches-snappy.yaml
index f7a70161c..ffd1d0990 100644
--- a/driver-kafka/kafka-big-batches-snappy.yaml
+++ b/driver-kafka/kafka-big-batches-snappy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-big-batches-zstd.yaml b/driver-kafka/kafka-big-batches-zstd.yaml
index 162ff7225..3a15a5e26 100644
--- a/driver-kafka/kafka-big-batches-zstd.yaml
+++ b/driver-kafka/kafka-big-batches-zstd.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-big-batches.yaml b/driver-kafka/kafka-big-batches.yaml
index 5c3b99b0b..b0c8638a5 100644
--- a/driver-kafka/kafka-big-batches.yaml
+++ b/driver-kafka/kafka-big-batches.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-compression-gzip.yaml b/driver-kafka/kafka-compression-gzip.yaml
index 68dc9dba7..011a98f4a 100644
--- a/driver-kafka/kafka-compression-gzip.yaml
+++ b/driver-kafka/kafka-compression-gzip.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-compression-lz4.yaml b/driver-kafka/kafka-compression-lz4.yaml
index f756ec8c6..8fdccc982 100644
--- a/driver-kafka/kafka-compression-lz4.yaml
+++ b/driver-kafka/kafka-compression-lz4.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-compression-snappy.yaml b/driver-kafka/kafka-compression-snappy.yaml
index dcdfcfd1b..ae229768b 100644
--- a/driver-kafka/kafka-compression-snappy.yaml
+++ b/driver-kafka/kafka-compression-snappy.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-compression-zstd.yaml b/driver-kafka/kafka-compression-zstd.yaml
index 1a5a577fa..d769bbb5f 100644
--- a/driver-kafka/kafka-compression-zstd.yaml
+++ b/driver-kafka/kafka-compression-zstd.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-exactly-once.yaml b/driver-kafka/kafka-exactly-once.yaml
index e85d6ea1c..8a6ec7706 100644
--- a/driver-kafka/kafka-exactly-once.yaml
+++ b/driver-kafka/kafka-exactly-once.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-latency.yaml b/driver-kafka/kafka-latency.yaml
index 6246c0f08..ed0d5a600 100644
--- a/driver-kafka/kafka-latency.yaml
+++ b/driver-kafka/kafka-latency.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-no-linger.yaml b/driver-kafka/kafka-no-linger.yaml
index 768bd7eed..aaccf8edb 100644
--- a/driver-kafka/kafka-no-linger.yaml
+++ b/driver-kafka/kafka-no-linger.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-sync.yaml b/driver-kafka/kafka-sync.yaml
index 2fa14cd01..357b570c4 100644
--- a/driver-kafka/kafka-sync.yaml
+++ b/driver-kafka/kafka-sync.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# Kafka driver profile that is configuring to sync all
@@ -24,7 +19,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
diff --git a/driver-kafka/kafka-throughput.yaml b/driver-kafka/kafka-throughput.yaml
index 4c00b39d4..01122b46d 100644
--- a/driver-kafka/kafka-throughput.yaml
+++ b/driver-kafka/kafka-throughput.yaml
@@ -1,20 +1,15 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
@@ -23,13 +18,14 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
# Kafka client-specific configuration
replicationFactor: 3
-reset: true
topicConfig: |
min.insync.replicas=2
commonConfig: |
bootstrap.servers=localhost:9092
+ default.api.timeout.ms=1200000
+ request.timeout.ms=1200000
producerConfig: |
acks=all
diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml
index a44de0b86..83324ab5c 100644
--- a/driver-kafka/pom.xml
+++ b/driver-kafka/pom.xml
@@ -1,46 +1,65 @@
+
- 4.0.0
-
- io.openmessaging.benchmark
- messaging-benchmark
- 0.0.1-SNAPSHOT
- ..
-
- driver-kafka
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+
+ io.openmessaging.benchmark
+ messaging-benchmark
+ 0.0.1-SNAPSHOT
+
+ driver-kafka
-
-
- ${project.groupId}
- driver-api
- ${project.version}
-
-
-
- org.apache.kafka
- kafka-clients
- 2.8.1
-
-
+
+
+ ${project.groupId}
+ driver-api
+ ${project.version}
+
+
+ org.apache.kafka
+ kafka-clients
+ 3.6.1
+
+
+ org.projectlombok
+ lombok
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ test
+
+
+ org.assertj
+ assertj-core
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter
+ test
+
+
+ org.mockito
+ mockito-junit-jupiter
+ test
+
+
diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java
index b73eb0b68..d2fafafa3 100644
--- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java
+++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java
@@ -1,20 +1,15 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.kafka;
@@ -28,6 +23,4 @@ public class Config {
public String producerConfig;
public String consumerConfig;
-
- public boolean reset;
}
diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java
index 2f3dd695a..e07b417d9 100644
--- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java
+++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java
@@ -1,23 +1,21 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package io.openmessaging.benchmark.driver.kafka;
+
+import io.openmessaging.benchmark.driver.BenchmarkConsumer;
+import io.openmessaging.benchmark.driver.ConsumerCallback;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
@@ -25,16 +23,12 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
-
-import io.openmessaging.benchmark.driver.BenchmarkConsumer;
-import io.openmessaging.benchmark.driver.ConsumerCallback;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,32 +42,51 @@ public class KafkaBenchmarkConsumer implements BenchmarkConsumer {
private final Future consumerTask;
private volatile boolean closing = false;
private boolean autoCommit;
- public KafkaBenchmarkConsumer(KafkaConsumer consumer, Properties consumerConfig, ConsumerCallback callback) {
+
+ public KafkaBenchmarkConsumer(
+ KafkaConsumer consumer,
+ Properties consumerConfig,
+ ConsumerCallback callback) {
+ this(consumer, consumerConfig, callback, 100L);
+ }
+
+ public KafkaBenchmarkConsumer(
+ KafkaConsumer consumer,
+ Properties consumerConfig,
+ ConsumerCallback callback,
+ long pollTimeoutMs) {
this.consumer = consumer;
this.executor = Executors.newSingleThreadExecutor();
- this.autoCommit= Boolean.valueOf((String)consumerConfig.getOrDefault(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false"));
- this.consumerTask = this.executor.submit(() -> {
- while (!closing) {
- try {
- ConsumerRecords records = consumer.poll(Duration.ofMillis(100));
+ this.autoCommit =
+ Boolean.valueOf(
+ (String)
+ consumerConfig.getOrDefault(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"));
+ this.consumerTask =
+ this.executor.submit(
+ () -> {
+ while (!closing) {
+ try {
+ ConsumerRecords