opensource/rocketmq
3
0
Fork 0
mirror of https://github.com/apache/rocketmq.git synced 2026-08-30 18:10:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support build with Bazel (#4865)

Browse Source 
* Support build with Bazel, fixing tests to make them capable of running in concurrency and hermetic.
This commit is contained in:
Zhanhui Li
2022-08-23 13:11:53 +08:00
committed by GitHub
parent 92d10baf24
commit ec8a93d2f7
105 changed files with 1881 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/BUILD.bazel
+107
View File
@@ -0,0 +1,107 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
load("//bazel:GenTestRules.bzl", "GenTestRules")
java_library(
name = "test",
srcs = glob(["src/main/java/**/*.java"]),
visibility = ["//visibility:public"],
deps = [
"//broker",
"//client",
"//common",
"//remoting",
"//logging",
"//srvutil",
"//tools",
"//namesrv",
"//controller",
"//container",
"//proxy",
"@maven//:org_apache_commons_commons_lang3",
"@maven//:commons_validator_commons_validator",
"@maven//:com_github_luben_zstd_jni",
"@maven//:org_lz4_lz4_java",
"@maven//:com_alibaba_fastjson",
"@maven//:io_netty_netty_all",
"@maven//:log4j_log4j",
"@maven//:org_slf4j_slf4j_api",
"@maven//:ch_qos_logback_logback_classic",
"@maven//:ch_qos_logback_logback_core",
"@maven//:com_google_truth_truth",
"@maven//:javax_annotation_javax_annotation_api",
"@maven//:org_awaitility_awaitility",
"@maven//:org_reflections_reflections",
"@maven//:com_google_protobuf_protobuf_java_util",
"@maven//:commons_cli_commons_cli",
"@maven//:com_google_guava_guava",
],
)
java_library(
name = "tests",
srcs = glob(["src/test/java/**/*.java"]),
visibility = ["//visibility:public"],
deps = [
":test",
"//broker",
"//client",
"//common",
"//remoting",
"//logging",
"//tools",
"//:test_deps",
"//store",
"//namesrv",
"//controller",
"//container",
"//proxy",
"@maven//:org_apache_commons_commons_lang3",
"@maven//:io_netty_netty_all",
"@maven//:com_google_truth_truth",
"@maven//:log4j_log4j",
"@maven//:io_grpc_grpc_testing",
"@maven//:com_google_protobuf_protobuf_java_util",
"@maven//:com_google_protobuf_protobuf_java",
"@maven//:io_grpc_grpc_netty_shaded",
"@maven//:io_grpc_grpc_context",
"@maven//:io_grpc_grpc_stub",
"@maven//:io_grpc_grpc_api",
"@maven//:org_apache_rocketmq_rocketmq_proto",
"@maven//:org_slf4j_slf4j_api",
"@maven//:com_google_guava_guava",
],
resources = [
"src/test/resources/rmq-proxy-home/conf/broker.conf",
"src/test/resources/rmq-proxy-home/conf/logback_proxy.xml",
"src/test/resources/rmq-proxy-home/conf/rmq-proxy.json",
"src/test/resources/log4j.xml",
"src/test/resources/logback-test.xml",
] + glob(["src/test/resources/schema/**/*.schema"]),
)
GenTestRules(
name = "GeneratedTestRules",
default_test_size = "large",
test_files = glob(["src/test/java/**/*IT.java"]),
deps = [
":tests",
],
exclude_tests = [
"src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT",
]
)
test/src/main/java/org/apache/rocketmq/test/schema/SchemaTools.java
+1 -1
View File
@@ -177,7 +177,7 @@ public class SchemaTools {
}
});
String key = String.format("Method %s(%s)", method.getName(), Arrays.stream(parameterTypes).map(Class::getName).collect(Collectors.joining(",")));
String value = String.format("%s throws (%s)",
String value = String.format("%s throws (%s): %s",
isPublicOrPrivate(method.getModifiers()),
method.getReturnType().getName(),
Arrays.stream(exceptionTypes).map(Class::getName).collect(Collectors.joining(",")));
test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java
+1 -1
View File
@@ -265,7 +265,7 @@ public final class RandomUtil {
Random rd = new Random();
int index = 0;
for (int i = 0; i < result.length; i++) {
index = Math.abs(rd.nextInt() % len--);
index = rd.nextInt(len--);
result[i] = source[index];
source[index] = source[len];
}
test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
+6 -9
View File
@@ -56,15 +56,12 @@ public class IntegrationTestBase {
public static volatile int COMMIT_LOG_SIZE = 1024 * 1024 * 100;
protected static final int INDEX_NUM = 1000;
private static final AtomicInteger port = new AtomicInteger(40000);
public static synchronized int nextPort() {
return port.addAndGet(random.nextInt(10) + 10);
}
protected static Random random = new Random();
static {
System.setProperty("rocketmq.client.logRoot", System.getProperty("java.io.tmpdir"));
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
@@ -99,7 +96,7 @@ public class IntegrationTestBase {
}
public static String createBaseDir() {
String baseDir = System.getProperty("user.home") + SEP + "unitteststore-" + UUID.randomUUID();
String baseDir = System.getProperty("java.io.tmpdir") + SEP + "unitteststore-" + UUID.randomUUID();
final File file = new File(baseDir);
if (file.exists()) {
logger.info(String.format("[%s] has already existed, please back up and remove it for integration tests", baseDir));
@@ -116,7 +113,7 @@ public class IntegrationTestBase {
namesrvConfig.setKvConfigPath(baseDir + SEP + "namesrv" + SEP + "kvConfig.json");
namesrvConfig.setConfigStorePath(baseDir + SEP + "namesrv" + SEP + "namesrv.properties");
nameServerNettyServerConfig.setListenPort(nextPort());
nameServerNettyServerConfig.setListenPort(0);
NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);
try {
Truth.assertThat(namesrvController.initialize()).isTrue();
@@ -154,8 +151,8 @@ public class IntegrationTestBase {
public static BrokerController createAndStartBroker(MessageStoreConfig storeConfig, BrokerConfig brokerConfig) {
NettyServerConfig nettyServerConfig = new NettyServerConfig();
NettyClientConfig nettyClientConfig = new NettyClientConfig();
nettyServerConfig.setListenPort(nextPort());
storeConfig.setHaListenPort(nextPort());
nettyServerConfig.setListenPort(0);
storeConfig.setHaListenPort(0);
BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, storeConfig);
try {
Truth.assertThat(brokerController.initialize()).isTrue();
test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java
+3 -3
View File
@@ -38,7 +38,6 @@ import org.apache.rocketmq.test.factory.ProducerFactory;
import org.junit.Assert;
import org.junit.Test;
import static org.apache.rocketmq.test.base.IntegrationTestBase.nextPort;
import static sun.util.locale.BaseLocale.SEP;
public class DLedgerProduceAndConsumeIT {
@@ -57,7 +56,7 @@ public class DLedgerProduceAndConsumeIT {
String baseDir = IntegrationTestBase.createBaseDir();
storeConfig.setStorePathRootDir(baseDir);
storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog");
storeConfig.setHaListenPort(nextPort());
storeConfig.setHaListenPort(0);
storeConfig.setMappedFileSizeCommitLog(10 * 1024 * 1024);
storeConfig.setEnableDLegerCommitLog(true);
storeConfig.setdLegerGroup(brokerName);
@@ -71,7 +70,8 @@ public class DLedgerProduceAndConsumeIT {
String cluster = UUID.randomUUID().toString();
String brokerName = UUID.randomUUID().toString();
String selfId = "n0";
String peers = String.format("n0-localhost:%d", nextPort());
// TODO: We need to acquire the actual listening port after the peer has started.
String peers = String.format("n0-localhost:%d", 0);
BrokerConfig brokerConfig = buildBrokerConfig(cluster, brokerName);
MessageStoreConfig storeConfig = buildStoreConfig(brokerName, peers, selfId);
BrokerController brokerController = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig);
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/BaseBroadCastIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/BaseBroadcast.java
+6 -6
View File
@@ -23,21 +23,21 @@ import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.factory.ConsumerFactory;
import org.apache.rocketmq.test.listener.AbstractListener;
public class BaseBroadCastIT extends BaseConf {
private static Logger logger = Logger.getLogger(BaseBroadCastIT.class);
public class BaseBroadcast extends BaseConf {
private static Logger logger = Logger.getLogger(BaseBroadcast.class);
public static RMQBroadCastConsumer getBroadCastConsumer(String nsAddr, String topic,
String subExpression,
AbstractListener listner) {
AbstractListener listener) {
String consumerGroup = initConsumerGroup();
return getBroadCastConsumer(nsAddr, consumerGroup, topic, subExpression, listner);
return getBroadCastConsumer(nsAddr, consumerGroup, topic, subExpression, listener);
}
public static RMQBroadCastConsumer getBroadCastConsumer(String nsAddr, String consumerGroup,
String topic, String subExpression,
AbstractListener listner) {
AbstractListener listener) {
RMQBroadCastConsumer consumer = ConsumerFactory.getRMQBroadCastConsumer(nsAddr,
consumerGroup, topic, subExpression, listner);
consumerGroup, topic, subExpression, listener);
consumer.setDebug();
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgNotRecvIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgNotReceiveIT.java
+2 -2
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.normal;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -30,7 +30,7 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastNormalMsgNotRecvIT extends BaseBroadCastIT {
public class BroadcastNormalMsgNotReceiveIT extends BaseBroadcast {
private static Logger logger = Logger
.getLogger(NormalMsgTwoSameGroupConsumerIT.class);
private RMQNormalProducer producer = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgRecvCrashIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgRecvCrashIT.java
+2 -2
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.normal;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,7 +31,7 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastNormalMsgRecvCrashIT extends BaseBroadCastIT {
public class BroadcastNormalMsgRecvCrashIT extends BaseBroadcast {
private static Logger logger = Logger
.getLogger(NormalMsgTwoSameGroupConsumerIT.class);
private RMQNormalProducer producer = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgRecvFailIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgRecvFailIT.java
+2 -2
View File
@@ -19,7 +19,7 @@ package org.apache.rocketmq.test.client.consumer.broadcast.normal;
import org.apache.log4j.Logger;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -32,7 +32,7 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastNormalMsgRecvFailIT extends BaseBroadCastIT {
public class BroadcastNormalMsgRecvFailIT extends BaseBroadcast {
private static Logger logger = Logger
.getLogger(NormalMsgTwoSameGroupConsumerIT.class);
private RMQNormalProducer producer = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgRecvStartLaterIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgRecvStartLaterIT.java
+2 -2
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.normal;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,7 +31,7 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastNormalMsgRecvStartLaterIT extends BaseBroadCastIT {
public class BroadcastNormalMsgRecvStartLaterIT extends BaseBroadcast {
private static Logger logger = Logger
.getLogger(NormalMsgTwoSameGroupConsumerIT.class);
private RMQNormalProducer producer = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadCastNormalMsgTwoDiffGroupRecvIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgTwoDiffGroupRecvIT.java
+2 -2
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.normal;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,7 +31,7 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastNormalMsgTwoDiffGroupRecvIT extends BaseBroadCastIT {
public class BroadcastNormalMsgTwoDiffGroupRecvIT extends BaseBroadcast {
private static Logger logger = Logger
.getLogger(NormalMsgTwoSameGroupConsumerIT.class);
private RMQNormalProducer producer = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/NormalMsgTwoSameGroupConsumerIT.java
+2 -2
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.normal;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,7 +31,7 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class NormalMsgTwoSameGroupConsumerIT extends BaseBroadCastIT {
public class NormalMsgTwoSameGroupConsumerIT extends BaseBroadcast {
private static Logger logger = Logger
.getLogger(NormalMsgTwoSameGroupConsumerIT.class);
private RMQNormalProducer producer = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadCastIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java
+3 -3
View File
@@ -20,7 +20,7 @@ package org.apache.rocketmq.test.client.consumer.broadcast.order;
import java.util.List;
import org.apache.log4j.Logger;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;
@@ -38,8 +38,8 @@ import static com.google.common.truth.Truth.assertThat;
* Currently, dose not support the ordered broadcast message
*/
@Ignore
public class OrderMsgBroadCastIT extends BaseBroadCastIT {
private static Logger logger = Logger.getLogger(OrderMsgBroadCastIT.class);
public class OrderMsgBroadcastIT extends BaseBroadcast {
private static Logger logger = Logger.getLogger(OrderMsgBroadcastIT.class);
private RMQNormalProducer producer = null;
private String topic = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadCastTwoConsumerFilterIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadcastTwoConsumerFilterIT.java
+3 -3
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.tag;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,8 +31,8 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastTwoConsumerFilterIT extends BaseBroadCastIT {
private static Logger logger = Logger.getLogger(BroadCastTwoConsumerSubTagIT.class);
public class BroadcastTwoConsumerFilterIT extends BaseBroadcast {
private static Logger logger = Logger.getLogger(BroadcastTwoConsumerSubTagIT.class);
private RMQNormalProducer producer = null;
private String topic = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadCastTwoConsumerSubDiffTagIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadcastTwoConsumerSubDiffTagIT.java
+3 -3
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.tag;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,8 +31,8 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastTwoConsumerSubDiffTagIT extends BaseBroadCastIT {
private static Logger logger = Logger.getLogger(BroadCastTwoConsumerSubTagIT.class);
public class BroadcastTwoConsumerSubDiffTagIT extends BaseBroadcast {
private static Logger logger = Logger.getLogger(BroadcastTwoConsumerSubTagIT.class);
private RMQNormalProducer producer = null;
private String topic = null;
test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadCastTwoConsumerSubTagIT.java → test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadcastTwoConsumerSubTagIT.java
+3 -3
View File
@@ -18,7 +18,7 @@
package org.apache.rocketmq.test.client.consumer.broadcast.tag;
import org.apache.log4j.Logger;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadCastIT;
import org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;
import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
@@ -31,8 +31,8 @@ import org.junit.Test;
import static com.google.common.truth.Truth.assertThat;
public class BroadCastTwoConsumerSubTagIT extends BaseBroadCastIT {
private static Logger logger = Logger.getLogger(BroadCastTwoConsumerSubTagIT.class);
public class BroadcastTwoConsumerSubTagIT extends BaseBroadcast {
private static Logger logger = Logger.getLogger(BroadcastTwoConsumerSubTagIT.class);
private RMQNormalProducer producer = null;
private String topic = null;
test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
+5 -1
View File
@@ -140,7 +140,11 @@ public class GrpcBaseIT extends BaseConf {
if (mockProxyHomeURL != null) {
mockProxyHome = mockProxyHomeURL.toURI().getPath();
}
System.setProperty(RMQ_PROXY_HOME, mockProxyHome);
if (null != mockProxyHome) {
System.setProperty(RMQ_PROXY_HOME, mockProxyHome);
}
ConfigurationManager.initEnv();
ConfigurationManager.intConfig();
ConfigurationManager.getProxyConfig().setGrpcServerPort(port);