[Infra Improvement] Build And Run Unit Tests in Parallel (#4878)

* Support build with Bazel (#4865)

* Support build with Bazel, fixing tests to make them capable of running in concurrency and hermetic.

* Make test cases capable of running in parallel. (#4874)

remoting/BUILD.bazel

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+load("//bazel:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+    name = "remoting",
+    srcs = glob(["src/main/java/**/*.java"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//logging",
+        "@maven//:com_alibaba_fastjson",
+        "@maven//:io_netty_netty_all",
+    ],
+)
+
+java_library(
+    name = "tests",
+    srcs = glob(["src/test/java/**/*.java"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":remoting",
+        "//:test_deps",
+        "@maven//:io_netty_netty_all",     
+        "@maven//:com_google_code_gson_gson",   
+        "@maven//:com_alibaba_fastjson",
+    ],
+    resources = glob(["src/test/resources/certs/*.pem"]) + glob(["src/test/resources/certs/*.key"])
+)
+
+GenTestRules(
+    name = "GeneratedTestRules",
+    test_files = glob(["src/test/java/**/*Test.java"]),
+    deps = [
+        ":tests",
+    ],
+)

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java

@@ -175,8 +175,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
 
     private static int initValueIndex() {
         Random r = new Random();
-
-        return Math.abs(r.nextInt() % 999) % 999;
+        return r.nextInt(999);
     }
 
     @Override

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java

@@ -80,13 +80,11 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
     private DefaultEventExecutorGroup defaultEventExecutorGroup;
 
     /**
-     * NettyRemotingServer may holds multiple SubRemotingServer, each server will be stored in this container with a
+     * NettyRemotingServer may hold multiple SubRemotingServer, each server will be stored in this container with a
      * ListenPort key.
      */
     private ConcurrentMap<Integer/*Port*/, NettyRemotingAbstract> remotingServerTable = new ConcurrentHashMap<Integer, NettyRemotingAbstract>();
 
-    private int port = 0;
-
     private static final String HANDSHAKE_HANDLER_NAME = "handshakeHandler";
     private static final String TLS_HANDLER_NAME = "sslHandler";
     private static final String FILE_REGION_ENCODER_NAME = "fileRegionEncoder";
@@ -163,8 +161,6 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
         }
 
         loadSslContext();
-
-        this.remotingServerTable.put(this.nettyServerConfig.getListenPort(), this);
     }
 
     public void loadSslContext() {
@@ -247,9 +243,13 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
         }
 
         try {
-            ChannelFuture sync = this.serverBootstrap.bind().sync();
+            ChannelFuture sync = this.serverBootstrap.bind(nettyServerConfig.getListenPort()).sync();
             InetSocketAddress addr = (InetSocketAddress) sync.channel().localAddress();
-            this.port = addr.getPort();
+            if (0 == nettyServerConfig.getListenPort()) {
+                this.nettyServerConfig.setListenPort(addr.getPort());
+                log.debug("Server is listening {}", this.nettyServerConfig.getListenPort());
+            }
+            this.remotingServerTable.put(this.nettyServerConfig.getListenPort(), this);
         } catch (InterruptedException e1) {
             throw new RuntimeException("this.serverBootstrap.bind().sync() InterruptedException", e1);
         }
@@ -320,7 +320,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
 
     @Override
     public int localListenPort() {
-        return this.port;
+        return this.nettyServerConfig.getListenPort();
     }
 
     @Override
@@ -540,7 +540,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
      * resources from its parent server.
      */
     class SubRemotingServer extends NettyRemotingAbstract implements RemotingServer {
-        private final int listenPort;
+        private volatile int listenPort;
         private volatile Channel serverChannel;
 
         SubRemotingServer(final int port, final int permitsOnway, final int permitsAsync) {
@@ -613,7 +613,14 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
         @Override
         public void start() {
             try {
+                if (listenPort < 0) {
+                    listenPort = 0;
+                }
                 this.serverChannel = NettyRemotingServer.this.serverBootstrap.bind(listenPort).sync().channel();
+                if (0 == listenPort) {
+                    InetSocketAddress addr = (InetSocketAddress) this.serverChannel.localAddress();
+                    this.listenPort = addr.getPort();
+                }
             } catch (InterruptedException e) {
                 throw new RuntimeException("this.subRemotingServer.serverBootstrap.bind().sync() InterruptedException", e);
             }

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java

@@ -17,7 +17,7 @@
 package org.apache.rocketmq.remoting.netty;
 
 public class NettyServerConfig implements Cloneable {
-    private int listenPort = 8888;
+    private int listenPort = 0;
     private int serverWorkerThreads = 8;
     private int serverCallbackExecutorThreads = 0;
     private int serverSelectorThreads = 3;

remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java

@@ -34,6 +34,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 public class RemotingServerTest {
@@ -90,8 +91,8 @@ public class RemotingServerTest {
         requestHeader.setCount(1);
         requestHeader.setMessageTitle("Welcome");
         RemotingCommand request = RemotingCommand.createRequestCommand(0, requestHeader);
-        RemotingCommand response = remotingClient.invokeSync("localhost:8888", request, 1000 * 3);
-        assertTrue(response != null);
+        RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3);
+        assertNotNull(response);
         assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);
         assertThat(response.getExtFields()).hasSize(2);
 
@@ -103,7 +104,7 @@ public class RemotingServerTest {
 
         RemotingCommand request = RemotingCommand.createRequestCommand(0, null);
         request.setRemark("messi");
-        remotingClient.invokeOneway("localhost:8888", request, 1000 * 3);
+        remotingClient.invokeOneway("localhost:" + remotingServer.localListenPort(), request, 1000 * 3);
     }
 
     @Test
@@ -113,7 +114,7 @@ public class RemotingServerTest {
         final CountDownLatch latch = new CountDownLatch(1);
         RemotingCommand request = RemotingCommand.createRequestCommand(0, null);
         request.setRemark("messi");
-        remotingClient.invokeAsync("localhost:8888", request, 1000 * 3, new InvokeCallback() {
+        remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() {
             @Override
             public void operationComplete(ResponseFuture responseFuture) {
                 latch.countDown();

remoting/src/test/java/org/apache/rocketmq/remoting/SubRemotingServerTest.java

@@ -57,7 +57,7 @@ public class SubRemotingServerTest {
         subServer.registerProcessor(1, new NettyRequestProcessor() {
             @Override
             public RemotingCommand processRequest(final ChannelHandlerContext ctx,
-                final RemotingCommand request) throws Exception {
+                    final RemotingCommand request) throws Exception {
                 request.setRemark(String.valueOf(RemotingHelper.parseSocketAddressPort(ctx.channel().localAddress())));
                 return request;
             }
@@ -72,14 +72,16 @@ public class SubRemotingServerTest {
     }
 
     @Test
-    public void testInvokeSubRemotingServer() throws InterruptedException, RemotingTimeoutException, RemotingConnectException, RemotingSendRequestException {
+    public void testInvokeSubRemotingServer() throws InterruptedException, RemotingTimeoutException,
+            RemotingConnectException, RemotingSendRequestException {
         RequestHeader requestHeader = new RequestHeader();
         requestHeader.setCount(1);
         requestHeader.setMessageTitle("Welcome");
 
         // Parent remoting server doesn't support RequestCode 1
         RemotingCommand request = RemotingCommand.createRequestCommand(1, requestHeader);
-        RemotingCommand response = remotingClient.invokeSync("localhost:8888", request, 1000 * 3);
+        RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), request,
+                1000 * 3);
         assertThat(response).isNotNull();
         assertThat(response.getCode()).isEqualTo(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED);
 

remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java

@@ -19,9 +19,14 @@ package org.apache.rocketmq.remoting;
 
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.UUID;
+import java.io.InputStream;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import org.apache.rocketmq.remoting.common.TlsMode;
 import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
@@ -65,7 +70,7 @@ import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerTrustC
 import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsTestModeEnable;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
 
 @RunWith(MockitoJUnitRunner.class)
 public class TlsTest {
@@ -198,7 +203,7 @@ public class TlsTest {
     @Test
     public void serverRejectsSSLClient() throws Exception {
         try {
-            RemotingCommand response = remotingClient.invokeSync("localhost:8888", createRequest(), 1000 * 5);
+            RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), createRequest(), 1000 * 5);
             failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);
         } catch (RemotingSendRequestException ignore) {
         }
@@ -211,7 +216,7 @@ public class TlsTest {
     @Test
     public void serverRejectsUntrustedClientCert() throws Exception {
         try {
-            RemotingCommand response = remotingClient.invokeSync("localhost:8888", createRequest(), 1000 * 5);
+            RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), createRequest(), 1000 * 5);
             failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);
         } catch (RemotingSendRequestException ignore) {
         }
@@ -229,7 +234,7 @@ public class TlsTest {
     @Test
     public void noClientAuthFailure() throws Exception {
         try {
-            RemotingCommand response = remotingClient.invokeSync("localhost:8888", createRequest(), 1000 * 3);
+            RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), createRequest(), 1000 * 3);
             failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);
         } catch (RemotingSendRequestException ignore) {
         }
@@ -242,7 +247,7 @@ public class TlsTest {
     @Test
     public void clientRejectsUntrustedServerCert() throws Exception {
         try {
-            RemotingCommand response = remotingClient.invokeSync("localhost:8888", createRequest(), 1000 * 3);
+            RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), createRequest(), 1000 * 3);
             failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);
         } catch (RemotingSendRequestException ignore) {
         }
@@ -301,8 +306,31 @@ public class TlsTest {
     }
 
     private static String getCertsPath(String fileName) {
-        File resourcesDirectory = new File("src/test/resources/certs");
-        return resourcesDirectory.getAbsolutePath() + "/" + fileName;
+        ClassLoader loader = TlsTest.class.getClassLoader();
+        InputStream stream = loader.getResourceAsStream("certs/" + fileName);
+        if (null == stream) {
+            throw new RuntimeException("File: " + fileName + " is not found");
+        }
+
+        try {
+            String[] segments = fileName.split("\\.");
+            File f = File.createTempFile(UUID.randomUUID().toString(), segments[1]);
+            f.deleteOnExit();
+    
+            try (BufferedInputStream bis = new BufferedInputStream(stream);
+                 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f))) {
+                byte[] buffer = new byte[1024];
+                int len;
+                while ((len = bis.read(buffer)) > 0) {
+                    bos.write(buffer, 0, len);
+                }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            return f.getAbsolutePath();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     private static RemotingCommand createRequest() {
@@ -317,8 +345,8 @@ public class TlsTest {
     }
 
     private void requestThenAssertResponse(RemotingClient remotingClient) throws Exception {
-        RemotingCommand response = remotingClient.invokeSync("localhost:8888", createRequest(), 1000 * 3);
-        assertTrue(response != null);
+        RemotingCommand response = remotingClient.invokeSync("localhost:" + remotingServer.localListenPort(), createRequest(), 1000 * 3);
+        assertNotNull(response);
         assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);
         assertThat(response.getExtFields()).hasSize(2);
         assertThat(response.getExtFields().get("messageTitle")).isEqualTo("Welcome");