From 6cee569f2b5c15ed1b5d3416552cd130f177d667 Mon Sep 17 00:00:00 2001
From: osgoo
Date: Sat, 30 May 2020 11:45:34 +0800
Subject: [PATCH 01/83] duplicate doAfterRpcHooks
fix bug: doAfterRpcHooks execute twice.
---
.../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 23376727ce..3f21eb8ff1 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -226,7 +226,6 @@ public abstract class NettyRemotingAbstract {
} else {
NettyRequestProcessor processor = pair.getObject1();
RemotingCommand response = processor.processRequest(ctx, cmd);
- doAfterRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response);
callback.callback(response);
}
} catch (Throwable e) {
From bc3eeed412ca33b7c0090d70cd3be72015bc8fb3 Mon Sep 17 00:00:00 2001
From: wangyalou <1374178468@qq.com>
Date: Sun, 31 May 2020 10:29:46 +0800
Subject: [PATCH 02/83] fix: fix issue 2055
---
.../rocketmq/broker/processor/SendMessageProcessor.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
index 801d886c43..1b836ebf43 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
@@ -178,7 +178,10 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();
if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {
- maxReconsumeTimes = requestHeader.getMaxReconsumeTimes();
+ Integer times = requestHeader.getMaxReconsumeTimes();
+ if (times != null){
+ maxReconsumeTimes = times;
+ }
}
if (msgExt.getReconsumeTimes() >= maxReconsumeTimes
From b954d358ab6b960484bd0c31a172e57df1c079a2 Mon Sep 17 00:00:00 2001
From: "ezlippi.ouyang"
Date: Tue, 9 Mar 2021 13:52:27 +0800
Subject: [PATCH 03/83] feat: issue2715, make netty parameter configure by
system property
---
.../remoting/netty/NettyClientConfig.java | 8 ++++----
.../remoting/netty/NettySystemConfig.java | 19 ++++++++++++++++++-
2 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
index fbc071b28f..5ba3534cb8 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
@@ -20,23 +20,23 @@ public class NettyClientConfig {
/**
* Worker thread number
*/
- private int clientWorkerThreads = 4;
+ private int clientWorkerThreads = NettySystemConfig.clientWorkerSize;
private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors();
private int clientOnewaySemaphoreValue = NettySystemConfig.CLIENT_ONEWAY_SEMAPHORE_VALUE;
private int clientAsyncSemaphoreValue = NettySystemConfig.CLIENT_ASYNC_SEMAPHORE_VALUE;
- private int connectTimeoutMillis = 3000;
+ private int connectTimeoutMillis = NettySystemConfig.connectTimeoutMillis;
private long channelNotActiveInterval = 1000 * 60;
/**
* IdleStateEvent will be triggered when neither read nor write was performed for
* the specified period of this time. Specify {@code 0} to disable
*/
- private int clientChannelMaxIdleTimeSeconds = 120;
+ private int clientChannelMaxIdleTimeSeconds = NettySystemConfig.clientChannelMaxIdleTimeSeconds;
private int clientSocketSndBufSize = NettySystemConfig.socketSndbufSize;
private int clientSocketRcvBufSize = NettySystemConfig.socketRcvbufSize;
private boolean clientPooledByteBufAllocatorEnable = false;
- private boolean clientCloseSocketIfTimeout = false;
+ private boolean clientCloseSocketIfTimeout = NettySystemConfig.clientCloseSocketIfTimeout;
private boolean useTLS;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettySystemConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettySystemConfig.java
index 6357c03ba8..ef767a36ed 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettySystemConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettySystemConfig.java
@@ -28,8 +28,17 @@ public class NettySystemConfig {
"com.rocketmq.remoting.clientAsyncSemaphoreValue";
public static final String COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE =
"com.rocketmq.remoting.clientOnewaySemaphoreValue";
+ public static final String COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE =
+ "com.rocketmq.remoting.client.worker.size";
+ public static final String COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT =
+ "com.rocketmq.remoting.client.connect.timeout";
+ public static final String COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS =
+ "com.rocketmq.remoting.client.channel.maxIdleTimeSeconds";
+ public static final String COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT =
+ "com.rocketmq.remoting.client.closeSocketIfTimeout";
- public static final boolean NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE = //
+
+ public static final boolean NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE = //
Boolean.parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE, "false"));
public static final int CLIENT_ASYNC_SEMAPHORE_VALUE = //
Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE, "65535"));
@@ -39,4 +48,12 @@ public class NettySystemConfig {
Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "65535"));
public static int socketRcvbufSize =
Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "65535"));
+ public static int clientWorkerSize =
+ Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "4"));
+ public static int connectTimeoutMillis =
+ Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "3000"));
+ public static int clientChannelMaxIdleTimeSeconds =
+ Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "120"));
+ public static boolean clientCloseSocketIfTimeout =
+ Boolean.parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "true"));
}
From 33a533b3b713ecc37f9c7168949a3e111a269ff5 Mon Sep 17 00:00:00 2001
From: "ezlippi.ouyang"
Date: Fri, 12 Mar 2021 14:14:14 +0800
Subject: [PATCH 04/83] feat: add some test case
---
.../remoting/netty/NettyClientConfigTest.java | 34 +++++++++++++++++++
1 file changed, 34 insertions(+)
create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
new file mode 100644
index 0000000000..2352871ed3
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
@@ -0,0 +1,34 @@
+package org.apache.rocketmq.remoting.netty;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@RunWith(MockitoJUnitRunner.class)
+public class NettyClientConfigTest {
+
+ @Test
+ public void testChangeConfigBySystemProperty() throws NoSuchFieldException, IllegalAccessException {
+
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "1");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE, "1023");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE, "1024");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "2000");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "60");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "16383");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "16384");
+ System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "false");
+
+ NettyClientConfig changedConfig = new NettyClientConfig();
+ assertThat(changedConfig.getClientWorkerThreads()).isEqualTo(1);
+ assertThat(changedConfig.getClientOnewaySemaphoreValue()).isEqualTo(1023);
+ assertThat(changedConfig.getClientAsyncSemaphoreValue()).isEqualTo(1024);
+ assertThat(changedConfig.getConnectTimeoutMillis()).isEqualTo(2000);
+ assertThat(changedConfig.getClientChannelMaxIdleTimeSeconds()).isEqualTo(60);
+ assertThat(changedConfig.getClientSocketSndBufSize()).isEqualTo(16383);
+ assertThat(changedConfig.getClientSocketRcvBufSize()).isEqualTo(16384);
+ assertThat(changedConfig.isClientCloseSocketIfTimeout()).isEqualTo(false);
+ }
+}
From 03adb63c4c6ea5cedf5a2a43546eaa9cede41c74 Mon Sep 17 00:00:00 2001
From: "ezlippi.ouyang"
Date: Fri, 12 Mar 2021 14:47:56 +0800
Subject: [PATCH 05/83] feat: add some test case
---
.../remoting/netty/NettyClientConfigTest.java | 21 +++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
index 2352871ed3..488bd0371a 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
@@ -11,20 +11,33 @@ public class NettyClientConfigTest {
@Test
public void testChangeConfigBySystemProperty() throws NoSuchFieldException, IllegalAccessException {
+
System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "1");
- System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE, "1023");
- System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE, "1024");
System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "2000");
System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "60");
System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "16383");
System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "16384");
System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "false");
+
+ NettySystemConfig.socketSndbufSize =
+ Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "65535"));
+ NettySystemConfig.socketRcvbufSize =
+ Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "65535"));
+ NettySystemConfig.clientWorkerSize =
+ Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "4"));
+ NettySystemConfig.connectTimeoutMillis =
+ Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "3000"));
+ NettySystemConfig.clientChannelMaxIdleTimeSeconds =
+ Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "120"));
+ NettySystemConfig.clientCloseSocketIfTimeout =
+ Boolean.parseBoolean(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "true"));
+
NettyClientConfig changedConfig = new NettyClientConfig();
assertThat(changedConfig.getClientWorkerThreads()).isEqualTo(1);
- assertThat(changedConfig.getClientOnewaySemaphoreValue()).isEqualTo(1023);
- assertThat(changedConfig.getClientAsyncSemaphoreValue()).isEqualTo(1024);
+ assertThat(changedConfig.getClientOnewaySemaphoreValue()).isEqualTo(65535);
+ assertThat(changedConfig.getClientAsyncSemaphoreValue()).isEqualTo(65535);
assertThat(changedConfig.getConnectTimeoutMillis()).isEqualTo(2000);
assertThat(changedConfig.getClientChannelMaxIdleTimeSeconds()).isEqualTo(60);
assertThat(changedConfig.getClientSocketSndBufSize()).isEqualTo(16383);
From 98237b139ae57f274c78f6765fe039995b7755eb Mon Sep 17 00:00:00 2001
From: "ezlippi.ouyang"
Date: Fri, 12 Mar 2021 15:27:31 +0800
Subject: [PATCH 06/83] feat: add some test case
---
.../apache/rocketmq/remoting/netty/NettyClientConfigTest.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
index 488bd0371a..3864ce87e0 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
@@ -43,5 +43,6 @@ public class NettyClientConfigTest {
assertThat(changedConfig.getClientSocketSndBufSize()).isEqualTo(16383);
assertThat(changedConfig.getClientSocketRcvBufSize()).isEqualTo(16384);
assertThat(changedConfig.isClientCloseSocketIfTimeout()).isEqualTo(false);
+
}
}
From 67ceb64eb6ae0e59f06347bbf74004e21f4ef188 Mon Sep 17 00:00:00 2001
From: "ezlippi.ouyang"
Date: Fri, 12 Mar 2021 18:05:22 +0800
Subject: [PATCH 07/83] fix: add license
---
.../remoting/netty/NettyClientConfigTest.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
index 3864ce87e0..c0acdbe0cb 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.rocketmq.remoting.netty;
import org.junit.Test;
From 59cbbc4337d79b070ab089cb91be02af19393863 Mon Sep 17 00:00:00 2001
From: "ezlippi.ouyang"
Date: Mon, 15 Mar 2021 10:08:26 +0800
Subject: [PATCH 08/83] feat: add some test case
---
.../apache/rocketmq/remoting/netty/NettyClientConfigTest.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
index c0acdbe0cb..15cf033864 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java
@@ -60,6 +60,5 @@ public class NettyClientConfigTest {
assertThat(changedConfig.getClientSocketSndBufSize()).isEqualTo(16383);
assertThat(changedConfig.getClientSocketRcvBufSize()).isEqualTo(16384);
assertThat(changedConfig.isClientCloseSocketIfTimeout()).isEqualTo(false);
-
}
}
From 56c3ad8718485d2b47953e2f6ef4cfa80223558c Mon Sep 17 00:00:00 2001
From: zhurunhua
Date: Wed, 12 May 2021 16:15:52 +0800
Subject: [PATCH 09/83] update the option's description of deleteAccessConfig
command
---
.../tools/command/acl/DeleteAccessConfigSubCommand.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
index 8570b2f758..0a1a4b2382 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
@@ -44,10 +44,10 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
public Options buildCommandlineOptions(Options options) {
OptionGroup optionGroup = new OptionGroup();
- Option opt = new Option("b", "brokerAddr", true, "delete acl config account to which broker");
+ Option opt = new Option("b", "brokerAddr", true, "delete acl config account from which broker");
optionGroup.addOption(opt);
- opt = new Option("c", "clusterName", true, "delete cl config account to which cluster");
+ opt = new Option("c", "clusterName", true, "delete acl config account from which cluster");
optionGroup.addOption(opt);
optionGroup.setRequired(true);
From 22647dea4df1fae4108abeafc68a9df4919a50cd Mon Sep 17 00:00:00 2001
From: zhurunhua
Date: Thu, 13 May 2021 14:47:56 +0800
Subject: [PATCH 10/83] update the option's description of updateAccessConfig
command
---
.../tools/command/acl/DeleteAccessConfigSubCommand.java | 4 ++--
.../tools/command/acl/UpdateAccessConfigSubCommand.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
index 0a1a4b2382..d82453dc9f 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
@@ -76,7 +76,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
defaultMQAdminExt.start();
defaultMQAdminExt.deletePlainAccessConfig(addr, accessKey);
- System.out.printf("delete plain access config account to %s success.%n", addr);
+ System.out.printf("delete plain access config account from %s success.%n", addr);
System.out.printf("account's accesskey is:%s", accessKey);
return;
@@ -89,7 +89,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
for (String addr : masterSet) {
defaultMQAdminExt.deletePlainAccessConfig(addr, accessKey);
- System.out.printf("delete plain access config account to %s success.%n", addr);
+ System.out.printf("delete plain access config account from %s success.%n", addr);
}
System.out.printf("account's accesskey is:%s", accessKey);
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
index 10241bf91e..b625cd94c1 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
@@ -50,7 +50,7 @@ public class UpdateAccessConfigSubCommand implements SubCommand {
Option opt = new Option("b", "brokerAddr", true, "update acl config file to which broker");
optionGroup.addOption(opt);
- opt = new Option("c", "clusterName", true, "update cl config file to which cluster");
+ opt = new Option("c", "clusterName", true, "update acl config file to which cluster");
optionGroup.addOption(opt);
optionGroup.setRequired(true);
From 5fe0041686739ce3959dfa17a6ae63b84ab6320c Mon Sep 17 00:00:00 2001
From: huangli
Date: Tue, 9 Feb 2021 20:42:35 +0800
Subject: [PATCH 11/83] Change default value of some parameters:
sendMessageThreadPoolNums/useReentrantLockWhenPutMessage/flushCommitLogTimed/endTransactionThreadPoolNums
---
.../java/org/apache/rocketmq/common/BrokerConfig.java | 8 ++++----
.../apache/rocketmq/store/config/MessageStoreConfig.java | 7 +++----
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
index d80b3d2169..f710cdb403 100644
--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
+++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
@@ -57,10 +57,9 @@ public class BrokerConfig {
@ImportantField
private boolean traceTopicEnable = false;
/**
- * thread numbers for send message thread pool, since spin lock will be used by default since 4.0.x, the default
- * value is 1.
+ * thread numbers for send message thread pool.
*/
- private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4;
+ private int sendMessageThreadPoolNums = Math.min(Runtime.getRuntime().availableProcessors(), 4);
private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2;
private int processReplyMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2;
private int queryMessageThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors();
@@ -73,7 +72,8 @@ public class BrokerConfig {
/**
* Thread numbers for EndTransactionProcessor
*/
- private int endTransactionThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors() * 2;
+ private int endTransactionThreadPoolNums = Math.max(8 + Runtime.getRuntime().availableProcessors() * 2,
+ sendMessageThreadPoolNums * 4);
private int flushConsumerOffsetInterval = 1000 * 5;
diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
index c13ad4cfa2..978e85baca 100644
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
@@ -54,13 +54,12 @@ public class MessageStoreConfig {
/**
* introduced since 4.0.x. Determine whether to use mutex reentrantLock when putting message.
- * By default it is set to false indicating using spin lock when putting message.
*/
- private boolean useReentrantLockWhenPutMessage = false;
+ private boolean useReentrantLockWhenPutMessage = true;
- // Whether schedule flush,default is real-time
+ // Whether schedule flush
@ImportantField
- private boolean flushCommitLogTimed = false;
+ private boolean flushCommitLogTimed = true;
// ConsumeQueue flush interval
private int flushIntervalConsumeQueue = 1000;
// Resource reclaim interval
From b186bcd53f7f211a6557d5121ffd4a7afdfc93bb Mon Sep 17 00:00:00 2001
From: zhurunhua
Date: Wed, 26 May 2021 15:10:38 +0800
Subject: [PATCH 12/83] update the option's description of consumer.sh
---
.../java/org/apache/rocketmq/example/benchmark/Consumer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
index b6c6ae410f..1b52a91ffa 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java
@@ -190,7 +190,7 @@ public class Consumer {
opt.setRequired(false);
options.addOption(opt);
- opt = new Option("p", "group prefix enable", true, "Consumer group name, Default: false");
+ opt = new Option("p", "group prefix enable", true, "Is group prefix enable, Default: false");
opt.setRequired(false);
options.addOption(opt);
From 20b6cddb9454bc8671aaf0a16d8e4b0b0bc529c3 Mon Sep 17 00:00:00 2001
From: huangli
Date: Thu, 27 May 2021 15:36:13 +0800
Subject: [PATCH 13/83] [ISSUE 2883] Remove an duplicate
MessageDecoder.string2messageProperties for each message, and prevent store
"WAIT=true" property (in most case) to save 9 bytes for each message.
---
.../broker/processor/SendMessageProcessor.java | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
index 724cf54c81..45d71f4aeb 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
@@ -277,18 +277,26 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
msgInner.setBody(body);
msgInner.setFlag(requestHeader.getFlag());
- MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties()));
- msgInner.setPropertiesString(requestHeader.getProperties());
+ Map origProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+ MessageAccessor.setProperties(msgInner, origProps);
msgInner.setBornTimestamp(requestHeader.getBornTimestamp());
msgInner.setBornHost(ctx.channel().remoteAddress());
msgInner.setStoreHost(this.getStoreHost());
msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);
- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+ if (origProps.containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) {
+ // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message.
+ // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it.
+ String waitStoreMsgOKValue = origProps.remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);
+ msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+ // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later
+ origProps.put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue);
+ } else {
+ msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+ }
CompletableFuture putMessageResult = null;
- Map origProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
String transFlag = origProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (transFlag != null && Boolean.parseBoolean(transFlag)) {
if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
From f42db53bb024633dae1350f0bead4e73458dde0a Mon Sep 17 00:00:00 2001
From: nilei3
Date: Thu, 3 Jun 2021 23:23:41 +0800
Subject: [PATCH 14/83] Changing the comma input method
---
docs/cn/design.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/cn/design.md b/docs/cn/design.md
index dcd39026f2..ab88e1c530 100644
--- a/docs/cn/design.md
+++ b/docs/cn/design.md
@@ -10,7 +10,7 @@
#### 1.1 消息存储整体架构
消息存储架构图中主要有下面三个跟消息存储相关的文件构成。
-(1) CommitLog:消息主体以及元数据的存储主体,存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G ,文件名长度为20位,左边补零,剩余为起始偏移量,比如00000000000000000000代表了第一个文件,起始偏移量为0,文件大小为1G=1073741824;当第一个文件写满了,第二个文件为00000000001073741824,起始偏移量为1073741824,以此类推。消息主要是顺序写入日志文件,当文件满了,写入下一个文件;
+(1) CommitLog:消息主体以及元数据的存储主体,存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G,文件名长度为20位,左边补零,剩余为起始偏移量,比如00000000000000000000代表了第一个文件,起始偏移量为0,文件大小为1G=1073741824;当第一个文件写满了,第二个文件为00000000001073741824,起始偏移量为1073741824,以此类推。消息主要是顺序写入日志文件,当文件满了,写入下一个文件;
(2) ConsumeQueue:消息消费队列,引入的目的主要是提高消息消费的性能,由于RocketMQ是基于主题topic的订阅模式,消息消费是针对主题进行的,如果要遍历commitlog文件中根据topic检索消息是非常低效的。Consumer即可根据ConsumeQueue来查找待消费的消息。其中,ConsumeQueue(逻辑消费队列)作为消费消息的索引,保存了指定Topic下的队列消息在CommitLog中的起始物理偏移量offset,消息大小size和消息Tag的HashCode值。consumequeue文件可以看成是基于topic的commitlog索引文件,故consumequeue文件夹的组织方式如下:topic/queue/file三层组织结构,具体存储路径为:$HOME/store/consumequeue/{topic}/{queueId}/{fileName}。同样consumequeue文件采取定长设计,每一个条目共20个字节,分别为8字节的commitlog物理偏移量、4字节的消息长度、8字节tag hashcode,单个文件由30W个条目组成,可以像数组一样随机访问每一个条目,每个ConsumeQueue文件大小约5.72M;
From 317fd7e01d859189812848b580ada087892bffaa Mon Sep 17 00:00:00 2001
From: "iostream2008@163.com"
Date: Thu, 3 Jun 2021 23:29:59 +0800
Subject: [PATCH 15/83] Changing the comma input method
---
docs/cn/design.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/cn/design.md b/docs/cn/design.md
index ab88e1c530..99950746f7 100644
--- a/docs/cn/design.md
+++ b/docs/cn/design.md
@@ -10,7 +10,7 @@
#### 1.1 消息存储整体架构
消息存储架构图中主要有下面三个跟消息存储相关的文件构成。
-(1) CommitLog:消息主体以及元数据的存储主体,存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G,文件名长度为20位,左边补零,剩余为起始偏移量,比如00000000000000000000代表了第一个文件,起始偏移量为0,文件大小为1G=1073741824;当第一个文件写满了,第二个文件为00000000001073741824,起始偏移量为1073741824,以此类推。消息主要是顺序写入日志文件,当文件满了,写入下一个文件;
+(1) CommitLog:消息主体以及元数据的存储主体,存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G, 文件名长度为20位,左边补零,剩余为起始偏移量,比如00000000000000000000代表了第一个文件,起始偏移量为0,文件大小为1G=1073741824;当第一个文件写满了,第二个文件为00000000001073741824,起始偏移量为1073741824,以此类推。消息主要是顺序写入日志文件,当文件满了,写入下一个文件;
(2) ConsumeQueue:消息消费队列,引入的目的主要是提高消息消费的性能,由于RocketMQ是基于主题topic的订阅模式,消息消费是针对主题进行的,如果要遍历commitlog文件中根据topic检索消息是非常低效的。Consumer即可根据ConsumeQueue来查找待消费的消息。其中,ConsumeQueue(逻辑消费队列)作为消费消息的索引,保存了指定Topic下的队列消息在CommitLog中的起始物理偏移量offset,消息大小size和消息Tag的HashCode值。consumequeue文件可以看成是基于topic的commitlog索引文件,故consumequeue文件夹的组织方式如下:topic/queue/file三层组织结构,具体存储路径为:$HOME/store/consumequeue/{topic}/{queueId}/{fileName}。同样consumequeue文件采取定长设计,每一个条目共20个字节,分别为8字节的commitlog物理偏移量、4字节的消息长度、8字节tag hashcode,单个文件由30W个条目组成,可以像数组一样随机访问每一个条目,每个ConsumeQueue文件大小约5.72M;
From a027f8d6a0d4d6406c8265199433781f21ad7566 Mon Sep 17 00:00:00 2001
From: zongtanghu
Date: Sat, 5 Jun 2021 12:29:37 +0800
Subject: [PATCH 16/83] [maven-release-plugin] prepare release
rocketmq-all-4.9.0
---
acl/pom.xml | 2 +-
broker/pom.xml | 2 +-
client/pom.xml | 2 +-
common/pom.xml | 2 +-
distribution/pom.xml | 2 +-
example/pom.xml | 2 +-
filter/pom.xml | 2 +-
logappender/pom.xml | 2 +-
logging/pom.xml | 2 +-
namesrv/pom.xml | 2 +-
openmessaging/pom.xml | 2 +-
pom.xml | 4 ++--
remoting/pom.xml | 2 +-
srvutil/pom.xml | 2 +-
store/pom.xml | 2 +-
test/pom.xml | 2 +-
tools/pom.xml | 2 +-
17 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/acl/pom.xml b/acl/pom.xml
index 36dae6ad04..5e61c7cbdd 100644
--- a/acl/pom.xml
+++ b/acl/pom.xml
@@ -13,7 +13,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
rocketmq-acl
rocketmq-acl ${project.version}
diff --git a/broker/pom.xml b/broker/pom.xml
index 0137b55678..f31f0af807 100644
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@ -13,7 +13,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/client/pom.xml b/client/pom.xml
index 87dc89426a..446c5efaee 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/common/pom.xml b/common/pom.xml
index 068de26dcb..0e3bf7bfb7 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/distribution/pom.xml b/distribution/pom.xml
index d5ab667229..1d090dec0c 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -20,7 +20,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
rocketmq-distribution
rocketmq-distribution ${project.version}
diff --git a/example/pom.xml b/example/pom.xml
index 568dd80295..1a54df7379 100644
--- a/example/pom.xml
+++ b/example/pom.xml
@@ -19,7 +19,7 @@
rocketmq-all
org.apache.rocketmq
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/filter/pom.xml b/filter/pom.xml
index c12dd1fb28..f7cccd71f8 100644
--- a/filter/pom.xml
+++ b/filter/pom.xml
@@ -20,7 +20,7 @@
rocketmq-all
org.apache.rocketmq
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/logappender/pom.xml b/logappender/pom.xml
index a6e979a83e..181f1bd478 100644
--- a/logappender/pom.xml
+++ b/logappender/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
rocketmq-logappender
diff --git a/logging/pom.xml b/logging/pom.xml
index 77d0f29a97..e9fa2a42f4 100644
--- a/logging/pom.xml
+++ b/logging/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/namesrv/pom.xml b/namesrv/pom.xml
index c5d6d05bc2..95cc7b2faa 100644
--- a/namesrv/pom.xml
+++ b/namesrv/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml
index 7f0e937996..069900cb48 100644
--- a/openmessaging/pom.xml
+++ b/openmessaging/pom.xml
@@ -20,7 +20,7 @@
rocketmq-all
org.apache.rocketmq
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/pom.xml b/pom.xml
index 39f368538e..ed700c9551 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
2012
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
pom
Apache RocketMQ ${project.version}
http://rocketmq.apache.org/
@@ -38,7 +38,7 @@
git@github.com:apache/rocketmq.git
scm:git:git@github.com:apache/rocketmq.git
scm:git:git@github.com:apache/rocketmq.git
- HEAD
+ rocketmq-all-4.9.0
diff --git a/remoting/pom.xml b/remoting/pom.xml
index d9dab9efa6..6df93dc692 100644
--- a/remoting/pom.xml
+++ b/remoting/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/srvutil/pom.xml b/srvutil/pom.xml
index b95808b7c2..7f1d4c0102 100644
--- a/srvutil/pom.xml
+++ b/srvutil/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/store/pom.xml b/store/pom.xml
index ed54db1b24..33e8dc9e60 100644
--- a/store/pom.xml
+++ b/store/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/test/pom.xml b/test/pom.xml
index 851d9ab9aa..02f90101d8 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -20,7 +20,7 @@
rocketmq-all
org.apache.rocketmq
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
diff --git a/tools/pom.xml b/tools/pom.xml
index eeccae81bf..355464e2ff 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.8.1-SNAPSHOT
+ 4.9.0
4.0.0
From a1babab507934e81f0e05b2867566c8b459be341 Mon Sep 17 00:00:00 2001
From: zongtanghu
Date: Sat, 5 Jun 2021 12:30:00 +0800
Subject: [PATCH 17/83] [maven-release-plugin] prepare for next development
iteration
---
acl/pom.xml | 2 +-
broker/pom.xml | 2 +-
client/pom.xml | 2 +-
common/pom.xml | 2 +-
distribution/pom.xml | 2 +-
example/pom.xml | 2 +-
filter/pom.xml | 2 +-
logappender/pom.xml | 2 +-
logging/pom.xml | 2 +-
namesrv/pom.xml | 2 +-
openmessaging/pom.xml | 2 +-
pom.xml | 4 ++--
remoting/pom.xml | 2 +-
srvutil/pom.xml | 2 +-
store/pom.xml | 2 +-
test/pom.xml | 2 +-
tools/pom.xml | 2 +-
17 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/acl/pom.xml b/acl/pom.xml
index 5e61c7cbdd..d624934178 100644
--- a/acl/pom.xml
+++ b/acl/pom.xml
@@ -13,7 +13,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
rocketmq-acl
rocketmq-acl ${project.version}
diff --git a/broker/pom.xml b/broker/pom.xml
index f31f0af807..3a58aa0c71 100644
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@ -13,7 +13,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/client/pom.xml b/client/pom.xml
index 446c5efaee..95ef4617dd 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/common/pom.xml b/common/pom.xml
index 0e3bf7bfb7..ac1d086b4c 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 1d090dec0c..a5698bca1c 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -20,7 +20,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
rocketmq-distribution
rocketmq-distribution ${project.version}
diff --git a/example/pom.xml b/example/pom.xml
index 1a54df7379..81f7515dbe 100644
--- a/example/pom.xml
+++ b/example/pom.xml
@@ -19,7 +19,7 @@
rocketmq-all
org.apache.rocketmq
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/filter/pom.xml b/filter/pom.xml
index f7cccd71f8..41bda9d0d2 100644
--- a/filter/pom.xml
+++ b/filter/pom.xml
@@ -20,7 +20,7 @@
rocketmq-all
org.apache.rocketmq
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/logappender/pom.xml b/logappender/pom.xml
index 181f1bd478..cac520b610 100644
--- a/logappender/pom.xml
+++ b/logappender/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
rocketmq-logappender
diff --git a/logging/pom.xml b/logging/pom.xml
index e9fa2a42f4..8ebb243045 100644
--- a/logging/pom.xml
+++ b/logging/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/namesrv/pom.xml b/namesrv/pom.xml
index 95cc7b2faa..01ef5b60fe 100644
--- a/namesrv/pom.xml
+++ b/namesrv/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml
index 069900cb48..4688c04043 100644
--- a/openmessaging/pom.xml
+++ b/openmessaging/pom.xml
@@ -20,7 +20,7 @@
rocketmq-all
org.apache.rocketmq
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/pom.xml b/pom.xml
index ed700c9551..b6f1770e2d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
2012
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
pom
Apache RocketMQ ${project.version}
http://rocketmq.apache.org/
@@ -38,7 +38,7 @@
git@github.com:apache/rocketmq.git
scm:git:git@github.com:apache/rocketmq.git
scm:git:git@github.com:apache/rocketmq.git
- rocketmq-all-4.9.0
+ HEAD
diff --git a/remoting/pom.xml b/remoting/pom.xml
index 6df93dc692..765e1a93bc 100644
--- a/remoting/pom.xml
+++ b/remoting/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/srvutil/pom.xml b/srvutil/pom.xml
index 7f1d4c0102..9c4544c0b2 100644
--- a/srvutil/pom.xml
+++ b/srvutil/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/store/pom.xml b/store/pom.xml
index 33e8dc9e60..23e9763d81 100644
--- a/store/pom.xml
+++ b/store/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/test/pom.xml b/test/pom.xml
index 02f90101d8..41296501c2 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -20,7 +20,7 @@
rocketmq-all
org.apache.rocketmq
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
diff --git a/tools/pom.xml b/tools/pom.xml
index 355464e2ff..abe8197de3 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -19,7 +19,7 @@
org.apache.rocketmq
rocketmq-all
- 4.9.0
+ 4.9.1-SNAPSHOT
4.0.0
From 2fe6dc3034f3d886227c0afbade7ad785e5f9edc Mon Sep 17 00:00:00 2001
From: yuz10 <845238369@qq.com>
Date: Wed, 9 Jun 2021 00:57:42 +0800
Subject: [PATCH 18/83] #2990 Add message tracing switch in sendMessage command
of mqadmin
---
.../command/message/SendMessageCommand.java | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java
index e4921c6f04..9550742e8f 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java
@@ -68,14 +68,18 @@ public class SendMessageCommand implements SubCommand {
opt.setRequired(false);
options.addOption(opt);
+ opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false");
+ opt.setRequired(false);
+ options.addOption(opt);
+
return options;
}
- private DefaultMQProducer createProducer(RPCHook rpcHook) {
+ private DefaultMQProducer createProducer(RPCHook rpcHook, boolean msgTraceEnable) {
if (this.producer != null) {
return producer;
} else {
- producer = new DefaultMQProducer(rpcHook);
+ producer = new DefaultMQProducer(null, rpcHook, msgTraceEnable, null);
producer.setProducerGroup(Long.toString(System.currentTimeMillis()));
return producer;
}
@@ -112,8 +116,11 @@ public class SendMessageCommand implements SubCommand {
} catch (Exception e) {
throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e);
}
-
- DefaultMQProducer producer = this.createProducer(rpcHook);
+ boolean msgTraceEnable = false;
+ if (commandLine.hasOption('m')) {
+ msgTraceEnable = Boolean.parseBoolean(commandLine.getOptionValue('m').trim());
+ }
+ DefaultMQProducer producer = this.createProducer(rpcHook, msgTraceEnable);
SendResult result;
try {
producer.start();
@@ -153,4 +160,4 @@ public class SendMessageCommand implements SubCommand {
);
}
}
-}
\ No newline at end of file
+}
From 93dd0867c8e52d978a02610d5aced4d71b961c33 Mon Sep 17 00:00:00 2001
From: "fuzhi.lfz"
Date: Wed, 16 Jun 2021 16:31:16 +0800
Subject: [PATCH 19/83] AsyncTraceDispatcher unique groupName
---
.../apache/rocketmq/client/trace/AsyncTraceDispatcher.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
index 8210b41cd4..754cbd55ac 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.client.AccessChannel;
@@ -54,6 +55,7 @@ import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAM
public class AsyncTraceDispatcher implements TraceDispatcher {
private final static InternalLogger log = ClientLogger.getLog();
+ private final static AtomicInteger COUNTER = new AtomicInteger();
private final int queueSize;
private final int batchSize;
private final int maxMsgSize;
@@ -165,7 +167,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
}
private String genGroupNameForTrace() {
- return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type ;
+ return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type + "-" + COUNTER.incrementAndGet();
}
@Override
From 0c36d14e934ad428cd8edabefe94fb9f9e54bb79 Mon Sep 17 00:00:00 2001
From: "fuzhi.lfz"
Date: Thu, 17 Jun 2021 11:27:58 +0800
Subject: [PATCH 20/83] remove SubAfter field
---
.../org/apache/rocketmq/client/trace/TraceDataEncoder.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java
index b65ba46a39..10aa241359 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java
@@ -188,9 +188,7 @@ public class TraceDataEncoder {
.append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//
.append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)//
.append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//
- .append(ctx.getContextCode()).append(TraceConstants.CONTENT_SPLITOR)
- .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)
- .append(ctx.getGroupName()).append(TraceConstants.FIELD_SPLITOR);
+ .append(ctx.getContextCode()).append(TraceConstants.FIELD_SPLITOR);
}
}
From e1263107ebbe3182a0b4871d8d20d3ed40c16076 Mon Sep 17 00:00:00 2001
From: zourunxin
Date: Sat, 19 Jun 2021 10:54:37 +0800
Subject: [PATCH 21/83] [ISSUE #2997] Add a Chinese version of
Configuration_System.md
---
docs/cn/Configuration_System.md | 70 +++++++++++++++++++++++++++++++++
1 file changed, 70 insertions(+)
create mode 100644 docs/cn/Configuration_System.md
diff --git a/docs/cn/Configuration_System.md b/docs/cn/Configuration_System.md
new file mode 100644
index 0000000000..d72ef93813
--- /dev/null
+++ b/docs/cn/Configuration_System.md
@@ -0,0 +1,70 @@
+# 系统配置
+
+本节重点介绍系统(JVM/OS)的配置
+---
+
+## **1 JVM 选项** ##
+
+建议使用最新发布的 JDK 1.8 版本。设置相同的 Xms 和 Xmx 值以防止 JVM 调整堆大小,并获得更好的性能。一种通用的JVM配置如下:
+
+ -server -Xms8g -Xmx8g -Xmn4g
+
+设置 Direct ByteBuffer 内存大小。当 Direct ByteBuffer 达到指定大小时,将触发 Full GC:
+
+ -XXMaxDirectMemorySize=15g
+
+如果你不在乎 RocketMQ broker 的启动时间,建议启用预分配 Java 堆以确保在 JVM 初始化期间为每个页面分配内存。你可以通过以下方式启用它:
+
+ -XX+AlwaysPreTouch
+
+禁用偏向锁定可以减少 JVM 停顿:
+
+ -XX-UseBiasedLocking
+
+关于垃圾收集器,推荐使用 JDK 1.8 的 G1 收集器:
+
+ -XX+UseG1GC -XXG1HeapRegionSize=16m
+ -XXG1ReservePercent=25
+ -XXInitiatingHeapOccupancyPercent=30
+
+这些 GC 选项看起来有点激进,但事实证明它在生产环境中具有良好的性能
+
+不要把-XXMaxGCPauseMillis 的值设置太小,否则JVM会使用一个小的新生代来实现这个目标,从而导致频繁发生minor GC。因此,建议使用滚动 GC 日志文件:
+
+ -XX+UseGCLogFileRotation
+ -XXNumberOfGCLogFiles=5
+ -XXGCLogFileSize=30m
+
+写 GC 文件会增加 broker 的延迟,因此可以考虑将 GC 日志文件重定向到内存文件系统:
+
+ -Xloggcdevshmmq_gc_%p.log123
+
+## 2 Linux 内核参数 ##
+
+在 bin 文件夹里,有一个 os.sh 脚本,里面列出了许多的内核参数,只需稍作更改即可用于生产用途。需特别关注以下参数,如想了解更多细节,请参考文档/proc/sys/vm/*。
+
+
+
+- **vm.extra_free_kbytes**, 控制VM在后台回收(kswapd)开始的阈值和直接回收(通过分配进程)开始的阈值之间保留额外的空闲内存。通过使用这个参数,RocketMQ 可以避免在内存分配过程中出现高延迟。(与内核版本版本有关)
+
+
+
+- **vm.min_free_kbytes**, 该值不应设置低于1024KB,否则系统将遭到破坏,并且在高负载环境下容易出现死锁。
+
+
+
+
+
+- **vm.max_map_count**, 规定进程可以拥有的最大内存映射区域数。 RocketMQ 使用 mmap 来加载 CommitLog 和 ConsumeQueue,因此建议将此参数设置为较大的值。
+
+
+
+- **vm.swappiness**, 定义内核交换内存页的频率。该值若较大,则会导致频繁交换,较小则会减少交换量。为了避免交换延迟,建议将此值设为 10。
+
+
+
+- **File descriptor limits**, RocketMQ 需要给文件(CommitLog 和 ConsumeQueue)和网络连接分配文件描述符。因此建议将该值设置为 655350。
+
+
+
+- **Disk scheduler**, 推荐使用deadline IO 调度器,它可以为请求提供有保证的延迟。
\ No newline at end of file
From ece364a14ea679d5eb765ca03fac5886d637feac Mon Sep 17 00:00:00 2001
From: CharliePu
Date: Mon, 21 Jun 2021 15:13:19 +0800
Subject: [PATCH 22/83] Improve the comment of maxReconsumeTimes
---
.../rocketmq/client/consumer/DefaultMQPushConsumer.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
index 9011117a79..caf166de47 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
@@ -236,11 +236,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
private boolean unitMode = false;
/**
- * Max re-consume times. -1 means 16 times.
- *
+ * Max re-consume times.
+ * In concurrently mode, -1 means 16;
+ * In orderly mode, -1 means Integer.MAX_VALUE.
*
- * If messages are re-consumed more than {@link #maxReconsumeTimes} before success, it's be directed to a deletion
- * queue waiting.
+ * If messages are re-consumed more than {@link #maxReconsumeTimes} before success.
*/
private int maxReconsumeTimes = -1;
From 2b236c3cd8ce2883c0188c759ccaa644fea8e00c Mon Sep 17 00:00:00 2001
From: Git_Yang <30995057+Git-Yang@users.noreply.github.com>
Date: Mon, 21 Jun 2021 20:35:32 +0800
Subject: [PATCH 23/83] [ISSUE 3019] Fixed bug with batch-sent message
statistics
Signed-off-by: zhangyang
---
.../org/apache/rocketmq/store/dledger/DLedgerCommitLog.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
index 9241ffe423..90fd6f3f16 100644
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
@@ -553,7 +553,7 @@ public class DLedgerCommitLog extends CommitLog {
msgIdBuilder.setLength(0);
long elapsedTimeInLock;
long queueOffset;
- long msgNum = 0;
+ int msgNum = 0;
try {
beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();
queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType);
@@ -592,6 +592,7 @@ public class DLedgerCommitLog extends CommitLog {
elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen,
msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
+ appendResult.setMsgNum(msgNum);
DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + msgNum);
} catch (Exception e) {
log.error("Put message error", e);
@@ -779,7 +780,7 @@ public class DLedgerCommitLog extends CommitLog {
msgIdBuilder.setLength(0);
long elapsedTimeInLock;
long queueOffset;
- long msgNum = 0;
+ int msgNum = 0;
try {
beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();
queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType);
@@ -818,6 +819,7 @@ public class DLedgerCommitLog extends CommitLog {
elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen,
msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
+ appendResult.setMsgNum(msgNum);
DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + msgNum);
} catch (Exception e) {
log.error("Put message error", e);
From 25f6711d7fd6fe0f00532a810b9fef7e81467b03 Mon Sep 17 00:00:00 2001
From: zhangyang
Date: Mon, 21 Jun 2021 20:42:29 +0800
Subject: [PATCH 24/83] [statistics] Fix the inconsistency of
putMessageDistributeTime statistics
Signed-off-by: zhangyang
---
.../java/org/apache/rocketmq/store/StoreStatsService.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java b/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java
index d43b34342f..8372845e7f 100644
--- a/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java
@@ -57,6 +57,7 @@ public class StoreStatsService extends ServiceThread {
private final LinkedList getTimesMissList = new LinkedList();
private final LinkedList transferedMsgCountList = new LinkedList();
private volatile AtomicLong[] putMessageDistributeTime;
+ private volatile AtomicLong[] lastPutMessageDistributeTime;
private long messageStoreBootTimestamp = System.currentTimeMillis();
private volatile long putMessageEntireTimeMax = 0;
private volatile long getMessageEntireTimeMax = 0;
@@ -80,11 +81,11 @@ public class StoreStatsService extends ServiceThread {
next[i] = new AtomicLong(0);
}
- AtomicLong[] old = this.putMessageDistributeTime;
+ this.lastPutMessageDistributeTime = this.putMessageDistributeTime;
this.putMessageDistributeTime = next;
- return old;
+ return lastPutMessageDistributeTime;
}
public long getPutMessageEntireTimeMax() {
@@ -298,7 +299,7 @@ public class StoreStatsService extends ServiceThread {
}
private String putMessageDistributeTimeToString() {
- final AtomicLong[] times = this.putMessageDistributeTime;
+ final AtomicLong[] times = this.lastPutMessageDistributeTime;
if (null == times)
return null;
From d8dec654b76ecb840668f64cd3632958ddcff2a2 Mon Sep 17 00:00:00 2001
From: patrick
Date: Wed, 23 Jun 2021 10:48:36 +0800
Subject: [PATCH 25/83] add different send examples
---
.../rocketmq/example/quickstart/Producer.java | 34 +++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java
index 53a1d4dd64..0da6177996 100644
--- a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java
@@ -18,6 +18,7 @@ package org.apache.rocketmq.example.quickstart;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendCallback;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -65,6 +66,39 @@ public class Producer {
* Call send message to deliver message to one of brokers.
*/
SendResult sendResult = producer.send(msg);
+ /*
+ * There are different ways to send message, if you don't care about the send result,you can use this way
+ * {@code
+ * producer.sendOneway(msg);
+ * }
+ */
+
+ /*
+ * if you want to get the send result in a synchronize way, you can use this send method
+ * {@code
+ * SendResult sendResult = producer.send(msg);
+ * System.out.println(sendResult.toString());
+ * }
+ */
+
+ /*
+ * if you want to get the send result in a asynchronize way, you can use this send method
+ * {@code
+ *
+ * producer.send(msg, new SendCallback() {
+ * @Override
+ * public void onSuccess(SendResult sendResult) {
+ * // do something
+ * }
+ *
+ * @Override
+ * public void onException(Throwable e) {
+ * // do something
+ * }
+ *});
+ *
+ *}
+ */
System.out.printf("%s%n", sendResult);
} catch (Exception e) {
From 160c5772fbdf02e55cdc67bc3b74bb9f9c4c644b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=97=AD?=
Date: Wed, 23 Jun 2021 11:00:07 +0800
Subject: [PATCH 26/83] test(benchmark): support delay message test (#3010)
Co-authored-by: zhangxu16
---
.../rocketmq/example/benchmark/Producer.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
index 6975ab58b0..32d4b9f746 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
@@ -61,9 +61,11 @@ public class Producer {
final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m'));
final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a'));
final long messageNum = commandLine.hasOption('q') ? Long.parseLong(commandLine.getOptionValue('q')) : 0;
+ final boolean delayEnable = commandLine.hasOption('d') && Boolean.parseBoolean(commandLine.getOptionValue('d'));
+ final int delayLevel = commandLine.hasOption('e') ? Integer.parseInt(commandLine.getOptionValue('e')) : 1;
- System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n",
- topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum);
+ System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n delayEnable: %s%n delayLevel: %s%n",
+ topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum, delayEnable, delayLevel);
final InternalLogger log = ClientLogger.getLog();
@@ -147,6 +149,9 @@ public class Producer {
if (keyEnable) {
msg.setKeys(String.valueOf(beginTimestamp / 1000));
}
+ if (delayEnable) {
+ msg.setDelayTimeLevel(delayLevel);
+ }
if (tagCount > 0) {
long sendSucCount = statsBenchmark.getReceiveResponseSuccessCount().get();
msg.setTags(String.format("tag%d", sendSucCount % tagCount));
@@ -266,6 +271,14 @@ public class Producer {
opt.setRequired(false);
options.addOption(opt);
+ opt = new Option("d", "delayEnable", true, "Delay message Enable, Default: false");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("e", "delayLevel", true, "Delay message level, Default: 1");
+ opt.setRequired(false);
+ options.addOption(opt);
+
return options;
}
From 361eb8d286c3bdc19260d4315a87fa8427673fa5 Mon Sep 17 00:00:00 2001
From: messense
Date: Tue, 22 Jun 2021 22:34:55 +0800
Subject: [PATCH 27/83] Upgrade netty-all dependency version to 4.1.65.Final
---
.../broker/pagecache/ManyMessageTransfer.java | 27 +++++++++++
.../broker/pagecache/OneMessageTransfer.java | 27 +++++++++++
.../pagecache/QueryMessageTransfer.java | 27 +++++++++++
pom.xml | 2 +-
.../rocketmq/remoting/netty/NettyLogger.java | 46 +++++++++++++++++++
5 files changed, 128 insertions(+), 1 deletion(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransfer.java b/broker/src/main/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransfer.java
index 968bcfb564..373a0a48b6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransfer.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransfer.java
@@ -53,6 +53,11 @@ public class ManyMessageTransfer extends AbstractReferenceCounted implements Fil
return transferred;
}
+ @Override
+ public long transferred() {
+ return transferred;
+ }
+
@Override
public long count() {
return byteBufferHeader.limit() + this.getMessageResult.getBufferTotalSize();
@@ -76,6 +81,28 @@ public class ManyMessageTransfer extends AbstractReferenceCounted implements Fil
return 0;
}
+ @Override
+ public FileRegion retain() {
+ super.retain();
+ return this;
+ }
+
+ @Override
+ public FileRegion retain(int increment) {
+ super.retain(increment);
+ return this;
+ }
+
+ @Override
+ public FileRegion touch() {
+ return this;
+ }
+
+ @Override
+ public FileRegion touch(Object hint) {
+ return this;
+ }
+
public void close() {
this.deallocate();
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java b/broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java
index b795d2d253..558f091763 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java
@@ -47,6 +47,11 @@ public class OneMessageTransfer extends AbstractReferenceCounted implements File
return transferred;
}
+ @Override
+ public long transferred() {
+ return transferred;
+ }
+
@Override
public long count() {
return this.byteBufferHeader.limit() + this.selectMappedBufferResult.getSize();
@@ -65,6 +70,28 @@ public class OneMessageTransfer extends AbstractReferenceCounted implements File
return 0;
}
+ @Override
+ public FileRegion retain() {
+ super.retain();
+ return this;
+ }
+
+ @Override
+ public FileRegion retain(int increment) {
+ super.retain(increment);
+ return this;
+ }
+
+ @Override
+ public FileRegion touch() {
+ return this;
+ }
+
+ @Override
+ public FileRegion touch(Object hint) {
+ return this;
+ }
+
public void close() {
this.deallocate();
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransfer.java b/broker/src/main/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransfer.java
index e8f3099667..db47b9e50e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransfer.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransfer.java
@@ -53,6 +53,11 @@ public class QueryMessageTransfer extends AbstractReferenceCounted implements Fi
return transferred;
}
+ @Override
+ public long transferred() {
+ return transferred;
+ }
+
@Override
public long count() {
return byteBufferHeader.limit() + this.queryMessageResult.getBufferTotalSize();
@@ -76,6 +81,28 @@ public class QueryMessageTransfer extends AbstractReferenceCounted implements Fi
return 0;
}
+ @Override
+ public FileRegion retain() {
+ super.retain();
+ return this;
+ }
+
+ @Override
+ public FileRegion retain(int increment) {
+ super.retain(increment);
+ return this;
+ }
+
+ @Override
+ public FileRegion touch() {
+ return this;
+ }
+
+ @Override
+ public FileRegion touch(Object hint) {
+ return this;
+ }
+
public void close() {
this.deallocate();
}
diff --git a/pom.xml b/pom.xml
index b6f1770e2d..59fe5c8072 100644
--- a/pom.xml
+++ b/pom.xml
@@ -537,7 +537,7 @@
io.netty
netty-all
- 4.0.42.Final
+ 4.1.65.Final
com.alibaba
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java
index 4b4e86e687..1a0c9b53e3 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java
@@ -52,6 +52,8 @@ public class NettyLogger {
private InternalLogger logger = null;
+ private static final String EXCEPTION_MESSAGE = "Unexpected exception:";
+
public NettyBridgeLogger(String name) {
logger = InternalLoggerFactory.getLogger(name);
}
@@ -161,6 +163,25 @@ public class NettyLogger {
}
}
+ @Override
+ public void log(InternalLogLevel internalLogLevel, Throwable throwable) {
+ if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {
+ logger.debug(EXCEPTION_MESSAGE, throwable);
+ }
+ if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
+ logger.info(EXCEPTION_MESSAGE, throwable);
+ }
+ if (internalLogLevel.equals(InternalLogLevel.INFO)) {
+ logger.info(EXCEPTION_MESSAGE, throwable);
+ }
+ if (internalLogLevel.equals(InternalLogLevel.WARN)) {
+ logger.warn(EXCEPTION_MESSAGE, throwable);
+ }
+ if (internalLogLevel.equals(InternalLogLevel.ERROR)) {
+ logger.error(EXCEPTION_MESSAGE, throwable);
+ }
+ }
+
@Override
public boolean isTraceEnabled() {
return isEnabled(InternalLogLevel.TRACE);
@@ -191,6 +212,11 @@ public class NettyLogger {
logger.info(var1, var2);
}
+ @Override
+ public void trace(Throwable var1) {
+ logger.info(EXCEPTION_MESSAGE, var1);
+ }
+
@Override
public boolean isDebugEnabled() {
return isEnabled(InternalLogLevel.DEBUG);
@@ -221,6 +247,11 @@ public class NettyLogger {
logger.debug(var1, var2);
}
+ @Override
+ public void debug(Throwable var1) {
+ logger.debug(EXCEPTION_MESSAGE, var1);
+ }
+
@Override
public boolean isInfoEnabled() {
return isEnabled(InternalLogLevel.INFO);
@@ -251,6 +282,11 @@ public class NettyLogger {
logger.info(var1, var2);
}
+ @Override
+ public void info(Throwable var1) {
+ logger.info(EXCEPTION_MESSAGE, var1);
+ }
+
@Override
public boolean isWarnEnabled() {
return isEnabled(InternalLogLevel.WARN);
@@ -281,6 +317,11 @@ public class NettyLogger {
logger.warn(var1, var2);
}
+ @Override
+ public void warn(Throwable var1) {
+ logger.warn(EXCEPTION_MESSAGE, var1);
+ }
+
@Override
public boolean isErrorEnabled() {
return isEnabled(InternalLogLevel.ERROR);
@@ -310,6 +351,11 @@ public class NettyLogger {
public void error(String var1, Throwable var2) {
logger.error(var1, var2);
}
+
+ @Override
+ public void error(Throwable var1) {
+ logger.error(EXCEPTION_MESSAGE, var1);
+ }
}
}
From 278d3285bd7c5d2563a98533a0bac593504a8576 Mon Sep 17 00:00:00 2001
From: CuCu <503598058@qq.com>
Date: Wed, 23 Jun 2021 14:41:59 +0800
Subject: [PATCH 28/83] add Chinese version
---
docs/cn/Example_Delay.md | 85 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 85 insertions(+)
create mode 100644 docs/cn/Example_Delay.md
diff --git a/docs/cn/Example_Delay.md b/docs/cn/Example_Delay.md
new file mode 100644
index 0000000000..31df40f448
--- /dev/null
+++ b/docs/cn/Example_Delay.md
@@ -0,0 +1,85 @@
+# Schedule example
+
+### 1 启动消费者等待传入的订阅消息
+
+```java
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.message.MessageExt;
+import java.util.List;
+
+public class ScheduledMessageConsumer {
+
+ public static void main(String[] args) throws Exception {
+ // Instantiate message consumer
+ DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ExampleConsumer");
+ // Subscribe topics
+ consumer.subscribe("TestTopic", "*");
+ // Register message listener
+ consumer.registerMessageListener(new MessageListenerConcurrently() {
+ @Override
+ public ConsumeConcurrentlyStatus consumeMessage(List messages, ConsumeConcurrentlyContext context) {
+ for (MessageExt message : messages) {
+ // Print approximate delay time period
+ System.out.println("Receive message[msgId=" + message.getMsgId() + "] "
+ + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later");
+ }
+ return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+ }
+ });
+ // Launch consumer
+ consumer.start();
+ }
+}
+```
+
+### 2 发送延迟消息
+
+```java
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.message.Message;
+
+public class ScheduledMessageProducer {
+
+ public static void main(String[] args) throws Exception {
+ // Instantiate a producer to send scheduled messages
+ DefaultMQProducer producer = new DefaultMQProducer("ExampleProducerGroup");
+ // Launch producer
+ producer.start();
+ int totalMessagesToSend = 100;
+ for (int i = 0; i < totalMessagesToSend; i++) {
+ Message message = new Message("TestTopic", ("Hello scheduled message " + i).getBytes());
+ // This message will be delivered to consumer 10 seconds later.
+ message.setDelayTimeLevel(3);
+ // Send the message
+ producer.send(message);
+ }
+
+ // Shutdown producer after use.
+ producer.shutdown();
+ }
+
+}
+```
+
+### 3 确认
+
+您应该会看到消息在其存储时间后大约 10 秒被消耗。
+
+### 4 延迟消息的使用场景
+
+例如在电子商务中,如果提交订单,可以发送延迟消息,1小时后可以查看订单状态。 如果订单仍未付款,则可以取消订单并释放库存。
+
+### 5 使用延迟消息的限制
+
+```java
+// org/apache/rocketmq/store/config/MessageStoreConfig.java
+
+private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h";
+```
+
+当前 RocketMQ 不支持任意时间的延迟。 生产者发送延迟消息前需要设置几个固定的延迟级别,分别对应1s到2h的1到18个延迟级,消息消费失败会进入延迟消息队列,消息发送时间与设置的延迟级别和重试次数有关。
+
+ See `SendMessageProcessor.java`
From 23f3893787f1e738fefc5babaf0f9e6302829e44 Mon Sep 17 00:00:00 2001
From: oumengfan
Date: Wed, 23 Jun 2021 16:00:01 +0800
Subject: [PATCH 29/83] Add a Chinese version of Deployment.md
---
docs/cn/Deployment.md | 159 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 159 insertions(+)
create mode 100644 docs/cn/Deployment.md
diff --git a/docs/cn/Deployment.md b/docs/cn/Deployment.md
new file mode 100644
index 0000000000..1f00986b4b
--- /dev/null
+++ b/docs/cn/Deployment.md
@@ -0,0 +1,159 @@
+# 部署架构和设置步骤
+
+## 集群的设置
+
+### 1 单master模式
+
+这是最简单但也是最危险的模式,一旦broker服务器重启或宕机,整个服务将不可用。 建议在生产环境中不要使用这种部署方式,在本地测试和开发可以选择这种模式。 以下是构建的步骤。
+
+**1)启动NameServer**
+
+```shell
+### 第一步启动namesrv
+$ nohup sh mqnamesrv &
+
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+我们可以在namesrv.log 中看到'The Name Server boot success..',表示NameServer 已成功启动。
+
+**2)启动Broker**
+
+```shell
+### 第一步先启动broker
+$ nohup sh bin/mqbroker -n localhost:9876 &
+
+### 验证broker是否启动成功, 比如, broker的ip是192.168.1.2 然后名字是broker-a
+$ tail -f ~/logs/rocketmqlogs/Broker.log
+The broker[broker-a,192.169.1.2:10911] boot success...
+```
+
+我们可以在 Broker.log 中看到“The broker[brokerName,ip:port] boot success..”,这表明 broker 已成功启动。
+
+### 2 多Master模式
+
+该模式是指所有节点都是master主节点(比如2个或3个主节点),没有slave从节点的模式。 这种模式的优缺点如下:
+
+- 优点:
+ 1. 配置简单。
+ 2. 一个master节点的宕机或者重启(维护)对应用程序没有影响。
+ 3. 当磁盘配置为RAID10时,消息不会丢失,因为RAID10磁盘非常可靠,即使机器不可恢复(消息异步刷盘模式的情况下,会丢失少量消息;如果消息是同步刷盘模式,不会丢失任何消息)。
+ 4. 在这种模式下,性能是最高的。
+- 缺点:
+ 1. 单台机器宕机时,本机未消费的消息,直到机器恢复后才会订阅,影响消息实时性。
+
+多Master模式的启动步骤如下:
+
+**1)启动 NameServer**
+
+```shell
+### 第一步先启动broker
+$ nohup sh mqnamesrv &
+
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+**2)启动 Broker 集群**
+
+```shell
+### 比如在A机器上启动第一个Master,假设配置的NameServer IP为:192.168.1.1
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties &
+
+### 然后在机器B上启动第二个Master,假设配置的NameServer IP是:192.168.1.1
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties &
+
+...
+```
+
+上面显示的boot命令用于单个NameServer的情况。对于多个NameServer的集群,broker boot命令中-n参数后面的地址列表用分号隔开,例如 192.168.1.1 : 9876; 192.161.2 : 9876
+
+### 3 多Master多Slave模式-异步复制
+
+每个主节点配置多个从节点,多对主从。HA采用异步复制,主节点和从节点之间有短消息延迟(毫秒)。这种模式的优缺点如下:
+
+- 优点:
+ 1. 即使磁盘损坏,也不会丢失极少的消息,不影响消息的实时性能。
+ 2. 同时,当主节点宕机时,消费者仍然可以消费从节点的消息,这个过程对应用本身是透明的,不需要人为干预。
+ 3. 性能几乎与多Master模式一样高。
+- 缺点:
+ 1. 主节点宕机、磁盘损坏时,会丢失少量消息。
+
+多主多从模式的启动步骤如下:
+
+**1)启动 NameServer**
+
+```shell
+### 第一步先启动broker
+$ nohup sh mqnamesrv &
+
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+**2)启动 Broker 集群**
+
+```shell
+### 例如在A机器上启动第一个Master,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties &
+
+### 然后在机器B上启动第二个Master,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties &
+
+### 然后在C机器上启动第一个Slave,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties &
+
+### 最后在D机启动第二个Slave,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties &
+```
+
+上图显示了 2M-2S-Async 模式的启动命令,类似于其他 nM-nS-Async 模式。
+
+### 4 多Master多Slave模式-同步双写
+
+这种模式下,每个master节点配置多个slave节点,有多对Master-Slave。HA采用同步双写,即只有消息成功写入到主节点并复制到多个从节点,才会返回成功响应给应用程序。
+
+这种模式的优缺点如下:
+
+- 优点:
+ 1. 数据和服务都没有单点故障。
+ 2. 在master节点关闭的情况下,消息也没有延迟。
+ 3. 服务可用性和数据可用性非常高;
+- 缺点:
+ 1. 这种模式下的性能略低于异步复制模式(大约低 10%)。
+ 2. 发送单条消息的RT略高,目前版本,master节点宕机后,slave节点无法自动切换到master。
+
+启动步骤如下:
+
+**1)启动NameServer**
+
+```shell
+### 第一步启动broker
+$ nohup sh mqnamesrv &
+
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+**2)启动 Broker 集群**
+
+```shell
+### 例如在A机器上启动第一个Master,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties &
+
+### 然后在B机器上启动第二个Master,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties &
+
+### 然后在C机器上启动第一个Slave,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties &
+
+### 最后在D机启动第二个Slave,假设配置的NameServer IP为:192.168.1.1,端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties &
+```
+
+上述Master和Slave是通过指定相同的config命名为“brokerName”来配对的,master节点的brokerId必须为0,slave节点的brokerId必须大于0。
\ No newline at end of file
From 0cc5ad2d4fe9354507427cd078a5a3dceb7dde79 Mon Sep 17 00:00:00 2001
From: CuCu <503598058@qq.com>
Date: Wed, 23 Jun 2021 16:03:14 +0800
Subject: [PATCH 30/83] add Chinese version of Example_Batch
---
docs/cn/Example_Batch.md | 82 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 82 insertions(+)
create mode 100644 docs/cn/Example_Batch.md
diff --git a/docs/cn/Example_Batch.md b/docs/cn/Example_Batch.md
new file mode 100644
index 0000000000..6c8897fa6b
--- /dev/null
+++ b/docs/cn/Example_Batch.md
@@ -0,0 +1,82 @@
+# 批量消息发送
+批量消息发送能够提高发送效率,提升系统吞吐量。同一批批量消息的topic、waitStoreMsgOK属性必须保持一致,批量消息不支持延迟消息。批量消息发送一次最多可以发送 4MiB 的消息,但是如果需要发送更大的消息,建议将较大的消息分成多个不超过 1MiB 的小消息。
+
+### 1 发送批量消息
+如果你一次只发送不超过 4MiB 的消息,使用批处理很容易:
+```java
+String topic = "BatchTest";
+List messages = new ArrayList<>();
+messages.add(new Message(topic, "TagA", "OrderID001", "Hello world 0".getBytes()));
+messages.add(new Message(topic, "TagA", "OrderID002", "Hello world 1".getBytes()));
+messages.add(new Message(topic, "TagA", "OrderID003", "Hello world 2".getBytes()));
+try {
+ producer.send(messages);
+} catch (Exception e) {
+ e.printStackTrace();
+ //handle the error
+}
+```
+### 2 拆分
+当您发送较大的消息时,复杂性会增加,如果您不确定它是否超过 4MiB的限制。 这时候,您最好将较大的消息分成多个不超过 1MiB 的小消息:
+
+```java
+public class ListSplitter implements Iterator> {
+ private final int SIZE_LIMIT = 1024 * 1024 * 4;
+ private final List messages;
+ private int currIndex;
+ public ListSplitter(List messages) {
+ this.messages = messages;
+ }
+ @Override public boolean hasNext() {
+ return currIndex < messages.size();
+ }
+ @Override public List next() {
+ int startIndex = getStartIndex();
+ int nextIndex = startIndex;
+ int totalSize = 0;
+ for (; nextIndex < messages.size(); nextIndex++) {
+ Message message = messages.get(nextIndex);
+ int tmpSize = calcMessageSize(message);
+ if (tmpSize + totalSize > SIZE_LIMIT) {
+ break;
+ } else {
+ totalSize += tmpSize;
+ }
+ }
+ List subList = messages.subList(startIndex, nextIndex);
+ currIndex = nextIndex;
+ return subList;
+ }
+ private int getStartIndex() {
+ Message currMessage = messages.get(currIndex);
+ int tmpSize = calcMessageSize(currMessage);
+ while(tmpSize > SIZE_LIMIT) {
+ currIndex += 1;
+ Message message = messages.get(curIndex);
+ tmpSize = calcMessageSize(message);
+ }
+ return currIndex;
+ }
+ private int calcMessageSize(Message message) {
+ int tmpSize = message.getTopic().length() + message.getBody().length();
+ Map properties = message.getProperties();
+ for (Map.Entry entry : properties.entrySet()) {
+ tmpSize += entry.getKey().length() + entry.getValue().length();
+ }
+ tmpSize = tmpSize + 20; // Increase the log overhead by 20 bytes
+ return tmpSize;
+ }
+}
+
+// then you could split the large list into small ones:
+ListSplitter splitter = new ListSplitter(messages);
+while (splitter.hasNext()) {
+ try {
+ List listItem = splitter.next();
+ producer.send(listItem);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // handle the error
+ }
+}
+```
\ No newline at end of file
From a99cf37366eee9da34bc59a37db431a8fec6e43e Mon Sep 17 00:00:00 2001
From: 12425
Date: Wed, 23 Jun 2021 19:20:08 +0800
Subject: [PATCH 31/83] Add Chinses Version Doc of Example_Simple.md
---
docs/cn/Example_Simple_cn.md | 136 +++++++++++++++++++++++++++++++++++
1 file changed, 136 insertions(+)
create mode 100644 docs/cn/Example_Simple_cn.md
diff --git a/docs/cn/Example_Simple_cn.md b/docs/cn/Example_Simple_cn.md
new file mode 100644
index 0000000000..f0a2b6a1de
--- /dev/null
+++ b/docs/cn/Example_Simple_cn.md
@@ -0,0 +1,136 @@
+# Basic Sample
+------
+基本示例中提供了以下两个功能
+* RocketMQ可用于以三种方式发送消息:可靠的同步、可靠的异步和单向传输。前两种消息类型是可靠的,因为无论它们是否成功发送都有响应。
+* RocketMQ可以用来消费消息。
+### 1 添加依赖
+maven:
+``` java
+
+ org.apache.rocketmq
+ rocketmq-client
+ 4.3.0
+
+```
+gradle:
+``` java
+compile 'org.apache.rocketmq:rocketmq-client:4.3.0'
+```
+### 2 发送消息
+##### 2.1 使用Producer发送同步消息
+可靠的同步传输被广泛应用于各种场景,如重要的通知消息、短消息通知等。
+``` java
+public class SyncProducer {
+ public static void main(String[] args) throws Exception {
+ // Instantiate with a producer group name
+ DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+ // Specify name server addresses
+ producer.setNamesrvAddr("localhost:9876");
+ // Launch the producer instance
+ producer.start();
+ for (int i = 0; i < 100; i++) {
+ // Create a message instance with specifying topic, tag and message body
+ Message msg = new Message("TopicTest" /* Topic */,
+ "TagA" /* Tag */,
+ ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
+ );
+ // Send message to one of brokers
+ SendResult sendResult = producer.send(msg);
+ // Check whether the message has been delivered by the callback of sendResult
+ System.out.printf("%s%n", sendResult);
+ }
+ // Shut down once the producer instance is not longer in use
+ producer.shutdown();
+ }
+}
+```
+##### 2.2 发送异步消息
+异步传输通常用于响应时间敏感的业务场景。这意味着发送方无法等待代理的响应太长时间。
+``` java
+public class AsyncProducer {
+ public static void main(String[] args) throws Exception {
+ // Instantiate with a producer group name
+ DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+ // Specify name server addresses
+ producer.setNamesrvAddr("localhost:9876");
+ // Launch the producer instance
+ producer.start();
+ producer.setRetryTimesWhenSendAsyncFailed(0);
+ for (int i = 0; i < 100; i++) {
+ final int index = i;
+ // Create a message instance with specifying topic, tag and message body
+ Message msg = new Message("TopicTest",
+ "TagA",
+ "OrderID188",
+ "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+ // SendCallback: receive the callback of the asynchronous return result.
+ producer.send(msg, new SendCallback() {
+ @Override
+ public void onSuccess(SendResult sendResult) {
+ System.out.printf("%-10d OK %s %n", index,
+ sendResult.getMsgId());
+ }
+ @Override
+ public void onException(Throwable e) {
+ System.out.printf("%-10d Exception %s %n", index, e);
+ e.printStackTrace();
+ }
+ });
+ }
+ // Shut down once the producer instance is not longer in use
+ producer.shutdown();
+ }
+}
+```
+##### 2.3 以单向模式发送消息
+单向传输用于需要中等可靠性的情况,如日志收集。
+``` java
+public class OnewayProducer {
+ public static void main(String[] args) throws Exception{
+ // Instantiate with a producer group name
+ DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+ // Specify name server addresses
+ producer.setNamesrvAddr("localhost:9876");
+ // Launch the producer instance
+ producer.start();
+ for (int i = 0; i < 100; i++) {
+ // Create a message instance with specifying topic, tag and message body
+ Message msg = new Message("TopicTest" /* Topic */,
+ "TagA" /* Tag */,
+ ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
+ );
+ // Send in one-way mode, no return result
+ producer.sendOneway(msg);
+ }
+ // Shut down once the producer instance is not longer in use
+ producer.shutdown();
+ }
+}
+```
+### 3 消费消息
+``` java
+public class Consumer {
+ public static void main(String[] args) throws InterruptedException, MQClientException {
+ // Instantiate with specified consumer group name
+ DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name");
+
+ // Specify name server addresses
+ consumer.setNamesrvAddr("localhost:9876");
+
+ // Subscribe one or more topics and tags for finding those messages need to be consumed
+ consumer.subscribe("TopicTest", "*");
+ // Register callback to execute on arrival of messages fetched from brokers
+ consumer.registerMessageListener(new MessageListenerConcurrently() {
+ @Override
+ public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) {
+ System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+ // Mark the message that have been consumed successfully
+ return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+ }
+ });
+ // Launch the consumer instance
+ consumer.start();
+ System.out.printf("Consumer Started.%n");
+ }
+}
+```
\ No newline at end of file
From 61cd395d68db3cd5682060a75fe1ab6e56c0d82d Mon Sep 17 00:00:00 2001
From: qingjiyuji <1114366223@qq.com>
Date: Wed, 23 Jun 2021 20:22:03 +0800
Subject: [PATCH 32/83] code style(client):polish lite pull consumer code style
---
.../consumer/DefaultLitePullConsumerImpl.java | 27 ++++++++++++-------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
index 4e139c44ce..b1caeedee8 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
@@ -100,7 +100,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
private static final String NOT_RUNNING_EXCEPTION_MESSAGE = "The consumer not running, please start it first.";
- private static final String SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE = "Subscribe and assign are mutually exclusive.";
+ private static final String SUBSCRIPTION_CONFLICT_EXCEPTION_MESSAGE = "Subscribe and assign are mutually exclusive.";
/**
* the type of subscription
*/
@@ -195,8 +195,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
}
private void checkServiceState() {
- if (this.serviceState != ServiceState.RUNNING)
+ if (this.serviceState != ServiceState.RUNNING) {
throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE);
+ }
}
public void updateNameServerAddr(String newAddresses) {
@@ -204,10 +205,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
}
private synchronized void setSubscriptionType(SubscriptionType type) {
- if (this.subscriptionType == SubscriptionType.NONE)
+ if (this.subscriptionType == SubscriptionType.NONE) {
this.subscriptionType = type;
- else if (this.subscriptionType != type)
- throw new IllegalStateException(SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE);
+ } else if (this.subscriptionType != type) {
+ throw new IllegalStateException(SUBSCRIPTION_CONFLICT_EXCEPTION_MESSAGE);
+ }
}
private void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) {
@@ -533,8 +535,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
public synchronized List poll(long timeout) {
try {
checkServiceState();
- if (timeout < 0)
+ if (timeout < 0) {
throw new IllegalArgumentException("Timeout must not be negative");
+ }
if (defaultLitePullConsumer.isAutoCommit()) {
maybeAutoCommit();
@@ -546,8 +549,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
if (endTime - System.currentTimeMillis() > 0) {
while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) {
consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
- if (endTime - System.currentTimeMillis() <= 0)
+ if (endTime - System.currentTimeMillis() <= 0) {
break;
+ }
}
}
@@ -671,8 +675,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
public long committed(MessageQueue messageQueue) throws MQClientException {
checkServiceState();
long offset = this.offsetStore.readOffset(messageQueue, ReadOffsetType.MEMORY_FIRST_THEN_STORE);
- if (offset == -2)
+ if (offset == -2) {
throw new MQClientException("Fetch consume offset from broker exception", null);
+ }
return offset;
}
@@ -683,8 +688,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
}
Iterator iter = consumeRequestCache.iterator();
while (iter.hasNext()) {
- if (iter.next().getMessageQueue().equals(messageQueue))
+ if (iter.next().getMessageQueue().equals(messageQueue)) {
iter.remove();
+ }
}
}
@@ -737,8 +743,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {
scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS);
- if ((consumeRequestFlowControlTimes++ % 1000) == 0)
+ if ((consumeRequestFlowControlTimes++ % 1000) == 0) {
log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes);
+ }
return;
}
From 622bbbf05cf41d4cbc6e2f1b1783ed6c4f791476 Mon Sep 17 00:00:00 2001
From: zplkq <869371335@qq.com>
Date: Thu, 24 Jun 2021 00:26:03 +0800
Subject: [PATCH 33/83] add some doc of the API_Reference_DefaultMQProducer of
English.
---
.../java/API_Reference_DefaultMQProducer.md | 71 +++++++++++++++++++
1 file changed, 71 insertions(+)
create mode 100644 docs/en/client/java/API_Reference_DefaultMQProducer.md
diff --git a/docs/en/client/java/API_Reference_DefaultMQProducer.md b/docs/en/client/java/API_Reference_DefaultMQProducer.md
new file mode 100644
index 0000000000..152f45dca2
--- /dev/null
+++ b/docs/en/client/java/API_Reference_DefaultMQProducer.md
@@ -0,0 +1,71 @@
+## DefaultMQProducer
+---
+### Class introduction
+
+`public class DefaultMQProducer
+extends ClientConfig
+implements MQProducer`
+
+>`DefaultMQProducer` is the entry point for an application to post messages, out of the box,ca quickly create a producer with a no-argument construction. it is mainly responsible for message sending, support synchronous、asynchronous、one-way send. All of these send methods support batch send. The parameters of the sender can be adjusted through the getter/setter methods , provided by this class. `DefaultMQProducer` has multi send method and each method is slightly different. Make sure you know the usage before you use it . Blow is a producer example . [see more examples](https://github.com/apache/rocketmq/blob/master/example/src/main/java/org/apache/rocketmq/example/)。
+
+``` java
+public class Producer {
+ public static void main(String[] args) throws MQClientException {
+ // create a produce with producer_group_name
+ DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName");
+
+ // start the producer
+ producer.start();
+
+ for (int i = 0; i < 128; i++)
+ try {
+ // construct the msg
+ Message msg = new Message("TopicTest",
+ "TagA",
+ "OrderID188",
+ "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+
+ // send sync
+ SendResult sendResult = producer.send(msg);
+
+ // print the result
+ System.out.printf("%s%n", sendResult);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ producer.shutdown();
+ }
+}
+```
+
+**Note** : This class is thread safe. It can be safely shared between multiple threads after configuration and startup is complete.
+
+### Variable
+
+|Type|Name| description |
+|------|-------|-------|
+|DefaultMQProducerImpl|defaultMQProducerImpl|The producer's internal default implementation|
+|String|producerGroup|The producer's group|
+|String|createTopicKey| Topics that do not exist on the server are automatically created when the message is sent |
+|int|defaultTopicQueueNums|The default number of queues to create a topic|
+|int|sendMsgTimeout|The timeout for the message to be sent|
+|int|compressMsgBodyOverHowmuch|the threshold of the compress of message body|
+|int|retryTimesWhenSendFailed|Maximum number of internal attempts to send a message in synchronous mode|
+|int|retryTimesWhenSendAsyncFailed|Maximum number of internal attempts to send a message in asynchronous mode|
+|boolean|retryAnotherBrokerWhenNotStoreOK|Whether to retry another broker if an internal send fails|
+|int|maxMessageSize| Maximum length of message |
+|TraceDispatcher|traceDispatcher| Message trackers. Use rcpHook to track messages |
+
+### construction method
+
+|方法名称|方法描述|
+|-------|------------|
+|DefaultMQProducer()| creates a producer with default parameter values |
+|DefaultMQProducer(final String producerGroup)| creates a producer with producer group name. |
+|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace)|creates a producer with producer group name and set whether to enable message tracking|
+|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic)|creates a producer with producer group name and set whether to enable message tracking、the trace topic.|
+|DefaultMQProducer(RPCHook rpcHook)|creates a producer with a rpc hook.|
+|DefaultMQProducer(final String producerGroup, RPCHook rpcHook)|creates a producer with a rpc hook and producer group.|
+|DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic)|all of above.|
+
From 893e8fd8881a1acdc18020016450463d642baa59 Mon Sep 17 00:00:00 2001
From: qingjiyuji <1114366223@qq.com>
Date: Thu, 24 Jun 2021 02:11:06 +0800
Subject: [PATCH 34/83] code style(client):polish lite pull consumer code style
---
.../impl/consumer/DefaultLitePullConsumerImpl.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
index b1caeedee8..e835be1cd5 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
@@ -466,7 +466,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
public synchronized void subscribe(String topic, String subExpression) throws MQClientException {
try {
- if (topic == null || topic.equals("")) {
+ if (topic == null || "".equals(topic)) {
throw new IllegalArgumentException("Topic can not be null or empty.");
}
setSubscriptionType(SubscriptionType.SUBSCRIBE);
@@ -485,7 +485,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
public synchronized void subscribe(String topic, MessageSelector messageSelector) throws MQClientException {
try {
- if (topic == null || topic.equals("")) {
+ if (topic == null || "".equals(topic)) {
throw new IllegalArgumentException("Topic can not be null or empty.");
}
setSubscriptionType(SubscriptionType.SUBSCRIBE);
@@ -741,7 +741,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
return;
}
- if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {
+ if ((long) consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {
scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS);
if ((consumeRequestFlowControlTimes++ % 1000) == 0) {
log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes);
@@ -797,11 +797,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
long pullDelayTimeMills = 0;
try {
SubscriptionData subscriptionData;
+ String topic = this.messageQueue.getTopic();
if (subscriptionType == SubscriptionType.SUBSCRIBE) {
- String topic = this.messageQueue.getTopic();
subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic);
} else {
- String topic = this.messageQueue.getTopic();
subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL);
}
From d0d2f157dc7cf8c892dda63d1746672a14d73f57 Mon Sep 17 00:00:00 2001
From: patrick
Date: Thu, 24 Jun 2021 10:36:07 +0800
Subject: [PATCH 35/83] add chinese version of FAQ.md
---
docs/cn/FAQ.md | 110 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 110 insertions(+)
create mode 100644 docs/cn/FAQ.md
diff --git a/docs/cn/FAQ.md b/docs/cn/FAQ.md
new file mode 100644
index 0000000000..b58807926a
--- /dev/null
+++ b/docs/cn/FAQ.md
@@ -0,0 +1,110 @@
+# 经常被问到的问题
+
+以下是关于RocketMQ项目的常见问题
+
+## 1 基本
+
+1. **为什么我们要使用RocketMQ而不是选择其他的产品?**
+
+ 请参考[为什么要选择RocketMQ](http://rocketmq.apache.org/docs/motivation/)
+
+2. **我是否需要安装其他的软件才能使用RocketMQ,例如zookeeper?**
+
+ 不需要,RocketMQ可以独立的运行。
+
+## 2 使用
+
+1. **新创建的Consumer ID从哪里开始消费消息?**
+
+ 1)如果发送的消息在三天之内,那么消费者会从服务器中保存的第一条消息开始消费。
+
+ 2)如果发送的消息已经超过三天,则消费者会从服务器中的最新消息开始消费,也就是从队列的尾部开始消费。
+
+ 3)如果消费者重新启动,那么它会从最后一个消费位置开始消费消息。
+
+2. **当消费失败的时候如何重新消费消息?**
+
+ 1)在集群模式下,消费的业务逻辑代码会返回Action.ReconsumerLater,NULL,或者抛出异常,如果一条消息消费失败,最多会重试16次,之后该消息会被丢弃。
+
+ 2)在广播消费模式下,广播消费仍然保证消息至少被消费一次,但不提供重发的选项。
+
+3. **当消费失败的时候如何找到失败的消息?**
+
+ 1)使用按时间的主题查询,可以查询到一段时间内的消息。
+
+ 2)使用主题和消息ID来准确查询消息。
+
+ 3)使用主题和消息的Key来准确查询所有消息Key相同的消息。
+
+4. **消息只会被传递一次吗?**
+
+ RocketMQ 确保所有消息至少传递一次。 在大多数情况下,消息不会重复。
+
+5. **如何增加一个新的Broker?**
+
+ 1)启动一个新的Broker并将其注册到name server中的Broker列表里。
+
+ 2)默认只自动创建内部系统topic和consumer group。 如果您希望在新节点上拥有您的业务主题和消费者组,请从现有的Broker中复制它们。 我们提供了管理工具和命令行来处理此问题。
+
+## 3 配置相关
+
+以下回答均为默认值,可通过配置修改。
+
+1. **消息在服务器上可以保存多长时间?**
+
+ 存储的消息将最多保存 3 天,超过 3 天未使用的消息将被删除。
+
+2. **消息体的大小限制是多少?**
+
+ 通常是256KB
+
+3. **怎么设置消费者线程数?**
+
+ 当你启动消费者的时候,可以设置 ConsumeThreadNums属性的值,举例如下:
+
+ ```java
+ consumer.setConsumeThreadMin(20);
+ consumer.setConsumeThreadMax(20);
+ ```
+
+## 4 错误
+
+1. **当你启动一个生产者或消费者的过程失败了并且错误信息是生产者组或消费者重复**
+
+ 原因:使用同一个Producer/Consumer Group在同一个JVM中启动多个Producer/Consumer实例可能会导致客户端无法启动。
+
+ 解决方案:确保一个 Producer/Consumer Group 对应的 JVM 只启动一个 Producer/Consumer 实例。
+
+2. **消费者无法在广播模式下开始加载 json 文件**
+
+ 原因:fastjson 版本太低,无法让广播消费者加载本地 offsets.json,导致消费者启动失败。 损坏的 fastjson 文件也会导致同样的问题。
+
+ 解决方案:Fastjson 版本必须升级到 RocketMQ 客户端依赖版本,以确保可以加载本地 offsets.json。 默认情况下,offsets.json 文件在 /home/{user}/.rocketmq_offsets 中。 或者检查fastjson的完整性。
+
+3. **Broker崩溃以后有什么影响?**
+
+ 1)Master节点崩溃
+
+ 消息不能再发送到该Broker集群,但是如果您有另一个可用的Broker集群,那么在主题存在的条件下仍然可以发送消息。消息仍然可以从Slave节点消费。
+
+ 2)一些Slave节点崩溃
+
+ 只要有另一个工作的slave,就不会影响发送消息。 对消费消息也不会产生影响,除非消费者组设置为优先从该Slave消费。 默认情况下,消费者组从 master 消费。
+
+ 3)所有Slave节点崩溃
+
+ 向master发送消息不会有任何影响,但是,如果master是SYNC_MASTER,producer会得到一个SLAVE_NOT_AVAILABLE,表示消息没有发送给任何slave。 对消费消息也没有影响,除非消费者组设置为优先从slave消费。 默认情况下,消费者组从master消费。
+
+4. **Producer提示“No Topic Route Info”,如何诊断?**
+
+ 当您尝试将消息发送到一个路由信息对生产者不可用的主题时,就会发生这种情况。
+
+ 1)确保生产者可以连接到名称服务器并且能够从中获取路由元信息。
+
+ 2)确保名称服务器确实包含主题的路由元信息。 您可以使用管理工具或 Web 控制台通过 topicRoute 从名称服务器查询路由元信息。
+
+ 3)确保您的Broker将心跳发送到您的生产者正在连接的同一name server列表。
+
+ 4)确保主题的权限为6(rw-),或至少为2(-w-)。
+
+ 如果找不到此主题,请通过管理工具命令updateTopic或Web控制台在Broker上创建它。
\ No newline at end of file
From a0a2192c6517e350be22c3cbf4b6d2cd9c74a976 Mon Sep 17 00:00:00 2001
From: Git_Yang <30995057+Git-Yang@users.noreply.github.com>
Date: Fri, 25 Jun 2021 10:28:20 +0800
Subject: [PATCH 36/83] [ISSUE #2873] add benchmark for batch message
Signed-off-by: zhangyang
---
distribution/benchmark/batchproducer.sh | 18 +
.../example/benchmark/BatchProducer.java | 411 ++++++++++++++++++
2 files changed, 429 insertions(+)
create mode 100644 distribution/benchmark/batchproducer.sh
create mode 100644 example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java
diff --git a/distribution/benchmark/batchproducer.sh b/distribution/benchmark/batchproducer.sh
new file mode 100644
index 0000000000..07297004e6
--- /dev/null
+++ b/distribution/benchmark/batchproducer.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sh ./runclass.sh org.apache.rocketmq.example.benchmark.BatchProducer $@ &
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java
new file mode 100644
index 0000000000..843b84bf8e
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.benchmark;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.srvutil.ServerUtil;
+
+public class BatchProducer {
+
+ public static void main(String[] args) throws MQClientException, UnsupportedEncodingException {
+
+ Options options = ServerUtil.buildCommandlineOptions(new Options());
+ CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkBatchProducer", args, buildCommandlineOptions(options), new PosixParser());
+ if (null == commandLine) {
+ System.exit(-1);
+ }
+
+ final String namesrv = getOptionValue(commandLine, 'n', "127.0.0.1:9876");
+ final String topic = getOptionValue(commandLine, 't', "BenchmarkTest");
+ final int threadCount = getOptionValue(commandLine, 'w', 64);
+ final int messageSize = getOptionValue(commandLine, 's', 128);
+ final int batchSize = getOptionValue(commandLine, 'b', 16);
+ final boolean keyEnable = getOptionValue(commandLine, 'k', false);
+ final int propertySize = getOptionValue(commandLine, 'p', 0);
+ final int tagCount = getOptionValue(commandLine, 'l', 0);
+ final boolean msgTraceEnable = getOptionValue(commandLine, 'm', false);
+ final boolean aclEnable = getOptionValue(commandLine, 'a', false);
+ final String ak = getOptionValue(commandLine, 'c', "rocketmq2");
+ final String sk = getOptionValue(commandLine, 'e', "12346789");
+
+ System.out.printf("topic: %s threadCount: %d messageSize: %d batchSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s%n",
+ topic, threadCount, messageSize, batchSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable);
+
+ final StatsBenchmarkBatchProducer statsBenchmark = new StatsBenchmarkBatchProducer();
+ statsBenchmark.start();
+
+ final DefaultMQProducer producer = initInstance(namesrv, msgTraceEnable, aclEnable, ak, sk);
+ producer.start();
+
+ final InternalLogger log = ClientLogger.getLog();
+ final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount);
+ for (int i = 0; i < threadCount; i++) {
+ sendThreadPool.execute(new Runnable() {
+ @Override
+ public void run() {
+ while (true) {
+ List msgs;
+
+ try {
+ msgs = buildBathMessage(batchSize, messageSize, topic);
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ return;
+ }
+
+ if (CollectionUtils.isEmpty(msgs)) {
+ return;
+ }
+
+ try {
+ long beginTimestamp = System.currentTimeMillis();
+ long sendSucCount = statsBenchmark.getSendMessageSuccessCount().get();
+
+ setKeys(keyEnable, msgs, String.valueOf(beginTimestamp / 1000));
+ setTags(tagCount, msgs, sendSucCount);
+ setProperties(propertySize, msgs);
+ SendResult sendResult = producer.send(msgs);
+ if (sendResult.getSendStatus() == SendStatus.SEND_OK) {
+ statsBenchmark.getSendRequestSuccessCount().incrementAndGet();
+ statsBenchmark.getSendMessageSuccessCount().addAndGet(msgs.size());
+ } else {
+ statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+ statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+ }
+ long currentRT = System.currentTimeMillis() - beginTimestamp;
+ statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT);
+ long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+ while (currentRT > prevMaxRT) {
+ boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT);
+ if (updated) {
+ break;
+ }
+
+ prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+ }
+ } catch (RemotingException e) {
+ statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+ statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+ log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException ignored) {
+ }
+ } catch (InterruptedException e) {
+ statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+ statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e1) {
+ }
+ statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+ statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+ log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+ } catch (MQClientException e) {
+ statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+ statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+ log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+ } catch (MQBrokerException e) {
+ statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+ statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+ log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException ignored) {
+ }
+ }
+ }
+ }
+ });
+ }
+ }
+
+ public static Options buildCommandlineOptions(final Options options) {
+ Option opt = new Option("w", "threadCount", true, "Thread count, Default: 64");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("s", "messageSize", true, "Message Size, Default: 128");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("b", "batchSize", true, "Batch Size, Default: 16");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("k", "keyEnable", true, "Message Key Enable, Default: false");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("l", "tagCount", true, "Tag count, Default: 0");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("a", "aclEnable", true, "Acl Enable, Default: false");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("c", "accessKey", true, "Acl Access Key, Default: rocketmq2");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("e", "secretKey", true, "Acl Secret Key, Default: 123456789");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("p", "propertySize", true, "Property Size, Default: 0");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option("n", "namesrv", true, "name server, Default: 127.0.0.1:9876");
+ opt.setRequired(false);
+ options.addOption(opt);
+ return options;
+ }
+
+ private static String getOptionValue(CommandLine commandLine, char key, String defaultValue) {
+ if (commandLine.hasOption(key)) {
+ return commandLine.getOptionValue(key).trim();
+ }
+ return defaultValue;
+ }
+
+ private static int getOptionValue(CommandLine commandLine, char key, int defaultValue) {
+ if (commandLine.hasOption(key)) {
+ return Integer.parseInt(commandLine.getOptionValue(key).trim());
+ }
+ return defaultValue;
+ }
+
+ private static boolean getOptionValue(CommandLine commandLine, char key, boolean defaultValue) {
+ if (commandLine.hasOption(key)) {
+ return Boolean.parseBoolean(commandLine.getOptionValue(key).trim());
+ }
+ return defaultValue;
+ }
+
+ private static List buildBathMessage(int batchSize, int messageSize,
+ String topic) throws UnsupportedEncodingException {
+ List batchMessage = new ArrayList<>(batchSize);
+
+ for (int i = 0; i < batchSize; i++) {
+ Message msg = new Message();
+ msg.setTopic(topic);
+
+ StringBuilder sb = new StringBuilder();
+ for (int j = 0; j < messageSize; j += 10) {
+ sb.append("hello baby");
+ }
+
+ msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET));
+ batchMessage.add(msg);
+ }
+
+ return batchMessage;
+ }
+
+ private static void setKeys(boolean keyEnable, List msgs, String keys) {
+ if (!keyEnable) {
+ return;
+ }
+
+ for (Message msg : msgs) {
+ msg.setKeys(keys);
+ }
+ }
+
+ private static void setTags(int tagCount, List msgs, long startTagId) {
+ if (tagCount <= 0) {
+ return;
+ }
+
+ long tagId = startTagId % tagCount;
+ for (Message msg : msgs) {
+ msg.setTags(String.format("tag%d", tagId++));
+ }
+ }
+
+ private static void setProperties(int propertySize, List msgs) {
+ if (propertySize <= 0) {
+ return;
+ }
+
+ for (Message msg : msgs) {
+ if (msg.getProperties() != null) {
+ msg.getProperties().clear();
+ }
+
+ int startValue = (new Random(System.currentTimeMillis())).nextInt(100);
+ int size = 0;
+ for (int i = 0; ; i++) {
+ String prop1 = "prop" + i, prop1V = "hello" + startValue;
+ msg.putUserProperty(prop1, prop1V);
+ size += prop1.length() + prop1V.length();
+ if (size > propertySize) {
+ break;
+ }
+ startValue++;
+ }
+ }
+ }
+
+ private static DefaultMQProducer initInstance(String namesrv, boolean traceEnable, boolean aclEnable, String ak,
+ String sk) {
+ RPCHook rpcHook = aclEnable ? new AclClientRPCHook(new SessionCredentials(ak, sk)) : null;
+ final DefaultMQProducer producer = new DefaultMQProducer("benchmark_batch_producer", rpcHook, traceEnable, null);
+ producer.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+ producer.setNamesrvAddr(namesrv);
+ producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);
+ return producer;
+ }
+}
+
+class StatsBenchmarkBatchProducer {
+
+ private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L);
+
+ private final AtomicLong sendRequestFailedCount = new AtomicLong(0L);
+
+ private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L);
+
+ private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);
+
+ private final AtomicLong sendMessageSuccessCount = new AtomicLong(0L);
+
+ private final AtomicLong sendMessageFailedCount = new AtomicLong(0L);
+
+ private final Timer timer = new Timer("BenchmarkTimerThread", true);
+
+ private final LinkedList snapshotList = new LinkedList<>();
+
+ public Long[] createSnapshot() {
+ Long[] snap = new Long[] {
+ System.currentTimeMillis(),
+ this.sendRequestSuccessCount.get(),
+ this.sendRequestFailedCount.get(),
+ this.sendMessageSuccessCount.get(),
+ this.sendMessageFailedCount.get(),
+ this.sendMessageSuccessTimeTotal.get(),
+ };
+
+ return snap;
+ }
+
+ public AtomicLong getSendRequestSuccessCount() {
+ return sendRequestSuccessCount;
+ }
+
+ public AtomicLong getSendRequestFailedCount() {
+ return sendRequestFailedCount;
+ }
+
+ public AtomicLong getSendMessageSuccessTimeTotal() {
+ return sendMessageSuccessTimeTotal;
+ }
+
+ public AtomicLong getSendMessageMaxRT() {
+ return sendMessageMaxRT;
+ }
+
+ public AtomicLong getSendMessageSuccessCount() {
+ return sendMessageSuccessCount;
+ }
+
+ public AtomicLong getSendMessageFailedCount() {
+ return sendMessageFailedCount;
+ }
+
+ public void start() {
+
+ timer.scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ snapshotList.addLast(createSnapshot());
+ if (snapshotList.size() > 10) {
+ snapshotList.removeFirst();
+ }
+ }
+ }, 1000, 1000);
+
+ timer.scheduleAtFixedRate(new TimerTask() {
+ private void printStats() {
+ if (snapshotList.size() >= 10) {
+ Long[] begin = snapshotList.getFirst();
+ Long[] end = snapshotList.getLast();
+
+ final long sendTps = (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L);
+ final long sendMps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L);
+ final double averageRT = (end[5] - begin[5]) / (double) (end[1] - begin[1]);
+ final double averageMsgRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]);
+
+ System.out.printf("Current Time: %s Send TPS: %d Send MPS: %d Max RT(ms): %d Average RT(ms): %7.3f Average Message RT(ms): %7.3f Send Failed: %d Send Message Failed: %d%n",
+ System.currentTimeMillis(), sendTps, sendMps, getSendMessageMaxRT().get(), averageRT, averageMsgRT, end[2], end[4]);
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ this.printStats();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }, 10000, 10000);
+ }
+
+ public void shutdown() {
+ timer.cancel();
+ }
+}
\ No newline at end of file
From 6b519ef1df5852cc12981369cc8d57718f6dea3a Mon Sep 17 00:00:00 2001
From: yuz10 <845238369@qq.com>
Date: Fri, 25 Jun 2021 10:32:26 +0800
Subject: [PATCH 37/83] [ISSUE #3068] remove jaeger tracing dependency in
release package
---
distribution/release.xml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/distribution/release.xml b/distribution/release.xml
index 64cffc81ea..fd6e3db942 100644
--- a/distribution/release.xml
+++ b/distribution/release.xml
@@ -74,6 +74,10 @@
lib/
+
+ io.jaegertracing:jaeger-core
+ io.jaegertracing:jaeger-client
+
From a2f8810c9adedcd82fd4cb9a69b17128a1a96b5e Mon Sep 17 00:00:00 2001
From: yuz10 <845238369@qq.com>
Date: Fri, 25 Jun 2021 10:47:48 +0800
Subject: [PATCH 38/83] [ISSUE #2988] fix fail to send trace of last message
before shutting down producer
---
.../client/trace/AsyncTraceDispatcher.java | 70 ++++++++++---------
1 file changed, 37 insertions(+), 33 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
index 754cbd55ac..7ff8bd77e0 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
@@ -16,7 +16,6 @@
*/
package org.apache.rocketmq.client.trace;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,6 +29,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.common.ThreadLocalIndex;
@@ -64,7 +64,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
// The last discard number of log
private AtomicLong discardCount;
private Thread worker;
- private ArrayBlockingQueue traceContextQueue;
+ private final ArrayBlockingQueue traceContextQueue;
private ArrayBlockingQueue appenderQueue;
private volatile Thread shutDownHook;
private volatile boolean stopped = false;
@@ -78,7 +78,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
private String group;
private Type type;
- public AsyncTraceDispatcher(String group, Type type,String traceTopicName, RPCHook rpcHook) {
+ public AsyncTraceDispatcher(String group, Type type, String traceTopicName, RPCHook rpcHook) {
// queueSize is greater than or equal to the n power of 2 of value
this.queueSize = 2048;
this.batchSize = 100;
@@ -95,12 +95,12 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
this.traceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC;
}
this.traceExecutor = new ThreadPoolExecutor(//
- 10, //
- 20, //
- 1000 * 60, //
- TimeUnit.MILLISECONDS, //
- this.appenderQueue, //
- new ThreadFactoryImpl("MQTraceSendThread_"));
+ 10, //
+ 20, //
+ 1000 * 60, //
+ TimeUnit.MILLISECONDS, //
+ this.appenderQueue, //
+ new ThreadFactoryImpl("MQTraceSendThread_"));
traceProducer = getAndCreateTraceProducer(rpcHook);
}
@@ -180,10 +180,15 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
}
@Override
- public void flush() throws IOException {
+ public void flush() {
// The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return.
long end = System.currentTimeMillis() + 500;
- while (traceContextQueue.size() > 0 || appenderQueue.size() > 0 && System.currentTimeMillis() <= end) {
+ while (System.currentTimeMillis() <= end) {
+ synchronized (traceContextQueue) {
+ if (traceContextQueue.size() == 0 && appenderQueue.size() == 0) {
+ break;
+ }
+ }
try {
Thread.sleep(1);
} catch (InterruptedException e) {
@@ -196,6 +201,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
@Override
public void shutdown() {
this.stopped = true;
+ flush();
this.traceExecutor.shutdown();
if (isStarted.get()) {
traceProducer.shutdown();
@@ -212,11 +218,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
public void run() {
synchronized (this) {
if (!this.hasShutdown) {
- try {
- flush();
- } catch (IOException e) {
- log.error("system MQTrace hook shutdown failed ,maybe loss some trace data");
- }
+ flush();
}
}
}
@@ -242,25 +244,27 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
public void run() {
while (!stopped) {
List contexts = new ArrayList(batchSize);
- for (int i = 0; i < batchSize; i++) {
- TraceContext context = null;
- try {
- //get trace data element from blocking Queue — traceContextQueue
- context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
+ synchronized (traceContextQueue) {
+ for (int i = 0; i < batchSize; i++) {
+ TraceContext context = null;
+ try {
+ //get trace data element from blocking Queue - traceContextQueue
+ context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ }
+ if (context != null) {
+ contexts.add(context);
+ } else {
+ break;
+ }
}
- if (context != null) {
- contexts.add(context);
- } else {
- break;
+ if (contexts.size() > 0) {
+ AsyncAppenderRequest request = new AsyncAppenderRequest(contexts);
+ traceExecutor.submit(request);
+ } else if (AsyncTraceDispatcher.this.stopped) {
+ this.stopped = true;
}
}
- if (contexts.size() > 0) {
- AsyncAppenderRequest request = new AsyncAppenderRequest(contexts);
- traceExecutor.submit(request);
- } else if (AsyncTraceDispatcher.this.stopped) {
- this.stopped = true;
- }
}
}
@@ -352,7 +356,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
* Send message trace data
*
* @param keySet the keyset in this batch(including msgId in original message not offsetMsgId)
- * @param data the message trace data in this batch
+ * @param data the message trace data in this batch
*/
private void sendTraceDataByMQ(Set keySet, final String data, String dataTopic, String regionId) {
String traceTopic = traceTopicName;
From fcc20dee22d92d4cf0cdba5db21e0de1663478a5 Mon Sep 17 00:00:00 2001
From: yumingyue98 <70364204+yumingyue98@users.noreply.github.com>
Date: Sat, 26 Jun 2021 14:34:53 +0800
Subject: [PATCH 39/83] [ISSUE #3059] typo transactional message wiki
Co-authored-by: yumingyue98
---
docs/en/Example_Transaction.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/en/Example_Transaction.md b/docs/en/Example_Transaction.md
index a35b040f82..5d1cff9320 100644
--- a/docs/en/Example_Transaction.md
+++ b/docs/en/Example_Transaction.md
@@ -89,7 +89,7 @@ public class TransactionListenerImpl implements TransactionListener {
## 3 Usage Constraint
1. Messages of the transactional have no schedule and batch support.
2. In order to avoid a single message being checked too many times and lead to half queue message accumulation, we limited the number of checks for a single message to 15 times by default, but users can change this limit by change the ```transactionCheckMax``` parameter in the configuration of the broker, if one message has been checked over ```transactionCheckMax``` times, broker will discard this message and print an error log at the same time by default. Users can change this behavior by override the ```AbstractTransactionalMessageCheckListener``` class.
-3. A transactional message will be checked after a certain period of time that determined by parameter g```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter.
+3. A transactional message will be checked after a certain period of time that determined by parameter ```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter.
4. A transactional message maybe checked or consumed more than once.
5. Committed message reput to the user’s target topic may fail. Currently, it depends on the log record. High availability is ensured by the high availability mechanism of RocketMQ itself. If you want to ensure that the transactional message isn’t lost and the transaction integrity is guaranteed, it is recommended to use synchronous double write. mechanism.
6. Producer IDs of transactional messages cannot be shared with producer IDs of other types of messages. Unlike other types of message, transactional messages allow backward queries. MQ Server query clients by their Producer IDs.
From 95a6275d54d589a410511c37e62d44eda5df77fc Mon Sep 17 00:00:00 2001
From: sunbufu
Date: Sat, 26 Jun 2021 15:19:42 +0800
Subject: [PATCH 40/83] typo
---
docs/cn/design.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/cn/design.md b/docs/cn/design.md
index 1c32521ed5..e4f4648415 100644
--- a/docs/cn/design.md
+++ b/docs/cn/design.md
@@ -95,7 +95,7 @@ M1 | NettyServerCodecThread_%d | Worker线程池
M2 | RemotingExecutorThread_%d | 业务processor处理线程池
### 3 消息过滤
-RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件,是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的,Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引,然后再从CommitLog里面读取真正的消息实体内容,所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下,可以看到其中有8个字节存储的Message Tag的哈希值,基于Tag的消息过滤正式基于这个字段值的。
+RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件,是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的,Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引,然后再从CommitLog里面读取真正的消息实体内容,所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下,可以看到其中有8个字节存储的Message Tag的哈希值,基于Tag的消息过滤正是基于这个字段值的。

From 92f2d03878c45740e0c8449260a7e7549f576b95 Mon Sep 17 00:00:00 2001
From: sunbufu
Date: Mon, 28 Jun 2021 22:00:05 +0800
Subject: [PATCH 41/83] log exception stack
---
.../impl/consumer/ConsumeMessageConcurrentlyService.java | 3 ++-
.../client/impl/consumer/ConsumeMessageOrderlyService.java | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
index b37f8a6359..e7b11a0c31 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
@@ -415,7 +415,8 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService
RemotingHelper.exceptionSimpleDesc(e),
ConsumeMessageConcurrentlyService.this.consumerGroup,
msgs,
- messageQueue);
+ messageQueue,
+ e);
hasException = true;
}
long consumeRT = System.currentTimeMillis() - beginTimestamp;
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
index 130effad9e..5bd856e823 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
@@ -491,7 +491,8 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService {
RemotingHelper.exceptionSimpleDesc(e),
ConsumeMessageOrderlyService.this.consumerGroup,
msgs,
- messageQueue);
+ messageQueue,
+ e);
hasException = true;
} finally {
this.processQueue.getConsumeLock().unlock();
From 7f1cb795bd80aa3792254d8c465ea8494184b83e Mon Sep 17 00:00:00 2001
From: codersfarm <1272198093@qq.com>
Date: Wed, 30 Jun 2021 15:11:17 +0800
Subject: [PATCH 42/83] remove useless code
---
.../apache/rocketmq/store/SelectMappedBufferResult.java | 7 -------
1 file changed, 7 deletions(-)
diff --git a/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java b/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java
index 034b964bac..03061e6b48 100644
--- a/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java
+++ b/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java
@@ -48,13 +48,6 @@ public class SelectMappedBufferResult {
this.byteBuffer.limit(this.size);
}
-// @Override
-// protected void finalize() {
-// if (this.mappedFile != null) {
-// this.release();
-// }
-// }
-
public synchronized void release() {
if (this.mappedFile != null) {
this.mappedFile.release();
From 9a5ec9082f6611629fc6c7459e32ac606a893c4d Mon Sep 17 00:00:00 2001
From: lizhiboo
Date: Thu, 1 Jul 2021 15:00:27 +0800
Subject: [PATCH 43/83] [ISSUE #3109]Return a no-support value when not match
any request code
---
.../broker/processor/AdminBrokerProcessor.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index 0a1d214b87..97ca51d966 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@ -116,6 +116,7 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.LanguageCode;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
import org.apache.rocketmq.store.ConsumeQueue;
import org.apache.rocketmq.store.ConsumeQueueExt;
import org.apache.rocketmq.store.DefaultMessageStore;
@@ -235,10 +236,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
case RequestCode.GET_BROKER_CLUSTER_ACL_CONFIG:
return getBrokerClusterAclConfig(ctx, request);
default:
- break;
+ return getUnknownCmdResponse(ctx, request);
}
-
- return null;
}
@Override
@@ -459,6 +458,13 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
return null;
}
+ private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) {
+ String error = " request type " + request.getCode() + " not supported";
+ final RemotingCommand response =
+ RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);
+ return response;
+ }
+
private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class);
// final GetAllTopicConfigResponseHeader responseHeader =
From 94e08992a2bd37bce257b924b65ec38659fb6b13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=90=8C=E5=AD=A6=E7=9A=84=E5=8F=AF=E7=88=B1=E5=A5=B3?=
=?UTF-8?q?=E5=8F=8B=E7=B3=96=E7=B3=96?=
Date: Thu, 1 Jul 2021 15:01:22 +0800
Subject: [PATCH 44/83] [ISSUE #3097] docs remove obsolete/error configuration
---
docs/cn/best_practice.md | 2 +-
docs/en/best_practice.md | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/cn/best_practice.md b/docs/cn/best_practice.md
index 4ad6bb23fe..b24c6eca57 100755
--- a/docs/cn/best_practice.md
+++ b/docs/cn/best_practice.md
@@ -186,8 +186,8 @@ msgId一定是全局唯一标识符,但是实际使用中,可能会存在相
| brokerName | null | broker 的名称 |
| brokerClusterName | DefaultCluster | 本 broker 所属的 Cluser 名称 |
| brokerId | 0 | broker id, 0 表示 master, 其他的正整数表示 slave |
+| storePathRootDir | $HOME/store/ | 存储根路径 |
| storePathCommitLog | $HOME/store/commitlog/ | 存储 commit log 的路径 |
-| storePathConsumerQueue | $HOME/store/consumequeue/ | 存储 consume queue 的路径 |
| mappedFileSizeCommitLog | 1024 * 1024 * 1024(1G) | commit log 的映射文件大小 |
| deleteWhen | 04 | 在每天的什么时间删除已经超过文件保留时间的 commit log |
| fileReservedTime | 72 | 以小时计算的文件保留时间 |
diff --git a/docs/en/best_practice.md b/docs/en/best_practice.md
index be3b937d10..213dc3174d 100755
--- a/docs/en/best_practice.md
+++ b/docs/en/best_practice.md
@@ -20,8 +20,8 @@
| brokerName | null | broker name |
| brokerClusterName | DefaultCluster | this broker belongs to which cluster |
| brokerId | 0 | broker id, 0 means master, positive integers mean slave |
+| storePathRootDir | $HOME/store/ | file path for root store |
| storePathCommitLog | $HOME/store/commitlog/ | file path for commit log |
-| storePathConsumerQueue | $HOME/store/consumequeue/ | file path for consume queue |
| mappedFileSizeCommitLog | 1024 * 1024 * 1024(1G) | mapped file size for commit log |
| deleteWhen | 04 | When to delete the commitlog which is out of the reserve time |
| fileReserverdTime | 72 | The number of hours to keep a commitlog before deleting it |
From 36422c52c95272e73579e2eb0cb9e72a8659b6ed Mon Sep 17 00:00:00 2001
From: sunhangda
Date: Fri, 2 Jul 2021 15:36:53 +0800
Subject: [PATCH 45/83] [ISSUE #3066]producing a ghost consumer
---
.../rocketmq/client/impl/factory/MQClientInstance.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index 81e6d8468e..976dcba17d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -863,7 +863,7 @@ public class MQClientInstance {
}
}
- public boolean registerConsumer(final String group, final MQConsumerInner consumer) {
+ public synchronized boolean registerConsumer(final String group, final MQConsumerInner consumer) {
if (null == group || null == consumer) {
return false;
}
@@ -877,9 +877,9 @@ public class MQClientInstance {
return true;
}
- public void unregisterConsumer(final String group) {
+ public synchronized void unregisterConsumer(final String group) {
this.consumerTable.remove(group);
- this.unregisterClientWithLock(null, group);
+ this.unregisterClient(null, group);
}
private void unregisterClientWithLock(final String producerGroup, final String consumerGroup) {
From 4e5a60cb62e1fcb3b72f90c9262348a37c28e61d Mon Sep 17 00:00:00 2001
From: sunhangda
Date: Fri, 2 Jul 2021 15:48:46 +0800
Subject: [PATCH 46/83] [ISSUE #2442] when happening system busy, retry
---
.../rocketmq/client/impl/producer/DefaultMQProducerImpl.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
index fac3ed3561..6238e3e2f0 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
@@ -633,6 +633,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
case ResponseCode.TOPIC_NOT_EXIST:
case ResponseCode.SERVICE_NOT_AVAILABLE:
case ResponseCode.SYSTEM_ERROR:
+ case ResponseCode.SYSTEM_BUSY:
case ResponseCode.NO_PERMISSION:
case ResponseCode.NO_BUYER_ID:
case ResponseCode.NOT_IN_CURRENT_UNIT:
From f4b3d4f8350b594609640579c983774b296e6290 Mon Sep 17 00:00:00 2001
From: panzhi33
Date: Fri, 2 Jul 2021 17:12:51 +0800
Subject: [PATCH 47/83] fix namesrv endpoint
---
.../src/main/java/org/apache/rocketmq/client/ClientConfig.java | 2 +-
.../apache/rocketmq/common/utils/NameServerAddressUtils.java | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
index 3ab214b8db..beeeb2f5e5 100644
--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
+++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
@@ -178,7 +178,7 @@ public class ClientConfig {
}
public String getNamesrvAddr() {
- if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.validateInstanceEndpoint(namesrvAddr.trim())) {
+ if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.NAMESRV_ENDPOINT_PATTERN.matcher(namesrvAddr.trim()).matches()) {
return NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(namesrvAddr);
}
return namesrvAddr;
diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java
index 85dc95feb4..68f883935a 100644
--- a/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java
@@ -20,6 +20,7 @@ public class NameServerAddressUtils {
public static final String INSTANCE_PREFIX = "MQ_INST_";
public static final String INSTANCE_REGEX = INSTANCE_PREFIX + "\\w+_\\w+";
public static final String ENDPOINT_PREFIX = "(\\w+://|)";
+ public static final Pattern NAMESRV_ENDPOINT_PATTERN = Pattern.compile("^http://.*");
public static final Pattern INST_ENDPOINT_PATTERN = Pattern.compile("^" + ENDPOINT_PREFIX + INSTANCE_REGEX + "\\..*");
public static String getNameServerAddresses() {
From 062e6f33e547b7290f75a0adde3a8f6211c454c3 Mon Sep 17 00:00:00 2001
From: sunhangda
Date: Fri, 2 Jul 2021 17:36:51 +0800
Subject: [PATCH 48/83] revert [ISSUE #2442] when happening system busy, retry
---
.../rocketmq/client/impl/producer/DefaultMQProducerImpl.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
index 6238e3e2f0..fac3ed3561 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
@@ -633,7 +633,6 @@ public class DefaultMQProducerImpl implements MQProducerInner {
case ResponseCode.TOPIC_NOT_EXIST:
case ResponseCode.SERVICE_NOT_AVAILABLE:
case ResponseCode.SYSTEM_ERROR:
- case ResponseCode.SYSTEM_BUSY:
case ResponseCode.NO_PERMISSION:
case ResponseCode.NO_BUYER_ID:
case ResponseCode.NOT_IN_CURRENT_UNIT:
From b74c48b3607838f8e8f653d7c926ee0a851a9d6c Mon Sep 17 00:00:00 2001
From: lizhiboo
Date: Fri, 2 Jul 2021 19:57:05 +0800
Subject: [PATCH 49/83] add trace data format test case
---
.../client/trace/TraceDataEncoderTest.java | 104 +++++++++++++++++-
1 file changed, 103 insertions(+), 1 deletion(-)
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java
index af5a705395..e268dff345 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java
@@ -60,7 +60,6 @@ public class TraceDataEncoderTest {
Assert.assertEquals(contexts.get(0).getTraceType(), TraceType.Pub);
}
-
@Test
public void testEncoderFromContextBean() {
TraceContext context = new TraceContext();
@@ -130,4 +129,107 @@ public class TraceDataEncoderTest {
Assert.assertEquals(before.getTransactionState(), after.getTransactionState());
Assert.assertEquals(before.isFromTransactionCheck(), after.isFromTransactionCheck());
}
+
+ @Test
+ public void testPubTraceDataFormatTest() {
+ TraceContext pubContext = new TraceContext();
+ pubContext.setTraceType(TraceType.Pub);
+ pubContext.setTimeStamp(time);
+ pubContext.setRegionId("Default-region");
+ pubContext.setGroupName("GroupName-test");
+ pubContext.setCostTime(34);
+ pubContext.setSuccess(true);
+ TraceBean bean = new TraceBean();
+ bean.setTopic("topic-test");
+ bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+ bean.setTags("tags");
+ bean.setKeys("keys");
+ bean.setStoreHost("127.0.0.1:10911");
+ bean.setBodyLength(100);
+ bean.setMsgType(MessageType.Normal_Msg);
+ bean.setOffsetMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+ pubContext.setTraceBeans(new ArrayList(1));
+ pubContext.getTraceBeans().add(bean);
+
+ TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(pubContext);
+ String transData = traceTransferBean.getTransData();
+ Assert.assertNotNull(transData);
+ String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+ Assert.assertEquals(14, items.length);
+
+ }
+
+ @Test
+ public void testSubBeforeTraceDataFormatTest() {
+ TraceContext subBeforeContext = new TraceContext();
+ subBeforeContext.setTraceType(TraceType.SubBefore);
+ subBeforeContext.setTimeStamp(time);
+ subBeforeContext.setRegionId("Default-region");
+ subBeforeContext.setGroupName("GroupName-test");
+ subBeforeContext.setRequestId("3455848576927");
+ TraceBean bean = new TraceBean();
+ bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+ bean.setRetryTimes(0);
+ bean.setKeys("keys");
+ subBeforeContext.setTraceBeans(new ArrayList(1));
+ subBeforeContext.getTraceBeans().add(bean);
+
+ TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(subBeforeContext);
+ String transData = traceTransferBean.getTransData();
+ Assert.assertNotNull(transData);
+ String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+ Assert.assertEquals(8, items.length);
+
+ }
+
+ @Test
+ public void testSubAfterTraceDataFormatTest() {
+ TraceContext subAfterContext = new TraceContext();
+ subAfterContext.setTraceType(TraceType.SubAfter);
+ subAfterContext.setRequestId("3455848576927");
+ subAfterContext.setCostTime(20);
+ subAfterContext.setSuccess(true);
+ subAfterContext.setContextCode(98623046);
+ TraceBean bean = new TraceBean();
+ bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+ bean.setKeys("keys");
+ subAfterContext.setTraceBeans(new ArrayList(1));
+ subAfterContext.getTraceBeans().add(bean);
+
+ TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(subAfterContext);
+ String transData = traceTransferBean.getTransData();
+ Assert.assertNotNull(transData);
+ String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+ Assert.assertEquals(7, items.length);
+
+ }
+
+ @Test
+ public void testEndTrxTraceDataFormatTest() {
+ TraceContext endTrxContext = new TraceContext();
+ endTrxContext.setTraceType(TraceType.EndTransaction);
+ endTrxContext.setGroupName("PID-test");
+ endTrxContext.setRegionId("DefaultRegion");
+ endTrxContext.setTimeStamp(time);
+ TraceBean endTrxTraceBean = new TraceBean();
+ endTrxTraceBean.setTopic("topic-test");
+ endTrxTraceBean.setKeys("Keys");
+ endTrxTraceBean.setTags("Tags");
+ endTrxTraceBean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+ endTrxTraceBean.setStoreHost("127.0.0.1:10911");
+ endTrxTraceBean.setMsgType(MessageType.Trans_msg_Commit);
+ endTrxTraceBean.setTransactionId("transactionId");
+ endTrxTraceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE);
+ endTrxTraceBean.setFromTransactionCheck(false);
+ List traceBeans = new ArrayList();
+ traceBeans.add(endTrxTraceBean);
+ endTrxContext.setTraceBeans(traceBeans);
+
+ TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(endTrxContext);
+ String transData = traceTransferBean.getTransData();
+ Assert.assertNotNull(transData);
+ String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+ Assert.assertEquals(13, items.length);
+
+ }
}
From bc73c32c905cf2a6d72edb4565cc0da1fee84cbc Mon Sep 17 00:00:00 2001
From: sunbufu
Date: Sat, 3 Jul 2021 20:46:06 +0800
Subject: [PATCH 50/83] use String.format() in warn log.
---
.../impl/consumer/ConsumeMessageConcurrentlyService.java | 5 ++---
.../client/impl/consumer/ConsumeMessageOrderlyService.java | 5 ++---
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
index e7b11a0c31..925e5258b5 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
@@ -411,12 +411,11 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService
}
status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);
} catch (Throwable e) {
- log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
+ log.warn(String.format("consumeMessage exception: %s Group: %s Msgs: %s MQ: %s",
RemotingHelper.exceptionSimpleDesc(e),
ConsumeMessageConcurrentlyService.this.consumerGroup,
msgs,
- messageQueue,
- e);
+ messageQueue), e);
hasException = true;
}
long consumeRT = System.currentTimeMillis() - beginTimestamp;
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
index 5bd856e823..8d92b5718a 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
@@ -487,12 +487,11 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService {
status = messageListener.consumeMessage(Collections.unmodifiableList(msgs), context);
} catch (Throwable e) {
- log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
+ log.warn(String.format("consumeMessage exception: %s Group: %s Msgs: %s MQ: %s",
RemotingHelper.exceptionSimpleDesc(e),
ConsumeMessageOrderlyService.this.consumerGroup,
msgs,
- messageQueue,
- e);
+ messageQueue), e);
hasException = true;
} finally {
this.processQueue.getConsumeLock().unlock();
From 65915891828ef5314209dd8b6df41b2f4da7e2c7 Mon Sep 17 00:00:00 2001
From: Git_Yang <30995057+Git-Yang@users.noreply.github.com>
Date: Mon, 5 Jul 2021 16:03:45 +0800
Subject: [PATCH 51/83] [ISSUE #3118] Fix type conversion exception
Signed-off-by: zhangyang21
---
.../rocketmq/store/dledger/DLedgerCommitLog.java | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
index 90fd6f3f16..3b9876030d 100644
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
@@ -562,11 +562,13 @@ public class DLedgerCommitLog extends CommitLog {
request.setGroup(dLedgerConfig.getGroup());
request.setRemoteId(dLedgerServer.getMemberState().getSelfId());
request.setBatchMsgs(encodeResult.batchData);
- dledgerFuture = (BatchAppendFuture) dLedgerServer.handleAppend(request);
- if (dledgerFuture.getPos() == -1) {
- log.warn("HandleAppend return false due to error code {}", dledgerFuture.get().getCode());
+ AppendFuture appendFuture = (AppendFuture) dLedgerServer.handleAppend(request);
+ if (appendFuture.getPos() == -1) {
+ log.warn("HandleAppend return false due to error code {}", appendFuture.get().getCode());
return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR));
}
+ dledgerFuture = (BatchAppendFuture) appendFuture;
+
long wroteOffset = 0;
int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;
@@ -789,11 +791,13 @@ public class DLedgerCommitLog extends CommitLog {
request.setGroup(dLedgerConfig.getGroup());
request.setRemoteId(dLedgerServer.getMemberState().getSelfId());
request.setBatchMsgs(encodeResult.batchData);
- dledgerFuture = (BatchAppendFuture) dLedgerServer.handleAppend(request);
- if (dledgerFuture.getPos() == -1) {
- log.warn("HandleAppend return false due to error code {}", dledgerFuture.get().getCode());
+ AppendFuture appendFuture = (AppendFuture) dLedgerServer.handleAppend(request);
+ if (appendFuture.getPos() == -1) {
+ log.warn("HandleAppend return false due to error code {}", appendFuture.get().getCode());
return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
}
+ dledgerFuture = (BatchAppendFuture) appendFuture;
+
long wroteOffset = 0;
int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;
From 7c364687c65bb1e9207600e1f4c1610363ae10bc Mon Sep 17 00:00:00 2001
From: mxsm
Date: Mon, 5 Jul 2021 16:09:34 +0800
Subject: [PATCH 52/83] [ISSUE #3089] fix Print log format is incorrect in
NettyRemotingAbstract class (#3126)
---
.../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 3f21eb8ff1..c9b39946c0 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -553,7 +553,7 @@ public abstract class NettyRemotingAbstract {
throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast");
} else {
String info = String.format(
- "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d",
+ "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreOnewayValue: %d",
timeoutMillis,
this.semaphoreOneway.getQueueLength(),
this.semaphoreOneway.availablePermits()
From d65778f3a83b2daf49cecf270a6bac166ad6f9b8 Mon Sep 17 00:00:00 2001
From: huangli
Date: Tue, 6 Jul 2021 21:46:14 +0800
Subject: [PATCH 53/83] [ISSUE #2883] [Part B] Improve produce performance in
M/S mode. (#2885)
* Optimise lock in WaitNotifyObject
* Remove lock in HAService
* Remove lock in GroupCommitService
---
.../org/apache/rocketmq/store/CommitLog.java | 65 ++++++++++---------
.../apache/rocketmq/store/ha/HAService.java | 55 +++++++++-------
.../rocketmq/store/ha/WaitNotifyObject.java | 64 +++++++++---------
3 files changed, 101 insertions(+), 83 deletions(-)
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index cce6481b8d..43b01f08d8 100644
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@ -19,8 +19,8 @@ package org.apache.rocketmq.store;
import java.net.Inet6Address;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@@ -1403,48 +1403,55 @@ public class CommitLog {
* GroupCommit Service
*/
class GroupCommitService extends FlushCommitLogService {
- private volatile List requestsWrite = new ArrayList();
- private volatile List requestsRead = new ArrayList();
+ private volatile LinkedList requestsWrite = new LinkedList();
+ private volatile LinkedList requestsRead = new LinkedList();
+ private final PutMessageSpinLock lock = new PutMessageSpinLock();
public synchronized void putRequest(final GroupCommitRequest request) {
- synchronized (this.requestsWrite) {
+ lock.lock();
+ try {
this.requestsWrite.add(request);
+ } finally {
+ lock.unlock();
}
this.wakeup();
}
private void swapRequests() {
- List tmp = this.requestsWrite;
- this.requestsWrite = this.requestsRead;
- this.requestsRead = tmp;
+ lock.lock();
+ try {
+ LinkedList tmp = this.requestsWrite;
+ this.requestsWrite = this.requestsRead;
+ this.requestsRead = tmp;
+ } finally {
+ lock.unlock();
+ }
}
private void doCommit() {
- synchronized (this.requestsRead) {
- if (!this.requestsRead.isEmpty()) {
- for (GroupCommitRequest req : this.requestsRead) {
- // There may be a message in the next file, so a maximum of
- // two times the flush
- boolean flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();
- for (int i = 0; i < 2 && !flushOK; i++) {
- CommitLog.this.mappedFileQueue.flush(0);
- flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();
- }
-
- req.wakeupCustomer(flushOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_DISK_TIMEOUT);
+ if (!this.requestsRead.isEmpty()) {
+ for (GroupCommitRequest req : this.requestsRead) {
+ // There may be a message in the next file, so a maximum of
+ // two times the flush
+ boolean flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();
+ for (int i = 0; i < 2 && !flushOK; i++) {
+ CommitLog.this.mappedFileQueue.flush(0);
+ flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();
}
- long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp();
- if (storeTimestamp > 0) {
- CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp);
- }
-
- this.requestsRead.clear();
- } else {
- // Because of individual messages is set to not sync flush, it
- // will come to this process
- CommitLog.this.mappedFileQueue.flush(0);
+ req.wakeupCustomer(flushOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_DISK_TIMEOUT);
}
+
+ long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp();
+ if (storeTimestamp > 0) {
+ CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp);
+ }
+
+ this.requestsRead = new LinkedList<>();
+ } else {
+ // Because of individual messages is set to not sync flush, it
+ // will come to this process
+ CommitLog.this.mappedFileQueue.flush(0);
}
}
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
index 34c51eb9e2..d4d4109967 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
@@ -25,7 +25,6 @@ import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
-import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
@@ -39,6 +38,7 @@ import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.store.CommitLog;
import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.PutMessageSpinLock;
import org.apache.rocketmq.store.PutMessageStatus;
public class HAService {
@@ -254,12 +254,16 @@ public class HAService {
class GroupTransferService extends ServiceThread {
private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject();
- private volatile List requestsWrite = new ArrayList<>();
- private volatile List requestsRead = new ArrayList<>();
+ private final PutMessageSpinLock lock = new PutMessageSpinLock();
+ private volatile LinkedList requestsWrite = new LinkedList<>();
+ private volatile LinkedList requestsRead = new LinkedList<>();
- public synchronized void putRequest(final CommitLog.GroupCommitRequest request) {
- synchronized (this.requestsWrite) {
+ public void putRequest(final CommitLog.GroupCommitRequest request) {
+ lock.lock();
+ try {
this.requestsWrite.add(request);
+ } finally {
+ lock.unlock();
}
this.wakeup();
}
@@ -269,32 +273,35 @@ public class HAService {
}
private void swapRequests() {
- List tmp = this.requestsWrite;
- this.requestsWrite = this.requestsRead;
- this.requestsRead = tmp;
+ lock.lock();
+ try {
+ LinkedList tmp = this.requestsWrite;
+ this.requestsWrite = this.requestsRead;
+ this.requestsRead = tmp;
+ } finally {
+ lock.unlock();
+ }
}
private void doWaitTransfer() {
- synchronized (this.requestsRead) {
- if (!this.requestsRead.isEmpty()) {
- for (CommitLog.GroupCommitRequest req : this.requestsRead) {
- boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
- long waitUntilWhen = HAService.this.defaultMessageStore.getSystemClock().now()
+ if (!this.requestsRead.isEmpty()) {
+ for (CommitLog.GroupCommitRequest req : this.requestsRead) {
+ boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
+ long waitUntilWhen = HAService.this.defaultMessageStore.getSystemClock().now()
+ HAService.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout();
- while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntilWhen) {
- this.notifyTransferObject.waitForRunning(1000);
- transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
- }
-
- if (!transferOK) {
- log.warn("transfer messsage to slave timeout, " + req.getNextOffset());
- }
-
- req.wakeupCustomer(transferOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
+ while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntilWhen) {
+ this.notifyTransferObject.waitForRunning(1000);
+ transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
}
- this.requestsRead.clear();
+ if (!transferOK) {
+ log.warn("transfer messsage to slave timeout, " + req.getNextOffset());
+ }
+
+ req.wakeupCustomer(transferOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
}
+
+ this.requestsRead = new LinkedList<>();
}
}
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java b/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java
index 75b7597fea..d5ed65f5f7 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java
@@ -20,40 +20,43 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
-import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
public class WaitNotifyObject {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
- protected final HashMap waitingThreadTable =
- new HashMap(16);
+ protected final ConcurrentHashMap waitingThreadTable =
+ new ConcurrentHashMap(16);
- protected volatile boolean hasNotified = false;
+ protected AtomicBoolean hasNotified = new AtomicBoolean(false);
public void wakeup() {
- synchronized (this) {
- if (!this.hasNotified) {
- this.hasNotified = true;
+ boolean needNotify = hasNotified.compareAndSet(false, true);
+ if (needNotify) {
+ synchronized (this) {
this.notify();
}
}
}
protected void waitForRunning(long interval) {
+ if (this.hasNotified.compareAndSet(true, false)) {
+ this.onWaitEnd();
+ return;
+ }
synchronized (this) {
- if (this.hasNotified) {
- this.hasNotified = false;
- this.onWaitEnd();
- return;
- }
-
try {
+ if (this.hasNotified.compareAndSet(true, false)) {
+ this.onWaitEnd();
+ return;
+ }
this.wait(interval);
} catch (InterruptedException e) {
log.error("Interrupted", e);
} finally {
- this.hasNotified = false;
+ this.hasNotified.set(false);
this.onWaitEnd();
}
}
@@ -63,15 +66,14 @@ public class WaitNotifyObject {
}
public void wakeupAll() {
- synchronized (this) {
- boolean needNotify = false;
-
- for (Map.Entry entry : this.waitingThreadTable.entrySet()) {
- needNotify = needNotify || !entry.getValue();
- entry.setValue(true);
+ boolean needNotify = false;
+ for (Map.Entry entry : this.waitingThreadTable.entrySet()) {
+ if (entry.getValue().compareAndSet(false, true)) {
+ needNotify = true;
}
-
- if (needNotify) {
+ }
+ if (needNotify) {
+ synchronized (this) {
this.notifyAll();
}
}
@@ -79,20 +81,22 @@ public class WaitNotifyObject {
public void allWaitForRunning(long interval) {
long currentThreadId = Thread.currentThread().getId();
+ AtomicBoolean notified = this.waitingThreadTable.computeIfAbsent(currentThreadId, k -> new AtomicBoolean(false));
+ if (notified.compareAndSet(true, false)) {
+ this.onWaitEnd();
+ return;
+ }
synchronized (this) {
- Boolean notified = this.waitingThreadTable.get(currentThreadId);
- if (notified != null && notified) {
- this.waitingThreadTable.put(currentThreadId, false);
- this.onWaitEnd();
- return;
- }
-
try {
+ if (notified.compareAndSet(true, false)) {
+ this.onWaitEnd();
+ return;
+ }
this.wait(interval);
} catch (InterruptedException e) {
log.error("Interrupted", e);
} finally {
- this.waitingThreadTable.put(currentThreadId, false);
+ notified.set(false);
this.onWaitEnd();
}
}
From 3183122c01cc2d0dc005634d028ccbcb62495bf6 Mon Sep 17 00:00:00 2001
From: huangli
Date: Tue, 6 Jul 2021 21:47:12 +0800
Subject: [PATCH 54/83] Eliminate array copy (#2886)
[Part C] Improve produce performance in M/S mode.
---
.../broker/plugin/AbstractPluginMessageStore.java | 4 ++--
.../main/java/org/apache/rocketmq/store/CommitLog.java | 4 ++--
.../org/apache/rocketmq/store/DefaultMessageStore.java | 4 ++--
.../java/org/apache/rocketmq/store/MessageStore.java | 4 +++-
.../rocketmq/store/dledger/DLedgerCommitLog.java | 2 +-
.../java/org/apache/rocketmq/store/ha/HAService.java | 10 ++++------
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
index 1db019bec8..b95bab6249 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java
@@ -180,8 +180,8 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
}
@Override
- public boolean appendToCommitLog(long startOffset, byte[] data) {
- return next.appendToCommitLog(startOffset, data);
+ public boolean appendToCommitLog(long startOffset, byte[] data, int dataStart, int dataLength) {
+ return next.appendToCommitLog(startOffset, data, dataStart, dataLength);
}
@Override
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index 43b01f08d8..57fa363785 100644
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@ -1185,7 +1185,7 @@ public class CommitLog {
this.mappedFileQueue.destroy();
}
- public boolean appendData(long startOffset, byte[] data) {
+ public boolean appendData(long startOffset, byte[] data, int dataStart, int dataLength) {
putMessageLock.lock();
try {
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(startOffset);
@@ -1194,7 +1194,7 @@ public class CommitLog {
return false;
}
- return mappedFile.appendMessage(data);
+ return mappedFile.appendMessage(data, dataStart, dataLength);
} finally {
putMessageLock.unlock();
}
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index b8ecdee8cc..7dd5a32b2f 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@ -921,13 +921,13 @@ public class DefaultMessageStore implements MessageStore {
}
@Override
- public boolean appendToCommitLog(long startOffset, byte[] data) {
+ public boolean appendToCommitLog(long startOffset, byte[] data, int dataStart, int dataLength) {
if (this.shutdown) {
log.warn("message store has shutdown, so appendToPhyQueue is forbidden");
return false;
}
- boolean result = this.commitLog.appendData(startOffset, data);
+ boolean result = this.commitLog.appendData(startOffset, data, dataStart, dataLength);
if (result) {
this.reputMessageService.wakeup();
} else {
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
index 64eb5250de..a8c658bfe2 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
@@ -245,9 +245,11 @@ public interface MessageStore {
*
* @param startOffset starting offset.
* @param data data to append.
+ * @param dataStart the start index of data array
+ * @param dataLength the length of data array
* @return true if success; false otherwise.
*/
- boolean appendToCommitLog(final long startOffset, final byte[] data);
+ boolean appendToCommitLog(final long startOffset, final byte[] data, int dataStart, int dataLength);
/**
* Execute file deletion manually.
diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
index 3b9876030d..ea791bd9ff 100644
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
@@ -905,7 +905,7 @@ public class DLedgerCommitLog extends CommitLog {
}
@Override
- public boolean appendData(long startOffset, byte[] data) {
+ public boolean appendData(long startOffset, byte[] data, int dataStart, int dataLength) {
//the old ha service will invoke method, here to prevent it
return false;
}
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
index d4d4109967..845935bb9d 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
@@ -440,7 +440,6 @@ public class HAService {
private boolean dispatchReadRequest() {
final int msgHeaderSize = 8 + 4; // phyoffset + size
- int readSocketPos = this.byteBufferRead.position();
while (true) {
int diff = this.byteBufferRead.position() - this.dispatchPosition;
@@ -459,13 +458,12 @@ public class HAService {
}
if (diff >= (msgHeaderSize + bodySize)) {
- byte[] bodyData = new byte[bodySize];
- this.byteBufferRead.position(this.dispatchPosition + msgHeaderSize);
- this.byteBufferRead.get(bodyData);
+ byte[] bodyData = byteBufferRead.array();
+ int dataStart = this.dispatchPosition + msgHeaderSize;
- HAService.this.defaultMessageStore.appendToCommitLog(masterPhyOffset, bodyData);
+ HAService.this.defaultMessageStore.appendToCommitLog(
+ masterPhyOffset, bodyData, dataStart, bodySize);
- this.byteBufferRead.position(readSocketPos);
this.dispatchPosition += msgHeaderSize + bodySize;
if (!reportSlaveMaxOffsetPlus()) {
From 44e2669a2f48a1526bcd3418b81feded6f634d67 Mon Sep 17 00:00:00 2001
From: zhangjidi2016
Date: Wed, 7 Jul 2021 00:01:59 +0800
Subject: [PATCH 55/83] [ISSUE #3128] fix the delete global white addresses
---
.../rocketmq/acl/plain/PlainPermissionManager.java | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java
index 078d34b10b..a22b0b25a7 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java
+++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java
@@ -251,12 +251,6 @@ public class PlainPermissionManager {
}
public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList) {
-
- if (globalWhiteAddrsList == null) {
- log.error("Parameter value globalWhiteAddrsList is null,Please check your parameter");
- return false;
- }
-
Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName,
Map.class);
if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) {
@@ -266,9 +260,10 @@ public class PlainPermissionManager {
if (globalWhiteRemoteAddrList != null) {
globalWhiteRemoteAddrList.clear();
- globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList);
-
- // Update globalWhiteRemoteAddr element in memeory map firstly
+ if (globalWhiteAddrsList != null) {
+ globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList);
+ }
+ // Update globalWhiteRemoteAddr element in memory map firstly
aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS, globalWhiteRemoteAddrList);
if (AclUtils.writeDataObject(fileHome + File.separator + fileName, updateAclConfigFileVersion(aclAccessConfigMap))) {
return true;
From 4c1998a54ef5b5b7f36130a626c99c9ca2064972 Mon Sep 17 00:00:00 2001
From: zhangjidi2016
Date: Wed, 7 Jul 2021 08:55:04 +0800
Subject: [PATCH 56/83] retrigger ci
From b13a4c862fac5889318079e5fc81c3835d128676 Mon Sep 17 00:00:00 2001
From: huangli
Date: Thu, 27 May 2021 18:55:30 +0800
Subject: [PATCH 57/83] [ISSUE 2883] [Part G] Optimise parse performance for
SendMessageRequestHeaderV2
---
.../AbstractSendMessageProcessor.java | 73 ++++++++++++++++++-
1 file changed, 70 insertions(+), 3 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
index ced7c2014f..95602206a8 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
@@ -19,8 +19,10 @@ package org.apache.rocketmq.broker.processor;
import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Random;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
@@ -288,9 +290,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
switch (request.getCode()) {
case RequestCode.SEND_BATCH_MESSAGE:
case RequestCode.SEND_MESSAGE_V2:
- requestHeaderV2 =
- (SendMessageRequestHeaderV2) request
- .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
+ requestHeaderV2 = decodeSendMessageHeaderV2(request);
case RequestCode.SEND_MESSAGE:
if (null == requestHeaderV2) {
requestHeader =
@@ -305,6 +305,73 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
return requestHeader;
}
+ private SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request) {
+ SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2();
+ HashMap fields = request.getExtFields();
+ if (fields == null) {
+ // keep same behavior with CommandCustomHeader.decodeCommandCustomHeader
+ return r;
+ }
+
+ String s = fields.get("a");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setA(s);
+
+ s = fields.get("b");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setB(s);
+
+ s = fields.get("c");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setC(s);
+
+ s = fields.get("d");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setD(Integer.parseInt(s));
+
+ s = fields.get("e");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setE(Integer.parseInt(s));
+
+ s = fields.get("f");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setF(Integer.parseInt(s));
+
+ s = fields.get("g");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setG(Long.parseLong(s));
+
+ s = fields.get("h");
+ Objects.requireNonNull(s, "the custom field is null");
+ r.setH(Integer.parseInt(s));
+
+ s = fields.get("i");
+ if (s != null) {
+ r.setI(s);
+ }
+
+ s = fields.get("j");
+ if (s != null) {
+ r.setJ(Integer.parseInt(s));
+ }
+
+ s = fields.get("k");
+ if (s != null) {
+ r.setK(Boolean.parseBoolean(s));
+ }
+
+ s = fields.get("l");
+ if (s != null) {
+ r.setL(Integer.parseInt(s));
+ }
+
+ s = fields.get("m");
+ if (s != null) {
+ r.setM(Boolean.parseBoolean(s));
+ }
+ return r;
+ }
+
public void executeSendMessageHookAfter(final RemotingCommand response, final SendMessageContext context) {
if (hasSendMessageHook()) {
for (SendMessageHook hook : this.sendMessageHookList) {
From 589e87c278147ee17cbcaac334d8558a8ef98ea2 Mon Sep 17 00:00:00 2001
From: xiaoyaofawai
Date: Fri, 9 Jul 2021 13:40:02 +0800
Subject: [PATCH 58/83] [ISSUE #3099] Fix the incorrect judegment 'writePos -
lastCommittedPosition > commitLeastPages' in the method 'commit0' of class
'MappedFile' (#3125)
---
.../src/main/java/org/apache/rocketmq/store/MappedFile.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
index b5bdc7766b..25f0e39314 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java
@@ -301,7 +301,7 @@ public class MappedFile extends ReferenceResource {
}
if (this.isAbleToCommit(commitLeastPages)) {
if (this.hold()) {
- commit0(commitLeastPages);
+ commit0();
this.release();
} else {
log.warn("in commit, hold failed, commit offset = " + this.committedPosition.get());
@@ -317,11 +317,11 @@ public class MappedFile extends ReferenceResource {
return this.committedPosition.get();
}
- protected void commit0(final int commitLeastPages) {
+ protected void commit0() {
int writePos = this.wrotePosition.get();
int lastCommittedPosition = this.committedPosition.get();
- if (writePos - lastCommittedPosition > commitLeastPages) {
+ if (writePos - lastCommittedPosition > 0) {
try {
ByteBuffer byteBuffer = writeBuffer.slice();
byteBuffer.position(lastCommittedPosition);
From 17a8e93ba4dce34e6a9b2f35af46e62a71923882 Mon Sep 17 00:00:00 2001
From: huangjunwei
Date: Wed, 14 Jul 2021 19:03:03 +0800
Subject: [PATCH 59/83] fix spell error
---
docs/cn/image/rocketmq_architecture_1.png | Bin 89290 -> 62810 bytes
docs/cn/image/rocketmq_architecture_3.png | Bin 106758 -> 74884 bytes
docs/en/image/rocketmq_architecture_1.png | Bin 89290 -> 62810 bytes
docs/en/image/rocketmq_architecture_3.png | Bin 106758 -> 74884 bytes
4 files changed, 0 insertions(+), 0 deletions(-)
diff --git a/docs/cn/image/rocketmq_architecture_1.png b/docs/cn/image/rocketmq_architecture_1.png
index 1d3e82b354b563e17aa90d64c9f94dd9bfe476b6..addb571a84459640ad8e021e31bd4e4cad2e0f5e 100644
GIT binary patch
literal 62810
zcmeFYc{tQ<`#3DAv?6WVC~1?7S&gy`V=RNQ4l|Y-%Z$N{Z5GB}Dk2q;H7%k-3Q^g*
zOSE8Qln_dm$dc@Y_nL7(zwdJ&-{-fz|Gw{WFf*U+I?wCew@<&g&tQ!!^z9{
zQ)q6?zZXOthkykR+tQh?-dun1)65!ts6oJKm@fDQE+FB{KLe2PBj60i$H&Xf*3B7D
z2hec{80t6-2~Hoz8(N#1C_(kWZ!fy18~8GCbMf>RKBDLD=gk0TaA>vTkmGPT3Lpc8
zg9}y^D#effZ_5aG3Nb10|OU_7jqiXj<;KrQcpef93iwyjM=N2}lynix>!OHiEEj}7T;Ck7#`I&NJTU=!95Wgf
ziL_?HoGpC_>T0HbC^e3*ocPt%h
zrv~*x*jfO(<8&F|gJT4qz*!?L-~ZhB@^b6-O%A_YOlSiwog
zJ_hZNWtoAD96gF8pQVV`<^z%OO~s2l7}ZL>eB0Q1jA!Pg9Z7?uXgbCe_YXd>!nFbVpSGXMlZDGaIHL-ND0OF$Sj&dhhnfTCs
zPuI0VCnF>PoBf~P6O7l|=-$C`N9`haa3TN^`YZ~{+PO_yt?rfb8sU_%Wk
zC_3JlLBrvU{9V1VFq$zH!ZT$WxWegAi}y76Ew-wg1{D1onqu|Y3{6N>ZMO`HbMBhgHsrW
zB_6|rvh~PD-ho&*sv+Fg*vyOQ&NA_KvGKR2Lu^ge{5*AO>Q+#Qu8@H+Jk?pvmg?%l
zG%zMHkp5VGbD9Yei6^10;ej|C3rn=Ar+EO~!&1%2#RLNbPgr^3T~W?HIG9jFAS^Tl
z3Go4t!4-cW2Hp(ig)??G;rWuy4J;9MFoLzWC52=wR6Sjqr5=yz?m{D4L5T&58_C358Gv&LjGnDCfo#qt
z1vu+_!s)&UEQN{nHiKe>Bqw?M8rvB-gZt>ZZq`<0e>0XhhsZQD2!MDXJ$(r*MTf>WaiO0+@6Iwh`1g5an+GGca^!x#;=%!c4rdEDK9_m@SQj2RG}v
zTj?A4>Jo?$ikho4(>Q=gv8L%-(hcz#UvD#FpgzkI$1q~!I05chB#o!;EmTe}R~@D6
zjNqsTvMud+RJ@-_Aj8eXl;Fy;@$xWJC$kZzc(|{#kSAEQx(`A>fNV&0^%L5362aBd
zl;8n`$(-S8;z#gtMH1EMezt5^b^r$BYHdcu8u*)0P;Oin(+%g!!}=N;sHwRixTek)
zYAhC759$iFgOTV67gwls0tFhc+~c(Tng7(E|#2GZ5d
z%82QV1=^x-2*)9)~!W=Jy_*@NQ7
zMf;+lzDPe~1{1&c_>
zaMs2y!ebQ8*w&8Can(b4qP)zFu(nVe0K=JN#e~y1ENefyE6g=OUEdlBGz3Quq!JK@
zMldpq>}$*Rg%X{4EO%Ew1`4SLapfZ1g#H5R=gM@2`oi3pcn=iJ*u@{|MmF(f5y|>U
z3o1*EMesx;)nG7`8O{I=^+EXCI@=+9EKE%iOqz$WsWF3P3aDl2Y-P@MAt3P-3Qm^<
zxAJFD$OPc~S;#<^v5P*PpbPOq5-ohG#(qR~mNzhcIGP&XP>)XL7~uSDJj_V&K&**3
z6U%ak6Ks45csyQci%k$b7c(Xliz1^@R6H7rpyIt<0cvh8CRTKFq>Fw4ih^b$t%VE&
z5rM7|mO_D3>?kBU)XS0%$3d*Dp-exDFU`nLk7*<{2<`@fI8z&4IvHW%XX%3Uq3WZk
z1~>yEO5dD9w1Ea7>>x}JbqfyC$A;sIx3KYN*gyi@@JLGwOCKu(Ux+B1t|Ht!OVE>I9&sdE{chBCJ?M(fu3v+Ym(3ib69~^M1(8&MYMD=
zU@=iHHlD)M0XA$sON^hLiMN>*9?LLvbw=Z93>3o!T;zEf0T+WKW6Zb#SQG~i_p%MJ
zvw^5li0%xSp_`4+@C16e!XauF1_%Pv49c+u0oa<_cpj)_;U8wFPvY)@Ztrbg8O=L3lfzm?VF(8hF>bipX%9!futd6iI5v(CR
z3>@$k>8EDqN8_SV9zGPSKtqh3FNvt;OaZt1IeQY37z)YJwwEo2jnuWV
zhB9ouT^MkEl&hx|&5h(~ZsmgK!ci0!aBaGDt^vZ!7=z@J88jPHQ$GTpO2eT|{H!1*
zC?7SUmgwqo2@GdDGzUvT2U=UBg=98ldD*FB&`4b`Po%S<7lY`^_JLxdFl3;Y5yHd_
z=B0+U^!7J&b`P|rb0IF?NRqpSM<5SL@JE+|LSd9D^ZPI
z$T%F!p9o%h8mKX#
zHtuAa9*ZEn;^7DLK
z$^CDRUc7x;TH@?%q|BPWbuow2Lh~qcx~Id#Z&oKRu`g;dD)iZ0wk`W3X{7Su^A9*0xZhaZ-&CH64W=;xJmn
zmTp)w3o0!syz7m$PO#->5p?OJE2?StQrsLwN9J#D
zvl6HMfA!qXY0Eh^HN5`GEOpd7EUfp7{LiZGU&!>mCRO8Mp?4}DErp(qtVavUaAsfl
z%!Y%B8ze+j@B`#MvR7(L6LSR!}$cr;%L@6^~|>2u{Ev$9pvD5!my>t4Vg
zN!wNRGa3%UiWJSEtMUA-eb9k^>syC$nS@a
zuXC>6>U#e5_LSO*Fjno-y5yB>5|(wJ9B+
zylXt2kV(wpw!~{`Ud5D`MFQhCC*|X&!T6+OA0pR!Duv{{&YA7>n(p$8iFWjN>%G6~
z^my?^;$6kxSGINP`}y!IrU&j`KV`;5U8%PS>wWt{xZN)bfNF|T*cBqGy#g%Ty~&n*
zPNaT)YqxC6^fyb*ug#p1&j@LFrbtwv+6Ggr_)P86F75fDqG3+nVUw;H!5_!xs>)yT
z0-iWGDirUBs`UOY>CN$-_UNjiH$2XD7$=@?`yO}lPh-!Gw2HPTzFmR+1!bLFcssMe
za=xaK7q>Lz0Wb|HJ>k}`%@FKu>n6$)J74T-)+_jPEvX^
zD^j$tMT^hBI=)~BqBiAm_T{%HR4Xd9Mt6^M2RH_o_9o`l46#S+
z${(|pZ->6_XKPnf{v7;#a*T9f&BDXy?8!?Qk+wGh5o!M0%m7T~{5W#EfNn-ST|1$q
zbt$~K{m$^kO0(&i;`m0UJ)e1}LG~h@lDaL7_>`)DH_^XUr2~1lRiz)J
z45&xblRA-ojOdx6gTAKP^k`C6hwt~AUZx&D>D!~%{kQdFibogMZ0q#sGTKJ-H=nG)
zeGd(S@+5U$Y!#0p*g`S8Mbv?80M>CQI~XX-8vnW1%ppZ_X3xEg4U2f*S%bkkmu
z!Fq}X%(O&Ctr9Ww!2!sX8I!J3frQKJ97lb3{hJ%%-nsJ%-LSImB6(>#W%i3dpBqem
z&L~OYtWGNmfPMZVW2>C6Rd
zrCMDv@k(HB=i-laM~WmG1B}p`TQwr{r0{iRL_~FtoE?vS$FY^ZHyPINNsA45@O&$E
zs`BR9yqPaPuQvLP#-t=AWW+r-GVF>OpH<>tw!5Fgjh!tk&mZX71Qy(s}ofkCx~HX(2pe_29AseN^$2#
zSCy97ip~EzsDKkPr~=>?B-7o$EZDFW)j9lDMUIGdKX9i{kI`4%Z2iOpWE!Jqmfz5#
ze!qk?iM+S<;zHgGe0p1bkeKDv;_cS?%WV<;`Gs{_A(~~kAGZ^M1*q53th6oDw+(Ll
zEGXS-7xXf+r$8kpgY)HDoY$ZMJOdz&C>dPN5Xac&YAB)zxP%HF@f{wVpp|
zFPMRE?C&@MCp^uzCAWo!`VA8jK6~wLs&)Nd?AI1oJC)W)^Y1*eCW>#Kv#w?p&@u(B
z?+w~_E&%>YYXE=nLyj6OBw@Ewqg8NC%=xQ&LLVw7;RM`6*5?22b*oK*&T<#!G(}t5
z0?gqtv1mQfeJ|{RHhEhad=;^z@vK;P!sg{zL2?4zmxcZRSNs18-C9pCo(h??(VT3y
zA{V<`%=2HzZ1)`c{Fuym@(~xTQd9@sh%;_EE`8I<;msLsHFMREee1?ZDg^R!C{_w|
zyW`!O52Bz#hYeKdBNAYXp5f2q+8!>E_vcVOn$N_$S?l7
z5Ci{9Z7*2%q+z_?K12
z{%B+SHt*}u6dC+-!)7O@^br*G1jq>=wt#vCCl&?aZ=aX&g4;)d8J~
z4GGKuhLvi4&z7;)4P4%G9KZwE|NLg6x(r4eD4BkpcllTimX%s?C^NXQV{r8@ubcI#
zu9Ku&DO=O~=J(rwF0v=)??8ktc5kn0T$t|a8sQt7E9%s@AltpHKKyXc_7HEzf5g6G
zUgpuYs-pOP{yGw(uA9i%xF?Bq7fCFoF`pli$)K|PWeF!gFb6b811}#W
z+(>^@J6$mvqQpDQ8O$&3Y)g75b@G?k=HeYCZI^F5c^HhaFEq#459kFyPn5d793XoD
zg5~xad5S=N-2@0G_U!3wUmE+1Z|&Qtwew_d2V3?7J+hyg#z5-ieoQYs{nwciiI87^
zt!clM`Z2SJ#{WdLXV{n;6k8^XOx<_Z=1CeD6HKd1qX)xG16;
zk*UY8qWGek%kl-9Q=b)OZrFQ0)|XfL=zlNqrmchTxoyWCl3Bbr
z`!4*bmL)V7zCUhHkd~R&t-z-LS#jR1VuM(Ijg@pJ;$^n-GN23qh||N86ahj2K-(lV
zy*p2ura<)x+6yed8ViFxiFr4J1!4D2-O`?BaozWS{^e)Y%KjTjX
zW+xEat-oqosYV=n+WpN!b?eGKy8co8eL8oIL~wk-h6E|r(vl(zX$@i=lGu;l>!%)S
zPJe-+{Vwd`7)JJIr!mQ2O-eUC`yO{<^9qfy#Xkoiu!n@u-wDiBaysy4yqHm*uVgd+
z(jz?U+|02Rm#su0^@~7YV*9PI;mgcd0J0_J)1Se;u}S@Fc+cS?(?7AWh8@yRPUP(2
ze5MThI{5sxs*T$f(y_K--0
zcbPl&Io<7Z%ItF^hKKej%CB{EzjWy=kG`Se`W`FQ(7PMQ25c*%wuyYl9Pk}Om$w^;
zT!|zcW5K!`uPGUl=Bmv&ZHar=
zL){T!$GVGVcT2AjQE|g%v2OnWFS?v){Zu8e{)?q+R%RbQ7|7>OwwyXO-N8LoMB3>*
zN)4`>?-+%L%(3KT-g!dAo8O=Q;;b3N8&e&RA}smgs+YAg9ms;Za-1m4Sy2TP_C(M7
zle58N-LId|4dzZhz8q3lkT)eWY-8(q_ie%<&vRRejW5rwM^NjMwHo@&@!`B%+vfJ^
zR7~i-zvLp5$?Sr3@49&b1y^{n()YL3&(_X7|EEHEh}zuIuYn=Y%Ui0G#?4pyx1510I}_0bOf
z4H>zBnx^gERTX{pX{g=5H$8a{MCssn`1UD~Xk6}`ZQyEb$(X+dD2ms%F|L>)sYdZb
zAEdi1%Ml95Q5O_fzD$j4AY!@D96CRovtG5_JZR)uP9-oETiphReKQv)awZo>!%pGC
z!8RVZ`Mp=E*n^jiuoAAzHuyhTH%Dvo$imk(+Ks%Q`PWa*9GbW;*AHx5!Ei~tnIPLV
zA+T-Yp2Zzqd5+unJ5GT27o&M&MNYzy6fF!%rNhI$0SPMIVgr`b%EN%Do9>^L(z2sv
zu#<=mzWOSp!mPu1@5`|7!DFwtCk!u~nrpfqf2g%;Vc2_`bCSKrtioJ%p?UXYH?2ZJ
zYyQ4poXAqI=mLxndB0;!1VzUJn9|*r_t!ggumq-U0NI$|6C+P|F`r@*4w@QwcLt4C
zBorW89AJS@Wec1o?rk5uYj?Mtv*mxk>fUZaUTFFJNZFG*==&{hgY~{)^X*;54H`dx
zTs7E>2~K)LCc9dwD(T?-j!vGLiOp)3J1CO-0AcuVQ7E}}S+Dj3x!5EztC
zsLHX8djh^jghWh!`zj!4JDytneci40?M41yg;f1*lh7SipBruiN9MLaqU0kk~c?Ozis_jcR#_siN&
z-T|+6!Gor}_W0MSoxRHWkvAa7`57r|X1HHttXg*Rdet9&YF-rouXxx0Q^Yl^bFN1A
zXNFY*+b6aFM3k_X#rmMXoj&}eWp8Z2S(@gbU$5F|nIRMJPc2${2Y=ti2Kl^4LB*7Z
zw%4jLw4Gm*XeSgC(N2;FGafHXn>^S_XixOvWoZioI}z^|?Dhl8dH^%i0yN@K#caLz
zC!y`RZ{&=^-0f9$1-?>_uqcv6tTlbzC3HAT-X!em>-1+-jwt<@6Bze)!%o#
zCpE9{h9!|{=_0-R3PO+^?PvQfMSkN4uyI8xfmh`^1kA3s(!i{jhZk)Zcz8*mwU)`H+bZSq6&U1g+T~8D*kOW^eq}LxuOQ`|)m=T4JMW4O@pxyDD9d0a)}wB_XU}y%Qo>L_sXbw?MM;2Na
zLHh@JIu=*;E3SRt{jOEW(`#hvauGoqL`gLDXRZf?csVOPD$SC|^KQL3Et9!5iqDcA
z@8`Fa2adBk*ey!a67@m(&j%kwOE`wk_C6UZd>=OhwVaR&n>ic%6&}`B0C=o+HG_Y0
z+=Q71LZLX$7hi+nVu4vIoNgJ>y!2hVns(}`6W6B-$nlZd_UWSg+pBX%5hi~`(j~nY
zU=o0K!wQdd51y{>ABbN|o|FRLUk5
zLec&0Rw5WJA8$BE*5#)a+C5yB}!3$`|qw)7Ygw83_
z@WsiO?L)rET&=RT<{-F!P0M)+6`zPcbi=UZb&f4J)#e8N`tgsz*vH%4q)z56_&@7S
z5)9@R+<4RQ;1u95Hg_|r8UO6$j2{x1!HK)XXju_|yFr~T_E1ag7I36b;%2=rwqEx7FaPG19{`>bpb_E0}8nLQI;_WkiSP2`?Vb
znM_~XdNYbpog+=^?Tn2&rmatWmM0a86q@qSz!IG_4Bc!!_
zRwbIPUKw#q`ZKoa_#SeJIIB484hRWu0b?emBq_g3-(=>?+SwZkwWYgio${Xd=31TdEmz#(wl{*Qtaa%ZTuO82>x=xB
zJ9;9tQ&$UJ+W!J@XYq_{d(dt)?V9wD>pGp9TKT`=O#&}&|K3crwTvuq+@W0xHhTyn
zXS?}7yG6U*WBawfzk%I}d~s8vkih1zBeWmymK}Th>eA|v>rWc=TxX9I8fS-m72Fq5
z5AE~TfB&*C)B)7P#P8oAGbTrAl7mXjDV(+q;%vSFJPkwyU8_?*aj&ru`WBH>-k_jm
zd#yM(3qN;X>Ti!P3CDXPKjL;VN4K=Y|1ijY2@WnjyJkZ1(#n
z&67bEy21OkS_eCSIyQap7fIOLVyR#aAyu|WgBlNgaY@bS`t$!0^4DJXh9
zKPt=^K&Awokt2vtWHO`p@m3Q1-xIFdK!;amnh8TL$7&iepsg+Wr3XKv>tOodDUGa7
z@TU#wBkP2z&C6;G&=n~V=2ai+F6x0ty!mbAnrJfrA_qk4pYU-rLkCWpG9I=0f8|OZ
zT8@8Yf!{MV&*3|P?+VE=e^85*Tmhbz9V7K*H1+PlCS~r=b!}Utl_8{vW%u|Nqk`g*
zZ}c8o=T8r#D!HeI>OjS!&V|qJZdfmF1}cdfS683;SJy{pD_9m_Lwn2J{LjrC8JoSG
zwRQDzF%czx?Tko?m(H|43j_BzWS!Bm1mD*{!PtD&-^l#Vi2UbfsIU8*K3(LSiC^2A
zxw1+RU|}Jh95)lYpYODB^VopR^V@5;jmeolkUK1K-dJobdIlSu6dbd!@Xr>Kl0fU%
zKyysyWc=pLzUgezeYcWl@@<%Kt)uk(h?q#(ePWK)%)g;CC@
zJJzg@nEMxZM^XPI?ei#t{rc9eH}mFp|G!*eT@?T13J>BLC#cV3AD}+c`&T^tu2<(F
z=As+C`bxFfg%^=>r!D0+OY8)NIV=!VeC3!lDfD7qJY9Z3b>vB0}~*iQpzShW5E6f(-|-&zcofoz)AT
zODnsgNPD>fT?WFENg${DfqW}V%S*
zn4~r6;w>{#`JYTYGqd|?4;ly|Xf&?e3qE;QvLyAe!MJ4B`3({Kz=>secK?KzRh=_i
zf%g9@{uK-QmK<@1+FGX-adGlv1MbfIrDE2?ev6-;k-iu8udPyb|6!!g0jaxwQm90V
zfJy{?bt7P?T*Ot
z^ax%+Qu4nvb8PJu!C6pJSSDe<%s=Gxf3mGVvHu|Yf61B%
z_`d+|i{Y1F*wv93pH9SSrI4aiN-uRh6k9O!RJx~0Ey-F>&czDa`^VcJL7O{+0Hm<@
zc?T{%l92OL0LIm^x?in!_Z?6yRJ&r$-COgxc5iM$=wxo&-wE#oaE6J8m9h7(XG6c*
zRsm}{zP?*fFfI2%_XS0KpHuCL*tEurpwGBX>9sF=v!w6Kx^=ksf_Lu?>~oLTcctZz
zEG6F)-?tFP94T`kic*=iJ=+LJOo*yWx-7hE1QWAffSZ6a^xB&O+4
z(jOD;BU(XWjb0x<=MP8mzjm5gNJ);RBrMkM4Kn#&d%5LkZr<&j%utOPaT2HxPu9Ws
zKYqVgZTb8d6n{df>AW#3*U6N=cbjsSgElws6+6%93`yvg$6eOU!Nkgs$zgY%-Gz;k
zrEUeyx`mDdS_{~jVu_4n1IqVRCNhY@)R;A6a$nky46fLxbs{V5$w#7~`*jR^@}uof
z7x{U+NBVvs8*&Hv`G@w-DP&5bd*5r~^rFDDB6d`sTun1qUHk~rkZ#m5y$d=|gm4Rz
zr1KHLP_{~PG=?z>)oY8c&%m3S3~ROf@@9(9jPFTN`H?X)M&D~ak@Sf;YfqkDH(c!4
zSwb0Ea^BhwySGuudBkqE~F%2QX^U
z)EZuSv2glJ@xbu({gHy=lE$tt&qs5vrncrh?_X5OHUC_hmR@juxt+
zN-}?!CpXDTj%jpKzC;p1&8z6}k+|FhohaGt#JE5LyL7q*9(y91FkUniqQQTkCswcZ
zq@%=b%0cedtb=iAn|dJw6!!C2b32hBq1gWNzn4e_IRwI9s;g5@1}7B>(%7D*rP<6S
zlKjkzHM(-t{O>oNJYEln(3)dT9l-|AC92*&7ye4fEA9@|t*szmK4yIJ`{02Wk)%V6
zxidOXfo^L01^s?|$aApy{#%O+yZxdX)W2^S&HdJ>C-pIfb|WSB`>$70Ee$6|zw8x6
zu9YnYwaXywn(Wm&6>$Ade(3$}VZYDMOdPVsu3J$HeA8&a8Rv~sV{)E-9eYu-+3mCY
zG$dz)BQXkAhca04ap=C}o;M)db~vKG28RstUybR}yC@Vzz3?CILN$brzf
zC~0vm;DWckX^L0EvleIj-{&gZ8kZaAE>VU_qsp>2v>YPKQ_v{WrVIbQxtw?5+p3hnu4W~>bq2_}SX9bt3f
zucW~fUVoig1(NdIiC6NA=9jhR+?&RF-X=U5t(^C0b-TVt=8eYBygMI>_W6}x#Ydxg
zYft{QRq8VD{*E8gsV=YkrEfO5%1T_+vbtsB)k-^hjjmkn|13^QEq6ui+A%qu%`10{
z%TmLQ+zh4Nfdej*`T?^0*wE>+u7MjtBOg|6ceq(L@;oiN%yX_DrO+v)|WX}k_l
znT<|Hyc`G0=6&HYD2;dmyC^uni#hYWvCgW)mgJV0t-Gn@)S%Mm=1p#=o8Oy1B7QpD
zT?D-bZ$7p{S*z;sK$7+0sZl~Ai}Y4nCbMIO-=0&2FVsW+Rjuxogiw;c*S1QBzwiW7
z7eahF&ezVAe677S%a5M<)2kYGKh3+6By5wE-39gbDizGIplN<6yWmyWWbP@Y+X6{~
zB6ZjkG|>QQ7sS}-Jn?S{le6%TyFNVIUykhX2
z_`n6R$tJ1%Ibak&)u!G!Z+!FK%x;wdmJ!!j!#sYrMUW4?tF1@=x`h^EmUcq%^{jq!
z7wvVs{l5tN{r%W>6g;n%Tv!{Qa+9{|~kCFXmD5-N=8dO>^_ntHa3M}w8~#E*DI3M
zlp({$jr})@T2{>NNwG;G?MvO7KJYOsFh8%j$@uNN2C0xAdc39it}ZrK`QcM%Kkufk
zM7vwnZ_sRM&>B%_?$*4(^HrXXB9#8dz*oxtDEZhCeDLHwB}o1Z7+7A)IuJCvJ81E;
z<^Cpz(UhO+K^w1c%k(syi^!T7Yh_H9sKO2_<4ozg1i
z7i{Yx_29@;u&WacHl(VqGJaNnCl?1s2U0;moEKgulgZbb&s&@hTSB#pXNIN(F?agX
zCGIUi+e!6feLp|io*I_>Yggy5)k4Se_Ihg)6PvRD?JjQ7)UQth&Szox?JO?%w?`6S
z(6O*CUXIh3H#@oomrhFS)5SaXGF_PtOr@N}t5i=s?`KfwlMKfda~Vll#kF-wER1KO
zdjYag_mhM2qCHiKKJeFxrf}7kvAD1MW1oY-`{#aB?FH#yZZC&8m$#2BH4$pi+r(c9
z@lMW7+(Ipz7uMQu<14xIQ~eV-@T9Wwd<9v|&VXS7ez
zs{1ckD(fhpYxHrcbf`teMvwdi;b~r)`oh+IU*AWznlhDb+|*efx$#k@0ODvZO{T%O6h
zyZeWP_!5XzvUWNgkUle0$D|v^@Fy*oy4~-(YDkeVDfJdPrv?XXuJ@(CKhQe=A$e4w
z*0bzP>d*E)u_EqwGzmoo3;*4lC7O
z+4D2U_*b^2D>UMRchcz(uP-JSbqm%ta9vAoa-Y}c1e$-aC%4#BG0`NuCc2toXSZz0
z^Ai?O37PdE3^|rE<(YmfhM>G<>B%d>pWj5!Y}n_4{?=1d17TO;{b@n1pTk}ceX40u
z+H-86^^>Z9nQn89v|!`&1zx{21I!B~q=NFWuzb5bqa(hr@#o%54#toZVLY`n=Z96t
zDZ0wBXV2JO1Vx>q6F+`O9#z&_c|`sEx8>%=#Uo9i3|*Q(LQTwCI5$6Q?L54ID4Jeb
z-&ZN%kB=yOjJmnB8G09>7|Exa0#)amUzv3a7R6Cviw3xRHP$(?A47m_O#5RaMMcg=
z2-wrBwKwMn`yD+Xnnji;|HdM+?*I2lcE-w9*v{;Ruk
zr+|S7d28+9I|rq30Gd;@3ZifJ=z=qmOqCD4_Aq?^^bFijP;RpOXX)ZnY5qJv+55zk
zt6cfUlVkMn2z!k>^wc*4Uz?+M*dV(WOm2{o7qI|t2IBRAb&_BPS^8J6hSrwO*gL`=
zKgN7FQF+CwgLQy*hA$=rO&iOdz$}<{@*BhVLslI{sP{ykl_@))%gq7!|Q&we!&L=;PJdrnGG++2q
z&YzkasvO?LUHlQJF%~(nQ`5sb1he7RArE&+^AqsdUy7IAyB4IH6Jl-{$9)c~3K0%d
zTiT07@!QM=G7`Fngf#sTcsf{4Nb;BZ;*}=J#6NCwi^piITsg04Q+v~QO;e<2F1ni|sk+D_}Pbr8t3P+U2QIV*V-R^`-@t
z@A(tj){v?V1#v@PX6|cp2z}Sg{Cj`xzXDqOS@?G8sEa~RVWD*HFXJe}gBSYZCdtRC
zy_0Hgw)f{`5EeC9g`;Ei&1s1jjz<$@-=`O3Tm^HCV6env%sEHjg7f47p*x{%M`tn3
zqouqlcYJ8D0)41{O1F=DJ0U-?@A|jqtFyJ&-iHPBMm>An9*mAX4B*~0PLqy`2LY}{
zcEYQ-PF_?C&IMJLBnI+D^lJ(#vT
zhfnl&vA?~(KKQU~)Tj-fTQQaPyVdn|_Ci^n#N+<9Oj_xDpW_#Y;^e_e_x
zvdw(EqaVFcbK>cJgTZd^uzq%XUy=B&_N1nAj%k&!FFv0n?q8{aH`wc6J~VjDw#$Tb
zthvU=Q_(xIFhq5C;LI%yAnd)xorB!UV<)8Uipp$7UP0+nD9saw6W7t;vb^;A8X=1_Be9QMr62+5o?M|Pt#u0Zb%}OKQSwYr(B<0
z4U~H}J5BwxQTM11yjPi{2&
z{MJmj&LipTS4d__g8Cl%Kh*c66KCM|j7vRbBRS)3P>AwrFwmWSIj}vsv|JI
z2_5g!N;Qs*c59ZkH5Dg1%C!yL*6ygi_u>`a=Qup4^U;8ix@
z;iYT5;WyI|uk4}sHb^n##5qdy`L|Y4U^aSm4G@!0=5wFKcX=06Mi`jYDeEt$pPf0T
z8||`lzj{vr8Rn4KhNxz@xEHC^dtcNICy0xf2o(6s0p}ilMYi2qK}RybDZ|
z@5vlw9;hij{`BxIhOIw&EX7IgdMTsy-myB_d}Kvy{?OEolnmng6s^nYyz;(uI35h$
z;j8`iu$h2I)_Mkjg9A#zDYxr=#T5XNxv_o-*Y=Rr;L(g|Go_VX6Ivds$p9z7q(
zTTCkaoxbxy&UusUpH7I_nY^V1{mbXpf|ki9qYEiv_Qn}*HxR$J8f9L)7OL*`?L6A&
zu*B0i`E|sdDtKVwlN9=;&Wg$QLGCpjl;{@iVtZXs2&b#({&&y3#Ri$eLm^=Vr@n#V@TR-N@e&%*1c6s+X5vti
zV9kLS>_6dO)fvKE{Z%ji=UJK4FL%f8qJcM26xU*pXl)b^3NszQa@)#GQ>Ar|=|WVZF;}2M_(;pxaksN}qPK+3#z-VX9P8GpVP0s9f4In>
z7_1pWP7Nhw3tm#M0C#%-Q{Z$%UAu1QKxfLpp#tryp4py#rbN58LcF}oD6mR{%(y3;
z*=hOOS@WUuWm6%)HKHRYh@nB3`Q;ovnat)I-v`f;y4P-WfR}Qt85?`+GucZk*DGr+
z%!kWbRCj>MkMlUo(40wO9X0*x?bW0bmGKI6?Yl;=puYaHyCnkqt2~bfRo<>8?d{0e
z2{;s4rlj3|UbC~B_I-yicOzu?&^x=H*{yb{Elb|&ytzAJzt2@M+3Xn7zJro81`1@+
z{5bRm$Ra|iBTHyr^F8e5Hb!rs(a+J)mNF-&6F495&XyGiAi8pTJv;gNUxz;7rjjQHZOWd1{3!lSqw`}OcJKCX(82Iw
z&6p&|9X$U?;j3K4z1KO3+qj*xKc}oZi>9B!37e40IzYM8H);G5TL_cyzr?h*x9@7<<kC@H&earFH`O=}wu!!I+-`wlLX1
zhbaD=^)EhrgHbe6cf33kPLnvZJ@ZEISs&4G!V8Rep6E?7*R>m-f9!aJDce6Wek6Tr
zF|~C*f*Q@83+xQh&sRV&Tu0G7{|nZsL4m191)T51IBO`N4=>jA`COZ-3PHZjDMGy8r16U`E(Y
z*#)e)%q7
zqEWtiT^?Uc@2}a+_D~$vy4jjPl8xR~5aJv%6+Ll0`_j*V?M1pKC%;7P1Wh&B>;(G0
z!;%38pk3woPI8AR+#UQoubjl(Oqi+DQt5!FY`YjJg?iU{2sUp$J(M5_()`%5>D7d1
zX>oi0#1ySny@(JuI(bD&_SM?>?7NX3@-rUV*M82lT@_Gd00V?BY49K@;h=oy&gvwC
z4O03NEtcQZI|q)ozRbG+V6ENQb;dL;zED}ANLfpu+m!A-no9>mBd>e7R$a7JcLo#}
z#`8YIbI5`8MQv{zYRRRES4q5R!in?cp}AB?{UC>0Tg+ADQ7{OUxBJLOqE7hQBJn6V
zpj$a7^OhWp>OMCy@b&LZM@QiT$a@OI!;2(>15?XZy#u0
z!58yF6-1;}zx9ik6B98Ihp%iL*enj-_RODNQW_}iyl}!-qGil1D?*_VY@|8z->u@m
z51YawuFb%!P_$29G&{h?3l4p4uh1Npd>WUNnjw7I5vuZG&Fe_HkhNm}{e8#0ndN5b
z6VM8+Bz|H2SKV4nmYPvKuwMKdsBWH5v&XF#jjL=0B{HH!JQZYg55M1EaR|&pWtiY6
zFlrRsDRHrI9VLKi;^{0I(a4!3=-$G5hM-8vW@8YJk$nXu;Sl8QEcJ(GAgBeBs45u=
zJg*#+DJLfqb!#w?NRly31@qwm{nhFJJM{m4UDI3{VB`7Pl%@~x0fPNhV=aX8$tbN-
zLHanL>(gg$t|BU%%YY+&1^Q}nY0&u<3h96E{QigcH+Av>HU(Qz8UMM6{;eaxJ;
zOIUUb{&c33EQ;XZ#m8y`ufY&%ZDS3EqV>iMZtDQ;1|;D3k*ONUL)kM
z)%-{~FAupL5+0z~p9k<2VHJK^;`88$%cm$gaWk-f=i6fGEucgDKVSXDMSg08V2{L0
zr5Ehrr_UyNu2F;n!FI1-Cdq$gCq-ax5&;tdGnZL51Crx`|BKcCZ*iB-)S_NW-uLwY
zU~z`Yddxo1%ia#y`yBYyGvQ(2#b7~IWArj*6wKu#2ePl-MP!
z^T6_=gKH)lwO1UhTTJm?Eurv>C+<*gu~qt4!ixZ-G{d>ancrmDXC-h(bLCm%2UHl4
zO_^nV+~t?W;KC$?Ix7eg<@=yMB~0YbB@3$|0OX|f%(*PafqbQb54`7D*L5U+$w9AR
zp&P(^gLv_}znrpF{@j$w{MP@u79?VG$q483ppep-5uVacGQuWK?0=M>wdA#Df?*Ui
ztr3x_cT@kgzR`N66zx;)(f;FyR7JbGcLAhylRnwkwSZ#GOVoUOB7OR0-YQszaLe~D
zgNyf6aB>X2Y_h`CAf
zhwH+Abi@nEK%N3%$C;bz8-Pm*Un4Q(LD+RvdFG@cdF8&o{3)yx;)kr!qZ4LWlh}7h7(nladB$}Ep;*|V09v{N)!St2&74)yRK_l
z50}0s
zDd0S?JL*ON${~TSxv+`@~8ma8uwxXkVABHUC|1@B{J6u5a|vr8Je%5>5>sdgjZY
z1>~DtbNuPAzO5f`sFry2#HWA6WmlRkQ8_~@cNawNCP*k8p3hYnO&H4Vlu=1gAi
z`(|*#7`lIbV6T^k=f=9`Gl3~{j|D-GU)ObZV&ztqa}=$oil3c)mei%;@id#p%2#wL@}^4HBf_6O()qwfba5$=lV
z+m%*m>ETj1tHE%xx^6i7^Hr>LjUf=|qo5x2Un;pp@ieioRV9a`^a*uf&*6cX`2UB!
zw+!oQ>-L5PQ9zWG5~NG%Qo2JLL}^eEM7mLFkd_dTjz1+xiM$e(`4
z^4|N7_ndQG&vQP!pWZKA>@91pImaAx^!SbL81w4aDFdDhU&lgvZ0#1#VW>=5DHYu^
zxZBHr-(QLr3w?1In*$#yeb?m0SeOd;?==zx1vucC8*dFFO6P0b;2-W$Y6?}|_KVqJ
zXO~#(i8v#;=hR7a&$51Jp2pvjxrE8eLaC>goQ6nP)vxCTK5NJ~V9F_69&BusnIv{7
zQdga&*cC|W5ljhjJVGje^%?#f_z^V}{>&wJKR$SevgEx_J;2grc`F=&T99(VN_vK*
z0g&>(YXtw4m9R=4A^!@)(J_Kpb0#)!99PE8!@*B)I*;#nJu81Aoyi7dLTW4!RawZ|
zL~SIlZBUG)5y8^Y5{5JW64OhETo?rqCRD8(#*4l$cF2^zDj(!L{!Zc;hmeEG`vAn0
z89Ytxc}_^grzJlpEKH1prSmhM-!!DyK7##lKCw3o|4~8iO_eLm%Nnw4VQsREO_ra(
z@qKv$)I#C#U?09L2{n`VQg~?_9sDKm?+yhi3`|QUzR|G{OdYp0O$e71rnqA#n3hpY
z$E0@%m!6$L(I5J?W+AJ;&H}6F)l4b+tNH6TC#>H3Y3~(e^?9)R)Na3f$EU`z`w<5U
zN@WM6BHwQyw7{yb^8aF
zJkpsOm|nrFBorY?OU~;TXIvbEc&rLVK-R42_1<;^YrW`!k_z$?Bgh)K(&PEG+^4ZN
zhZIz!!XXi#mWUj}ILyy9VS7tJKgC36$@3BD9Y&3E`HnzJ!!}CpfA-^++Iu;~?1T&s
z85Y?jBrwneNYOo`^~Pr>@5gN#(hUjOJ$%tLCinchT%q16z(^N->4n4|U@E`0jk1SV2&)W(4&O%Vs*ExNsAirm(sn
z*||vGiH*sur~4MxbUijBVJ9N@?nf+t3EWM}w;K1Uk^X^98PKsT@+59MAVNf{R95YX1WS>}WnfUZ~lmt#A{#42_zUC&UwNd*C5
zzrg(aHTv5X&vQIJQ1T}L2ltgd!E?QcjrpDhTo8{w982t|GHIid^q%s#IyWhV!Cux>Mw`{5ueam1p`bAhTJ#d{X8)YE$
zM#?Gr2@Ab{dB}UeC7Ql$I)%pvnCgtNu13peI@19t6PY*
z5-Z2A3P-C1)Kkw3$^;+9@1H&jP>v$4$*RO|{a(V~E6>QCR66jDojA*av!evZ=Z!h*
zh^&s3cxqbVKj}XuYXf^g4$%(}>q}@)1C#enOTx!W2f4Y2>=aW=2``SLat)bE*q49C
z;+MWYb`7O{=~~akU%Ggq#Xjmot=pI-ld;YeHHu)LTxn1Fy+-f(OP2yuwrdtZNPSBU
zwns=-nK5TS$Ex9$Pn`y>Pp;RmV)ZdJh3F{%k)nE7U&1w9CfoHXnH1EI4QVmUslFGZ
zsacMF0AVC|!6z{bPvff>qmqL4B&XG^DNDA>XkPVeVM&`PkRuINOMiM55j#5`_-CNA
zyD{S`t}?xrlnn-uI-L|hic{9a!yRaGPs
z{c`7=63XFtthlhAv1(>pU&&JCKtpMF2Se#u#&4^p&U}!K=
z_#wN}<7%C(k$1Kz-_E1sO8n!UPd{Egm>j)wSjKEOqIti6%x22FSXe^##!gL{u!BquD2Ag%teYCfmr@MX$s(&0;q
zVd5x-%oC|zazo5PUTs2_w}g@*03f(9IiZte6bZ6y8X5^;D5?82siJ9VBq-4eb`{Bf;;cCS(QMC-&ye(*=_!?~+9OwUaO)Xmk?cH-xuKOo!p!-?wJ4V}
z1t+uW_PZgS$JX{jdrh3F&JwbJBbJ3bsgGlQt*nGNwyfG-Heg#;m+d_H8mjTB`NEK{L1YT&=|lF$
z{c-))XBh__T|808ez~Xi-jq6O^{;Dv`zsv;+but{Oz`gDg3Or`gj;hbAP%I3dLHvZ
zW)e>hH^vz_34bCs2vHPmtPcLt)+RWDCM!mXu4A>X>&DwvZn7KCR$(LRBC4?jZ`1z|
z7)|pPTEKPHTOg8&pWwz9&u>JA%0^T5ORNk{!kxHhc`1a79REqie?R6$3=`)9+x%;sv
zWx1z)-{>ILZoOU=_|C}1E7*2PO|}L(ccv{z#xMW!c;GfPVx0kTfu)cMI+aNNAWK=I
zZPL7RR3(LV*+WTGfHKt5Tid0KIilx91}WDxXe4Q(`}p|R)2A<<-=b9KlD&dU^6=KM
zIS2CgTw#Q#l=+ASa&>lQ1qBHMJS^A}>d{zxgf
z9oiSvAq*puoUruFtU2QE<J~Fs=WiTr7@K%#_1k|1cD*x
zonmub(4nD$;0MMqjntSjKu0Px$?%!B#aJo;BeuxGTPQWieo`*CGk3+i#^-kIx+-yF
z5KZx$OLdroX7+mwnz=zmpBjS^I%_H%a?G1|I3KhiY?3MJuMPFK>x#=&b{l^*?0RhJ
z3)fe<8h*H#-o)?0{?;cZf2zoO>N)oX}2{@zvnC6^dnuGCYAP;wq)IT>_88OYn
zH^8}oNWfcu
zxP7g~gd~V{`w=&dt;5zuQs?hgYsISDd#j!MUm3v)FH${t1I9v~!Ho$)9G9%tKV5i*
z%;2~HPDv3I=vOej5YWB`j*GHWF9ouKHpK1H@V|*c&Ap7JRKcXS&0>w|76gwQcVIbz
z>fi-CIEwctO$6mXI@w}y`I~e3&EcpXVOAIx?M2y?EZ}*(xmbD`0)yDttuGyc;lX~2y
zvC%%U{?B7Sx0xKWSu}MKGa9~#4Q4bW92rH$`o|<96b;k0X2N8iKYc1^0J#p@Ty>ef
zMFq3fi_o3r3vNn0IhJ%5xM>W_x0Du^*ImBcXxo=2`J8Ga;vH;Wiu?XD1vQ8j90(p<
zRl+ij0Og*`@b8X+6fY*bvd5iXVx@-?H!%gESurq)%PJywE7pW8dV>e`w@c?3;g{a7rl$}v!MXe(c0qZhl3BE?r}L9dXLLCGkxPPf|ASsSc>6RZ_Uo_^%pmQz_Lczc=V@>lD0v31I~Th^gIUw-;dt9j*WJ
za{GW1xLWEXJAZVl2d2)WdA6CLyNkqVC!iK-+r-8{1OL;CP
z$)@lxZ}>a}29Gl5gd7lrM%h-C*|R`&YC`J#KaN|3RK_>6s{98^&P-|QGvNanQB0q=
zc#>?WH3+BS540c$AQ3Kbc=J|EzOV3edNYkCd;ZW==|iZ+
zW<-Sm#zIOjjumXnPwE0nw85vK3Mn<|m4kG&J53
z^fe|TvGNV+lv-@Ho3NglOY9k!$Uj*%DLP`Xoc&GWKHkLzd9b4tp7?5W5Aqjt=LIxL~LN9}bRy0>nexzV$gFKp!jrifm8O`kQkl^FC{3
zKY0}u`hew)>+k=8;#amjPPgX>cXz
z^7`L*I?P*HXX2dyEEfZ34mwtEQv$u^k^Bt~Q`a4&%mWdx!SIkEwbePya`=c!U?$M?Tx4CHT*-*Za*Jfnm>J2{Z_!{sVqTDia&>r_A6$-;o1~giH0R
zR>8jx{K~DehMISsPcb^sEd=MfO4@`*j5(=HR`@RsIcoU*GLo9$Rj0lS=yH}_P0OW(
zAnaw5gaD^>Njmcl+6x_{h#(8DbjWKoly(3NWB+>GW6n#5{e9j2+Rw>O|
zaPU960srTquAE|~!bwn2mbte+n{oabX^
zk`~?o$dd*9dnPN>d29bm?<*1a5mmdl--F*T#j3=WY`M!@xEt=bQP~NZ9X`?VGQg-^
z&fBmnYwWbIorIb}1svUq@a+uGjXo9b55*{>Lv0IRd+mO;w<)}0JKsKyI6m`L!q<4-
zqm`q`JUa?_o{T#0+{4Ewt~5mrWH^}dmpzU!R0DuL>Jo8NnYYK{4QoD&(d$BHqrTtE
z_jQBi5Hd8`!A+^zmUSiE3K`=UfE8V)?e^P+o7TPnSif3JB8clW4Gsuj$lI`=CeiU+(7=T4fM|czD%hUpl0sdn5SKiG(_iYowtNxTr#D_J;
z#AB>nUDsoKy!`N`94QhtcD=S|))cD2-7EKk&=K@&-~MGHB;f;YGQe^UyPjX7#r%j^
zO-Xlx1)(foZ~vl&fND~fd90IErap*y$aRbU`THmT0Mo!B!t{Z^pNJNdCug4FfB({x
z_dbj2|NnlR|1as2#MR+-k}-VH7yUYn?+KjJl=kZU8K_9fi_D~-qK4Ru%QbqsKiuIM
z35Q-4K}djA50-U(O#z;$V~gdF{arU#)uhIUQSa!d$l&h{_vyf_2lo3H`uP!vuzJ)}
zJPGR-%gt-Qo&8jk{{8)wK2l|k2Zb3PE7v#;DqkNjLi4ep`>I|^^@ufNV}ky}xON-W6QL7ds>GbV3+6c{YoV6Z^#B@$OSf>hZKw
zJvV;bKWo>xztC}^t9VLal6<&1V_e3z_4WQX`3H2+`iIjK_t#3FL;tP?@Kp%7f??fQ
zJ^YZWr-0Z;8|(FA-se6CC@||W2}BD+b{diU=cJwV{*nbXW=H{`(+nGEKCn*nLr9(b
z)NVqXW%0V#Oc?73;}fLg8Gs2{e{Fehcusy(bpKgWQ~m1aSpqRXtg9khJffKypE_+G
zfouv+&}uvD-v(y+B!!2_`X8j4e+5qdY>>_J=A>5z(pgQZWq$8xX`MZTNaW_2ZAI4=
zsxE-S8X?~Kh)rwTZmQ$kogr>$XjgUjLHYOcRIJyQH@9W(?1KQi1En`my;VBPj;X{y
z0A@OpTQ8)CAmSxO1Al0RCq&WQY1;5TJ(PQr;WntRREf0J1EgRKP7i1GkLFq;JzS4g
zQr$<>%|S|DNUO8D*p?h;-Rj%!Ru!pT;>-+0KMX*c?2pZU2E*y929eNzhqO_QHM@Bw
z`KCigo@(-+p8e&y*EvzjI+~5kY7SRS@=2?XpbPjeboLi7bSB4DK3c!&0nNxZ`RVS3
zud!3MBG2ru8n%TF!5(ho0fWn9J$a`EHEDSUa9l{3@}C-`1!6ACj^~U3y=w$8(c*9(
z|1b@adt(27IS*ekjdtYfe*nZDAvZzk!1w^MGF_W6u$dH->}D*}oPu*=$OwME
z`Gjv1JsW_>^$ifb43EsCc2Z#FZU;CfP|eH-&eC-Tuv-3e;gF}EzQs-$igX=)yzp*1
zm`cE^H<#(HRbL)6`B+wcXM=KAgG3kfz8`;9lpy}{jY1I{v!DMJ5k!D3arDhE5;_I|
zr~XeRkJv6Z{uAc&e&D=J3hx622A8ht<9o*|y7O0NZ5MGhd%E*O&)~sflKHjJ1C5Cg
zXD+LSw+eCQqSEm3pbF|AxYGU^z&YYX1D;LS^3$?&q9qIf0-|xfZllpdw5;uEF<(SI|(T+V9?kLTrL(cK!9;Mq`PVVY0}NW#dhUb
zT0Dt)?Z>-m4Fy