Merge branch 'develop'

.travis.yml

@@ -16,8 +16,8 @@ matrix:
   # On Linux we install latest OpenJDK 1.8 from Ubuntu repositories
   - name: Linux x86_64
     arch: amd64
-  - name: Linux aarch64
-    arch: arm64
+#  - name: Linux aarch64
+#    arch: arm64
 
 cache:
   directories:

acl/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
     <artifactId>rocketmq-acl</artifactId>
     <name>rocketmq-acl ${project.version}</name>

acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java

@@ -130,6 +130,7 @@ public class PlainPermissionManager {
             log.error("Parameter value plainAccessConfig is null,Please check your parameter");
             throw new AclException("Parameter value plainAccessConfig is null, Please check your parameter");
         }
+        checkPlainAccessConfig(plainAccessConfig);
 
         Permission.checkResourcePerms(plainAccessConfig.getTopicPerms());
         Permission.checkResourcePerms(plainAccessConfig.getGroupPerms());
@@ -251,12 +252,6 @@ public class PlainPermissionManager {
     }
 
     public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList) {
-
-        if (globalWhiteAddrsList == null) {
-            log.error("Parameter value globalWhiteAddrsList is null,Please check your parameter");
-            return false;
-        }
-
         Map<String, Object> aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName,
             Map.class);
         if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) {
@@ -266,9 +261,10 @@ public class PlainPermissionManager {
 
         if (globalWhiteRemoteAddrList != null) {
             globalWhiteRemoteAddrList.clear();
-            globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList);
-
-            // Update globalWhiteRemoteAddr element in memeory map firstly
+            if (globalWhiteAddrsList != null) {
+                globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList);
+            }
+            // Update globalWhiteRemoteAddr element in memory map firstly
             aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS, globalWhiteRemoteAddrList);
             if (AclUtils.writeDataObject(fileHome + File.separator + fileName, updateAclConfigFileVersion(aclAccessConfigMap))) {
                 return true;
@@ -362,15 +358,19 @@ public class PlainPermissionManager {
         this.globalWhiteRemoteAddressStrategy.clear();
     }
 
-    public PlainAccessResource buildPlainAccessResource(PlainAccessConfig plainAccessConfig) throws AclException {
+    public void checkPlainAccessConfig(PlainAccessConfig plainAccessConfig) throws AclException {
         if (plainAccessConfig.getAccessKey() == null
-            || plainAccessConfig.getSecretKey() == null
-            || plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH
-            || plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) {
+                || plainAccessConfig.getSecretKey() == null
+                || plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH
+                || plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) {
             throw new AclException(String.format(
-                "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6",
-                plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey()));
+                    "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6",
+                    plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey()));
         }
+    }
+
+    public PlainAccessResource buildPlainAccessResource(PlainAccessConfig plainAccessConfig) throws AclException {
+        checkPlainAccessConfig(plainAccessConfig);
         PlainAccessResource plainAccessResource = new PlainAccessResource();
         plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey());
         plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey());

acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java

@@ -566,6 +566,16 @@ public class PlainAccessValidatorTest {
         plainAccessValidator.updateAccessConfig(plainAccessConfig);
     }
 
+    @Test(expected = AclException.class)
+    public void createAndUpdateAccessAclNullSkExceptionTest() {
+        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+        plainAccessConfig.setAccessKey("RocketMQ33");
+        // secret key is null
+
+        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+        plainAccessValidator.updateAccessConfig(plainAccessConfig);
+    }
+
     @Test
     public void updateGlobalWhiteAddrsNormalTest() {
         System.setProperty("rocketmq.home.dir", "src/test/resources");

broker/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

broker/src/main/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransfer.java

@@ -53,6 +53,11 @@ public class ManyMessageTransfer extends AbstractReferenceCounted implements Fil
         return transferred;
     }
 
+    @Override
+    public long transferred() {
+        return transferred;
+    }
+
     @Override
     public long count() {
         return byteBufferHeader.limit() + this.getMessageResult.getBufferTotalSize();
@@ -76,6 +81,28 @@ public class ManyMessageTransfer extends AbstractReferenceCounted implements Fil
         return 0;
     }
 
+    @Override
+    public FileRegion retain() {
+        super.retain();
+        return this;
+    }
+
+    @Override
+    public FileRegion retain(int increment) {
+        super.retain(increment);
+        return this;
+    }
+
+    @Override
+    public FileRegion touch() {
+        return this;
+    }
+
+    @Override
+    public FileRegion touch(Object hint) {
+        return this;
+    }
+
     public void close() {
         this.deallocate();
     }

broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java

@@ -47,6 +47,11 @@ public class OneMessageTransfer extends AbstractReferenceCounted implements File
         return transferred;
     }
 
+    @Override
+    public long transferred() {
+        return transferred;
+    }
+
     @Override
     public long count() {
         return this.byteBufferHeader.limit() + this.selectMappedBufferResult.getSize();
@@ -65,6 +70,28 @@ public class OneMessageTransfer extends AbstractReferenceCounted implements File
         return 0;
     }
 
+    @Override
+    public FileRegion retain() {
+        super.retain();
+        return this;
+    }
+
+    @Override
+    public FileRegion retain(int increment) {
+        super.retain(increment);
+        return this;
+    }
+
+    @Override
+    public FileRegion touch() {
+        return this;
+    }
+
+    @Override
+    public FileRegion touch(Object hint) {
+        return this;
+    }
+
     public void close() {
         this.deallocate();
     }

broker/src/main/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransfer.java

@@ -53,6 +53,11 @@ public class QueryMessageTransfer extends AbstractReferenceCounted implements Fi
         return transferred;
     }
 
+    @Override
+    public long transferred() {
+        return transferred;
+    }
+
     @Override
     public long count() {
         return byteBufferHeader.limit() + this.queryMessageResult.getBufferTotalSize();
@@ -76,6 +81,28 @@ public class QueryMessageTransfer extends AbstractReferenceCounted implements Fi
         return 0;
     }
 
+    @Override
+    public FileRegion retain() {
+        super.retain();
+        return this;
+    }
+
+    @Override
+    public FileRegion retain(int increment) {
+        super.retain(increment);
+        return this;
+    }
+
+    @Override
+    public FileRegion touch() {
+        return this;
+    }
+
+    @Override
+    public FileRegion touch(Object hint) {
+        return this;
+    }
+
     public void close() {
         this.deallocate();
     }

broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java

@@ -180,8 +180,8 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
     }
 
     @Override
-    public boolean appendToCommitLog(long startOffset, byte[] data) {
-        return next.appendToCommitLog(startOffset, data);
+    public boolean appendToCommitLog(long startOffset, byte[] data, int dataStart, int dataLength) {
+        return next.appendToCommitLog(startOffset, data, dataStart, dataLength);
     }
 
     @Override

broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java

@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor;
 import io.netty.channel.ChannelHandlerContext;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -257,8 +258,8 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
                 try {
                     final SendMessageRequestHeader requestHeader = parseRequestHeader(request);
 
-                    String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic());
                     if (null != requestHeader) {
+                        String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic());
                         context.setNamespace(namespace);
                         context.setProducerGroup(requestHeader.getProducerGroup());
                         context.setTopic(requestHeader.getTopic());
@@ -288,9 +289,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         switch (request.getCode()) {
             case RequestCode.SEND_BATCH_MESSAGE:
             case RequestCode.SEND_MESSAGE_V2:
-                requestHeaderV2 =
-                    (SendMessageRequestHeaderV2) request
-                        .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
+                requestHeaderV2 = decodeSendMessageHeaderV2(request);
             case RequestCode.SEND_MESSAGE:
                 if (null == requestHeaderV2) {
                     requestHeader =
@@ -305,6 +304,79 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         return requestHeader;
     }
 
+    static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request)
+            throws RemotingCommandException {
+        SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2();
+        HashMap<String, String> fields = request.getExtFields();
+        if (fields == null) {
+            throw new RemotingCommandException("the ext fields is null");
+        }
+
+        String s = fields.get("a");
+        checkNotNull(s, "the custom field <a> is null");
+        r.setA(s);
+
+        s = fields.get("b");
+        checkNotNull(s, "the custom field <b> is null");
+        r.setB(s);
+
+        s = fields.get("c");
+        checkNotNull(s, "the custom field <c> is null");
+        r.setC(s);
+
+        s = fields.get("d");
+        checkNotNull(s, "the custom field <d> is null");
+        r.setD(Integer.parseInt(s));
+
+        s = fields.get("e");
+        checkNotNull(s, "the custom field <e> is null");
+        r.setE(Integer.parseInt(s));
+
+        s = fields.get("f");
+        checkNotNull(s, "the custom field <f> is null");
+        r.setF(Integer.parseInt(s));
+
+        s = fields.get("g");
+        checkNotNull(s, "the custom field <g> is null");
+        r.setG(Long.parseLong(s));
+
+        s = fields.get("h");
+        checkNotNull(s, "the custom field <h> is null");
+        r.setH(Integer.parseInt(s));
+
+        s = fields.get("i");
+        if (s != null) {
+            r.setI(s);
+        }
+
+        s = fields.get("j");
+        if (s != null) {
+            r.setJ(Integer.parseInt(s));
+        }
+
+        s = fields.get("k");
+        if (s != null) {
+            r.setK(Boolean.parseBoolean(s));
+        }
+
+        s = fields.get("l");
+        if (s != null) {
+            r.setL(Integer.parseInt(s));
+        }
+
+        s = fields.get("m");
+        if (s != null) {
+            r.setM(Boolean.parseBoolean(s));
+        }
+        return r;
+    }
+
+    private static void checkNotNull(String s, String msg) throws RemotingCommandException {
+        if (s == null) {
+            throw new RemotingCommandException(msg);
+        }
+    }
+
     public void executeSendMessageHookAfter(final RemotingCommand response, final SendMessageContext context) {
         if (hasSendMessageHook()) {
             for (SendMessageHook hook : this.sendMessageHookList) {

broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java

@@ -116,6 +116,7 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
 import org.apache.rocketmq.store.ConsumeQueue;
 import org.apache.rocketmq.store.ConsumeQueueExt;
 import org.apache.rocketmq.store.DefaultMessageStore;
@@ -235,10 +236,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
             case RequestCode.GET_BROKER_CLUSTER_ACL_CONFIG:
                 return getBrokerClusterAclConfig(ctx, request);
             default:
-                break;
+                return getUnknownCmdResponse(ctx, request);
         }
-
-        return null;
     }
 
     @Override
@@ -459,6 +458,13 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
         return null;
     }
 
+    private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) {
+        String error = " request type " + request.getCode() + " not supported";
+        final RemotingCommand response =
+                RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);
+        return response;
+    }
+
     private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) {
         final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class);
         // final GetAllTopicConfigResponseHeader responseHeader =
@@ -1439,7 +1445,7 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
 
         java.io.File commitLogDir = new java.io.File(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
         if (commitLogDir.exists()) {
-            runtimeInfo.put("commitLogDirCapacity", String.format("Total : %s, Free : %s.", MixAll.humanReadableByteCount(commitLogDir.getTotalSpace(), false), MixAll.humanReadableByteCount(commitLogDir.getFreeSpace(), false)));
+            runtimeInfo.put("commitLogDirCapacity", String.format("Total : %s, Free : %s.", MixAll.humanReadableByteCount(commitLogDir.getTotalSpace(), false), MixAll.humanReadableByteCount(commitLogDir.getUsableSpace(), false)));
         }
 
         return runtimeInfo;

broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java

@@ -178,7 +178,10 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
 
         int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();
         if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {
-            maxReconsumeTimes = requestHeader.getMaxReconsumeTimes();
+            Integer times = requestHeader.getMaxReconsumeTimes();
+            if (times != null) {
+                maxReconsumeTimes = times;
+            }
         }
 
         if (msgExt.getReconsumeTimes() >= maxReconsumeTimes 
@@ -277,18 +280,26 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
 
         msgInner.setBody(body);
         msgInner.setFlag(requestHeader.getFlag());
-        MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties()));
-        msgInner.setPropertiesString(requestHeader.getProperties());
+        Map<String, String> origProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+        MessageAccessor.setProperties(msgInner, origProps);
         msgInner.setBornTimestamp(requestHeader.getBornTimestamp());
         msgInner.setBornHost(ctx.channel().remoteAddress());
         msgInner.setStoreHost(this.getStoreHost());
         msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
         String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
         MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);
-        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        if (origProps.containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) {
+            // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message.
+            // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it.
+            String waitStoreMsgOKValue = origProps.remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);
+            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+            // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later
+            origProps.put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue);
+        } else {
+            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        }
 
         CompletableFuture<PutMessageResult> putMessageResult = null;
-        Map<String, String> origProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
         String transFlag = origProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
         if (transFlag != null && Boolean.parseBoolean(transFlag)) {
             if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
@@ -609,6 +620,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
         }
     }
 
+    @Override
     public SocketAddress getStoreHost() {
         return storeHost;
     }

broker/src/test/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessorTest.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AbstractSendMessageProcessorTest {
+    @Test
+    public void testDecodeSendMessageHeaderV2() throws Exception {
+        Field[] declaredFields = SendMessageRequestHeaderV2.class.getDeclaredFields();
+        List<Field> declaredFieldsList = new ArrayList<>();
+        for (Field f : declaredFields) {
+            if (f.getName().startsWith("$")) {
+                continue;
+            }
+            f.setAccessible(true);
+            declaredFieldsList.add(f);
+        }
+        RemotingCommand command = RemotingCommand.createRequestCommand(0, null);
+        HashMap<String, String> m = buildExtFields(declaredFieldsList);
+        command.setExtFields(m);
+        check(command, declaredFieldsList);
+    }
+
+    private HashMap<String, String> buildExtFields(List<Field> fields) {
+        HashMap<String, String> extFields = new HashMap<>();
+        for (Field f: fields) {
+            Class<?> c = f.getType();
+            if (c.equals(String.class)) {
+                extFields.put(f.getName(), "str");
+            } else if (c.equals(Integer.class) || c.equals(int.class)) {
+                extFields.put(f.getName(), "123");
+            } else if (c.equals(Long.class) || c.equals(long.class)) {
+                extFields.put(f.getName(), "1234");
+            } else if (c.equals(Boolean.class) || c.equals(boolean.class)) {
+                extFields.put(f.getName(), "true");
+            } else {
+                throw new RuntimeException(f.getName() + ":" + f.getType().getName());
+            }
+        }
+        return extFields;
+    }
+
+    private void check(RemotingCommand command, List<Field> fields) throws Exception {
+        SendMessageRequestHeaderV2 o1 = (SendMessageRequestHeaderV2) command.decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
+        SendMessageRequestHeaderV2 o2 = AbstractSendMessageProcessor.decodeSendMessageHeaderV2(command);
+        for (Field f : fields) {
+            Object value1 = f.get(o1);
+            Object value2 = f.get(o2);
+            if (value1 == null) {
+                Assert.assertNull(value2);
+            } else {
+                Assert.assertEquals(value1, value2);
+            }
+        }
+    }
+
+}

client/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

client/src/main/java/org/apache/rocketmq/client/ClientConfig.java

@@ -178,7 +178,7 @@ public class ClientConfig {
     }
 
     public String getNamesrvAddr() {
-        if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.validateInstanceEndpoint(namesrvAddr.trim())) {
+        if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.NAMESRV_ENDPOINT_PATTERN.matcher(namesrvAddr.trim()).matches()) {
             return NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(namesrvAddr);
         }
         return namesrvAddr;

client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java

@@ -236,11 +236,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     private boolean unitMode = false;
 
     /**
-     * Max re-consume times. -1 means 16 times.
-     * </p>
+     * Max re-consume times. 
+     * In concurrently mode, -1 means 16;
+     * In orderly mode, -1 means Integer.MAX_VALUE.
      *
-     * If messages are re-consumed more than {@link #maxReconsumeTimes} before success, it's be directed to a deletion
-     * queue waiting.
+     * If messages are re-consumed more than {@link #maxReconsumeTimes} before success.
      */
     private int maxReconsumeTimes = -1;
 

client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java

@@ -110,32 +110,12 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService
 
     @Override
     public void incCorePoolSize() {
-        // long corePoolSize = this.consumeExecutor.getCorePoolSize();
-        // if (corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax())
-        // {
-        // this.consumeExecutor.setCorePoolSize(this.consumeExecutor.getCorePoolSize()
-        // + 1);
-        // }
-        // log.info("incCorePoolSize Concurrently from {} to {}, ConsumerGroup:
-        // {}",
-        // corePoolSize,
-        // this.consumeExecutor.getCorePoolSize(),
-        // this.consumerGroup);
+
     }
 
     @Override
     public void decCorePoolSize() {
-        // long corePoolSize = this.consumeExecutor.getCorePoolSize();
-        // if (corePoolSize > this.defaultMQPushConsumer.getConsumeThreadMin())
-        // {
-        // this.consumeExecutor.setCorePoolSize(this.consumeExecutor.getCorePoolSize()
-        // - 1);
-        // }
-        // log.info("decCorePoolSize Concurrently from {} to {}, ConsumerGroup:
-        // {}",
-        // corePoolSize,
-        // this.consumeExecutor.getCorePoolSize(),
-        // this.consumerGroup);
+
     }
 
     @Override
@@ -411,11 +391,11 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService
                 }
                 status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);
             } catch (Throwable e) {
-                log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
+                log.warn(String.format("consumeMessage exception: %s Group: %s Msgs: %s MQ: %s",
                     RemotingHelper.exceptionSimpleDesc(e),
                     ConsumeMessageConcurrentlyService.this.consumerGroup,
                     msgs,
-                    messageQueue);
+                    messageQueue), e);
                 hasException = true;
             }
             long consumeRT = System.currentTimeMillis() - beginTimestamp;

client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java

@@ -487,11 +487,11 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService {
 
                                 status = messageListener.consumeMessage(Collections.unmodifiableList(msgs), context);
                             } catch (Throwable e) {
-                                log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
+                                log.warn(String.format("consumeMessage exception: %s Group: %s Msgs: %s MQ: %s",
                                     RemotingHelper.exceptionSimpleDesc(e),
                                     ConsumeMessageOrderlyService.this.consumerGroup,
                                     msgs,
-                                    messageQueue);
+                                    messageQueue), e);
                                 hasException = true;
                             } finally {
                                 this.processQueue.getConsumeLock().unlock();

client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java

@@ -100,7 +100,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
 
     private static final String NOT_RUNNING_EXCEPTION_MESSAGE = "The consumer not running, please start it first.";
 
-    private static final String SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE = "Subscribe and assign are mutually exclusive.";
+    private static final String SUBSCRIPTION_CONFLICT_EXCEPTION_MESSAGE = "Subscribe and assign are mutually exclusive.";
     /**
      * the type of subscription
      */
@@ -195,8 +195,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
     }
 
     private void checkServiceState() {
-        if (this.serviceState != ServiceState.RUNNING)
+        if (this.serviceState != ServiceState.RUNNING) {
             throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE);
+        }
     }
 
     public void updateNameServerAddr(String newAddresses) {
@@ -204,10 +205,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
     }
 
     private synchronized void setSubscriptionType(SubscriptionType type) {
-        if (this.subscriptionType == SubscriptionType.NONE)
+        if (this.subscriptionType == SubscriptionType.NONE) {
             this.subscriptionType = type;
-        else if (this.subscriptionType != type)
-            throw new IllegalStateException(SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE);
+        } else if (this.subscriptionType != type) {
+            throw new IllegalStateException(SUBSCRIPTION_CONFLICT_EXCEPTION_MESSAGE);
+        }
     }
 
     private void updateAssignedMessageQueue(String topic, Set<MessageQueue> assignedMessageQueue) {
@@ -464,7 +466,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
 
     public synchronized void subscribe(String topic, String subExpression) throws MQClientException {
         try {
-            if (topic == null || topic.equals("")) {
+            if (topic == null || "".equals(topic)) {
                 throw new IllegalArgumentException("Topic can not be null or empty.");
             }
             setSubscriptionType(SubscriptionType.SUBSCRIBE);
@@ -483,7 +485,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
 
     public synchronized void subscribe(String topic, MessageSelector messageSelector) throws MQClientException {
         try {
-            if (topic == null || topic.equals("")) {
+            if (topic == null || "".equals(topic)) {
                 throw new IllegalArgumentException("Topic can not be null or empty.");
             }
             setSubscriptionType(SubscriptionType.SUBSCRIBE);
@@ -533,8 +535,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
     public synchronized List<MessageExt> poll(long timeout) {
         try {
             checkServiceState();
-            if (timeout < 0)
+            if (timeout < 0) {
                 throw new IllegalArgumentException("Timeout must not be negative");
+            }
 
             if (defaultLitePullConsumer.isAutoCommit()) {
                 maybeAutoCommit();
@@ -546,8 +549,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
             if (endTime - System.currentTimeMillis() > 0) {
                 while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) {
                     consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
-                    if (endTime - System.currentTimeMillis() <= 0)
+                    if (endTime - System.currentTimeMillis() <= 0) {
                         break;
+                    }
                 }
             }
 
@@ -671,8 +675,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
     public long committed(MessageQueue messageQueue) throws MQClientException {
         checkServiceState();
         long offset = this.offsetStore.readOffset(messageQueue, ReadOffsetType.MEMORY_FIRST_THEN_STORE);
-        if (offset == -2)
+        if (offset == -2) {
             throw new MQClientException("Fetch consume offset from broker exception", null);
+        }
         return offset;
     }
 
@@ -683,8 +688,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
         }
         Iterator<ConsumeRequest> iter = consumeRequestCache.iterator();
         while (iter.hasNext()) {
-            if (iter.next().getMessageQueue().equals(messageQueue))
+            if (iter.next().getMessageQueue().equals(messageQueue)) {
                 iter.remove();
+            }
         }
     }
 
@@ -735,10 +741,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
                     return;
                 }
 
-                if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {
+                if ((long) consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {
                     scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS);
-                    if ((consumeRequestFlowControlTimes++ % 1000) == 0)
+                    if ((consumeRequestFlowControlTimes++ % 1000) == 0) {
                         log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes);
+                    }
                     return;
                 }
 
@@ -778,7 +785,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
                 long offset = 0L;
                 try {
                     offset = nextPullOffset(messageQueue);
-                } catch (MQClientException e) {
+                } catch (Exception e) {
                     log.error("Failed to get next pull offset", e);
                     scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_ON_EXCEPTION, TimeUnit.MILLISECONDS);
                     return;
@@ -790,11 +797,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
                 long pullDelayTimeMills = 0;
                 try {
                     SubscriptionData subscriptionData;
+                    String topic = this.messageQueue.getTopic();
                     if (subscriptionType == SubscriptionType.SUBSCRIBE) {
-                        String topic = this.messageQueue.getTopic();
                         subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic);
                     } else {
-                        String topic = this.messageQueue.getTopic();
                         subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL);
                     }
                     

client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java

@@ -273,7 +273,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
                     long offset = -1L;
                     try {
                         offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue());
-                    } catch (MQClientException e) {
+                    } catch (Exception e) {
                         this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
                         log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e);
                         return;

client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java

@@ -378,7 +378,7 @@ public abstract class RebalanceImpl {
                 long nextOffset = -1L;
                 try {
                     nextOffset = this.computePullFromWhereWithException(mq);
-                } catch (MQClientException e) {
+                } catch (Exception e) {
                     log.info("doRebalance, {}, compute offset failed, {}", consumerGroup, mq);
                     continue;
                 }

client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java

@@ -102,7 +102,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl {
                         try {
                             result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq);
                         } catch (MQClientException e) {
-                            result = -1;
+                            log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e);
+                            throw e;
                         }
                     }
                 } else {

client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java

@@ -863,7 +863,7 @@ public class MQClientInstance {
         }
     }
 
-    public boolean registerConsumer(final String group, final MQConsumerInner consumer) {
+    public synchronized boolean registerConsumer(final String group, final MQConsumerInner consumer) {
         if (null == group || null == consumer) {
             return false;
         }
@@ -877,27 +877,9 @@ public class MQClientInstance {
         return true;
     }
 
-    public void unregisterConsumer(final String group) {
+    public synchronized void unregisterConsumer(final String group) {
         this.consumerTable.remove(group);
-        this.unregisterClientWithLock(null, group);
-    }
-
-    private void unregisterClientWithLock(final String producerGroup, final String consumerGroup) {
-        try {
-            if (this.lockHeartbeat.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
-                try {
-                    this.unregisterClient(producerGroup, consumerGroup);
-                } catch (Exception e) {
-                    log.error("unregisterClient exception", e);
-                } finally {
-                    this.lockHeartbeat.unlock();
-                }
-            } else {
-                log.warn("lock heartBeat, but failed. [{}]", this.clientId);
-            }
-        } catch (InterruptedException e) {
-            log.warn("unregisterClientWithLock exception", e);
-        }
+        this.unregisterClient(null, group);
     }
 
     private void unregisterClient(final String producerGroup, final String consumerGroup) {
@@ -927,7 +909,7 @@ public class MQClientInstance {
         }
     }
 
-    public boolean registerProducer(final String group, final DefaultMQProducerImpl producer) {
+    public synchronized boolean registerProducer(final String group, final DefaultMQProducerImpl producer) {
         if (null == group || null == producer) {
             return false;
         }
@@ -941,9 +923,9 @@ public class MQClientInstance {
         return true;
     }
 
-    public void unregisterProducer(final String group) {
+    public synchronized void unregisterProducer(final String group) {
         this.producerTable.remove(group);
-        this.unregisterClientWithLock(group, null);
+        this.unregisterClient(group, null);
     }
 
     public boolean registerAdminExt(final String group, final MQAdminExtInner admin) {

client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java

@@ -24,14 +24,14 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -107,7 +107,12 @@ public class DefaultMQProducerImpl implements MQProducerInner {
     private final RPCHook rpcHook;
     private final BlockingQueue<Runnable> asyncSenderThreadPoolQueue;
     private final ExecutorService defaultAsyncSenderExecutor;
-    private final Timer timer = new Timer("RequestHouseKeepingService", true);
+    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+        @Override
+        public Thread newThread(Runnable r) {
+            return new Thread(r, "RequestHouseKeepingService");
+        }
+    });
     protected BlockingQueue<Runnable> checkRequestQueue;
     protected ExecutorService checkExecutor;
     private ServiceState serviceState = ServiceState.CREATE_JUST;
@@ -227,16 +232,23 @@ public class DefaultMQProducerImpl implements MQProducerInner {
 
         this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
 
-        this.timer.scheduleAtFixedRate(new TimerTask() {
-            @Override
-            public void run() {
-                try {
-                    RequestFutureTable.scanExpiredRequest();
-                } catch (Throwable e) {
-                    log.error("scan RequestFutureTable exception", e);
+        this.startScheduledTask();
+
+    }
+
+    private void startScheduledTask() {
+        if (RequestFutureTable.getProducerNum().incrementAndGet() == 1) {
+            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        RequestFutureTable.scanExpiredRequest();
+                    } catch (Throwable e) {
+                        log.error("scan RequestFutureTable exception", e);
+                    }
                 }
-            }
-        }, 1000 * 3, 1000);
+            }, 1000 * 3, 1000, TimeUnit.MILLISECONDS);
+        }
     }
 
     private void checkConfig() throws MQClientException {
@@ -266,7 +278,9 @@ public class DefaultMQProducerImpl implements MQProducerInner {
                 if (shutdownFactory) {
                     this.mQClientFactory.shutdown();
                 }
-                this.timer.cancel();
+                if (RequestFutureTable.getProducerNum().decrementAndGet() == 0) {
+                    scheduledExecutorService.shutdown();
+                }
                 log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup());
                 this.serviceState = ServiceState.SHUTDOWN_ALREADY;
                 break;

client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java

@@ -22,6 +22,8 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import org.apache.rocketmq.client.common.ClientErrorCode;
 import org.apache.rocketmq.client.exception.RequestTimeoutException;
 import org.apache.rocketmq.client.log.ClientLogger;
@@ -30,6 +32,7 @@ import org.apache.rocketmq.logging.InternalLogger;
 public class RequestFutureTable {
     private static InternalLogger log = ClientLogger.getLog();
     private static ConcurrentHashMap<String, RequestResponseFuture> requestFutureTable = new ConcurrentHashMap<String, RequestResponseFuture>();
+    private static final AtomicInteger PRODUCER_NUM = new AtomicInteger(0);
 
     public static ConcurrentHashMap<String, RequestResponseFuture> getRequestFutureTable() {
         return requestFutureTable;
@@ -59,4 +62,8 @@ public class RequestFutureTable {
             }
         }
     }
+
+    public static AtomicInteger getProducerNum() {
+        return PRODUCER_NUM;
+    }
 }

client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java

@@ -16,7 +16,6 @@
  */
 package org.apache.rocketmq.client.trace;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -28,7 +27,9 @@ import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.AccessChannel;
 import org.apache.rocketmq.client.common.ThreadLocalIndex;
@@ -54,6 +55,7 @@ import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAM
 public class AsyncTraceDispatcher implements TraceDispatcher {
 
     private final static InternalLogger log = ClientLogger.getLog();
+    private final static AtomicInteger COUNTER = new AtomicInteger();
     private final int queueSize;
     private final int batchSize;
     private final int maxMsgSize;
@@ -62,7 +64,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
     // The last discard number of log
     private AtomicLong discardCount;
     private Thread worker;
-    private ArrayBlockingQueue<TraceContext> traceContextQueue;
+    private final ArrayBlockingQueue<TraceContext> traceContextQueue;
     private ArrayBlockingQueue<Runnable> appenderQueue;
     private volatile Thread shutDownHook;
     private volatile boolean stopped = false;
@@ -76,7 +78,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
     private String group;
     private Type type;
 
-    public AsyncTraceDispatcher(String group, Type type,String traceTopicName, RPCHook rpcHook) {
+    public AsyncTraceDispatcher(String group, Type type, String traceTopicName, RPCHook rpcHook) {
         // queueSize is greater than or equal to the n power of 2 of value
         this.queueSize = 2048;
         this.batchSize = 100;
@@ -93,12 +95,12 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
             this.traceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC;
         }
         this.traceExecutor = new ThreadPoolExecutor(//
-            10, //
-            20, //
-            1000 * 60, //
-            TimeUnit.MILLISECONDS, //
-            this.appenderQueue, //
-            new ThreadFactoryImpl("MQTraceSendThread_"));
+                10, //
+                20, //
+                1000 * 60, //
+                TimeUnit.MILLISECONDS, //
+                this.appenderQueue, //
+                new ThreadFactoryImpl("MQTraceSendThread_"));
         traceProducer = getAndCreateTraceProducer(rpcHook);
     }
 
@@ -165,7 +167,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
     }
 
     private String genGroupNameForTrace() {
-        return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type ;
+        return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type + "-" + COUNTER.incrementAndGet();
     }
 
     @Override
@@ -178,10 +180,15 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
     }
 
     @Override
-    public void flush() throws IOException {
+    public void flush() {
         // The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return.
         long end = System.currentTimeMillis() + 500;
-        while (traceContextQueue.size() > 0 || appenderQueue.size() > 0 && System.currentTimeMillis() <= end) {
+        while (System.currentTimeMillis() <= end) {
+            synchronized (traceContextQueue) {
+                if (traceContextQueue.size() == 0 && appenderQueue.size() == 0) {
+                    break;
+                }
+            }
             try {
                 Thread.sleep(1);
             } catch (InterruptedException e) {
@@ -194,6 +201,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
     @Override
     public void shutdown() {
         this.stopped = true;
+        flush();
         this.traceExecutor.shutdown();
         if (isStarted.get()) {
             traceProducer.shutdown();
@@ -210,11 +218,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
                 public void run() {
                     synchronized (this) {
                         if (!this.hasShutdown) {
-                            try {
-                                flush();
-                            } catch (IOException e) {
-                                log.error("system MQTrace hook shutdown failed ,maybe loss some trace data");
-                            }
+                            flush();
                         }
                     }
                 }
@@ -240,25 +244,27 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
         public void run() {
             while (!stopped) {
                 List<TraceContext> contexts = new ArrayList<TraceContext>(batchSize);
-                for (int i = 0; i < batchSize; i++) {
-                    TraceContext context = null;
-                    try {
-                        //get trace data element from blocking Queue — traceContextQueue
-                        context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS);
-                    } catch (InterruptedException e) {
+                synchronized (traceContextQueue) {
+                    for (int i = 0; i < batchSize; i++) {
+                        TraceContext context = null;
+                        try {
+                            //get trace data element from blocking Queue - traceContextQueue
+                            context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS);
+                        } catch (InterruptedException e) {
+                        }
+                        if (context != null) {
+                            contexts.add(context);
+                        } else {
+                            break;
+                        }
                     }
-                    if (context != null) {
-                        contexts.add(context);
-                    } else {
-                        break;
+                    if (contexts.size() > 0) {
+                        AsyncAppenderRequest request = new AsyncAppenderRequest(contexts);
+                        traceExecutor.submit(request);
+                    } else if (AsyncTraceDispatcher.this.stopped) {
+                        this.stopped = true;
                     }
                 }
-                if (contexts.size() > 0) {
-                    AsyncAppenderRequest request = new AsyncAppenderRequest(contexts);
-                    traceExecutor.submit(request);
-                } else if (AsyncTraceDispatcher.this.stopped) {
-                    this.stopped = true;
-                }
             }
 
         }
@@ -350,7 +356,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher {
          * Send message trace data
          *
          * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId)
-         * @param data the message trace data in this batch
+         * @param data   the message trace data in this batch
          */
         private void sendTraceDataByMQ(Set<String> keySet, final String data, String dataTopic, String regionId) {
             String traceTopic = traceTopicName;

client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java

@@ -188,9 +188,7 @@ public class TraceDataEncoder {
                         .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//
                         .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)//
                         .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//
-                        .append(ctx.getContextCode()).append(TraceConstants.CONTENT_SPLITOR)
-                        .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)
-                        .append(ctx.getGroupName()).append(TraceConstants.FIELD_SPLITOR);
+                        .append(ctx.getContextCode()).append(TraceConstants.FIELD_SPLITOR);
 
                 }
             }

client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java

@@ -60,7 +60,6 @@ public class TraceDataEncoderTest {
         Assert.assertEquals(contexts.get(0).getTraceType(), TraceType.Pub);
     }
 
-
     @Test
     public void testEncoderFromContextBean() {
         TraceContext context = new TraceContext();
@@ -130,4 +129,107 @@ public class TraceDataEncoderTest {
         Assert.assertEquals(before.getTransactionState(), after.getTransactionState());
         Assert.assertEquals(before.isFromTransactionCheck(), after.isFromTransactionCheck());
     }
+    
+    @Test
+    public void testPubTraceDataFormatTest() {
+        TraceContext pubContext = new TraceContext();
+        pubContext.setTraceType(TraceType.Pub);
+        pubContext.setTimeStamp(time);
+        pubContext.setRegionId("Default-region");
+        pubContext.setGroupName("GroupName-test");
+        pubContext.setCostTime(34);
+        pubContext.setSuccess(true);
+        TraceBean bean = new TraceBean();
+        bean.setTopic("topic-test");
+        bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+        bean.setTags("tags");
+        bean.setKeys("keys");
+        bean.setStoreHost("127.0.0.1:10911");
+        bean.setBodyLength(100);
+        bean.setMsgType(MessageType.Normal_Msg);
+        bean.setOffsetMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+        pubContext.setTraceBeans(new ArrayList<TraceBean>(1));
+        pubContext.getTraceBeans().add(bean);
+
+        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(pubContext);
+        String transData = traceTransferBean.getTransData();
+        Assert.assertNotNull(transData);
+        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+        Assert.assertEquals(14, items.length);
+
+    }
+
+    @Test
+    public void testSubBeforeTraceDataFormatTest() {
+        TraceContext subBeforeContext = new TraceContext();
+        subBeforeContext.setTraceType(TraceType.SubBefore);
+        subBeforeContext.setTimeStamp(time);
+        subBeforeContext.setRegionId("Default-region");
+        subBeforeContext.setGroupName("GroupName-test");
+        subBeforeContext.setRequestId("3455848576927");
+        TraceBean bean = new TraceBean();
+        bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+        bean.setRetryTimes(0);
+        bean.setKeys("keys");
+        subBeforeContext.setTraceBeans(new ArrayList<TraceBean>(1));
+        subBeforeContext.getTraceBeans().add(bean);
+
+        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(subBeforeContext);
+        String transData = traceTransferBean.getTransData();
+        Assert.assertNotNull(transData);
+        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+        Assert.assertEquals(8, items.length);
+
+    }
+
+    @Test
+    public void testSubAfterTraceDataFormatTest() {
+        TraceContext subAfterContext = new TraceContext();
+        subAfterContext.setTraceType(TraceType.SubAfter);
+        subAfterContext.setRequestId("3455848576927");
+        subAfterContext.setCostTime(20);
+        subAfterContext.setSuccess(true);
+        subAfterContext.setContextCode(98623046);
+        TraceBean bean = new TraceBean();
+        bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+        bean.setKeys("keys");
+        subAfterContext.setTraceBeans(new ArrayList<TraceBean>(1));
+        subAfterContext.getTraceBeans().add(bean);
+
+        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(subAfterContext);
+        String transData = traceTransferBean.getTransData();
+        Assert.assertNotNull(transData);
+        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+        Assert.assertEquals(7, items.length);
+
+    }
+
+    @Test
+    public void testEndTrxTraceDataFormatTest() {
+        TraceContext endTrxContext = new TraceContext();
+        endTrxContext.setTraceType(TraceType.EndTransaction);
+        endTrxContext.setGroupName("PID-test");
+        endTrxContext.setRegionId("DefaultRegion");
+        endTrxContext.setTimeStamp(time);
+        TraceBean endTrxTraceBean = new TraceBean();
+        endTrxTraceBean.setTopic("topic-test");
+        endTrxTraceBean.setKeys("Keys");
+        endTrxTraceBean.setTags("Tags");
+        endTrxTraceBean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
+        endTrxTraceBean.setStoreHost("127.0.0.1:10911");
+        endTrxTraceBean.setMsgType(MessageType.Trans_msg_Commit);
+        endTrxTraceBean.setTransactionId("transactionId");
+        endTrxTraceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE);
+        endTrxTraceBean.setFromTransactionCheck(false);
+        List<TraceBean> traceBeans = new ArrayList<TraceBean>();
+        traceBeans.add(endTrxTraceBean);
+        endTrxContext.setTraceBeans(traceBeans);
+
+        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(endTrxContext);
+        String transData = traceTransferBean.getTransData();
+        Assert.assertNotNull(transData);
+        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+        Assert.assertEquals(13, items.length);
+
+    }
 }

common/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java

@@ -57,10 +57,9 @@ public class BrokerConfig {
     @ImportantField
     private boolean traceTopicEnable = false;
     /**
-     * thread numbers for send message thread pool, since spin lock will be used by default since 4.0.x, the default
-     * value is 1.
+     * thread numbers for send message thread pool.
      */
-    private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4;
+    private int sendMessageThreadPoolNums = Math.min(Runtime.getRuntime().availableProcessors(), 4);
     private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2;
     private int processReplyMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2;
     private int queryMessageThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors();
@@ -73,7 +72,8 @@ public class BrokerConfig {
     /**
      * Thread numbers for EndTransactionProcessor
      */
-    private int endTransactionThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors() * 2;
+    private int endTransactionThreadPoolNums = Math.max(8 + Runtime.getRuntime().availableProcessors() * 2,
+            sendMessageThreadPoolNums * 4);
 
     private int flushConsumerOffsetInterval = 1000 * 5;
 

common/src/main/java/org/apache/rocketmq/common/MQVersion.java

@@ -18,7 +18,7 @@ package org.apache.rocketmq.common;
 
 public class MQVersion {
 
-    public static final int CURRENT_VERSION = Version.V4_9_0.ordinal();
+    public static final int CURRENT_VERSION = Version.V4_9_1.ordinal();
 
     public static String getVersionDesc(int value) {
         int length = Version.values().length;

common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java

@@ -409,7 +409,23 @@ public class MessageDecoder {
     }
 
     public static String messageProperties2String(Map<String, String> properties) {
-        StringBuilder sb = new StringBuilder();
+        if (properties == null) {
+            return "";
+        }
+        int len = 0;
+        for (final Map.Entry<String, String> entry : properties.entrySet()) {
+            final String name = entry.getKey();
+            final String value = entry.getValue();
+            if (value == null) {
+                continue;
+            }
+            if (name != null) {
+                len += name.length();
+            }
+            len += value.length();
+            len += 2; // separator
+        }
+        StringBuilder sb = new StringBuilder(len);
         if (properties != null) {
             for (final Map.Entry<String, String> entry : properties.entrySet()) {
                 final String name = entry.getKey();
@@ -423,6 +439,9 @@ public class MessageDecoder {
                 sb.append(value);
                 sb.append(PROPERTY_SEPARATOR);
             }
+            if (sb.length() > 0) {
+                sb.deleteCharAt(sb.length() - 1);
+            }
         }
         return sb.toString();
     }
@@ -430,12 +449,22 @@ public class MessageDecoder {
     public static Map<String, String> string2messageProperties(final String properties) {
         Map<String, String> map = new HashMap<String, String>();
         if (properties != null) {
-            String[] items = properties.split(String.valueOf(PROPERTY_SEPARATOR));
-            for (String i : items) {
-                String[] nv = i.split(String.valueOf(NAME_VALUE_SEPARATOR));
-                if (2 == nv.length) {
-                    map.put(nv[0], nv[1]);
+            int len = properties.length();
+            int index = 0;
+            while (index < len) {
+                int newIndex = properties.indexOf(PROPERTY_SEPARATOR, index);
+                if (newIndex < 0) {
+                    newIndex = len;
                 }
+                if (newIndex - index >= 3) {
+                    int kvSepIndex = properties.indexOf(NAME_VALUE_SEPARATOR, index);
+                    if (kvSepIndex > index && kvSepIndex < newIndex - 1) {
+                        String k = properties.substring(index, kvSepIndex);
+                        String v = properties.substring(kvSepIndex + 1, newIndex);
+                        map.put(k, v);
+                    }
+                }
+                index = newIndex + 1;
             }
         }
 

common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java

@@ -20,6 +20,7 @@ public class NameServerAddressUtils {
     public static final String INSTANCE_PREFIX = "MQ_INST_";
     public static final String INSTANCE_REGEX = INSTANCE_PREFIX + "\\w+_\\w+";
     public static final String ENDPOINT_PREFIX = "(\\w+://|)";
+    public static final Pattern NAMESRV_ENDPOINT_PATTERN = Pattern.compile("^http://.*");
     public static final Pattern INST_ENDPOINT_PATTERN = Pattern.compile("^" + ENDPOINT_PREFIX + INSTANCE_REGEX + "\\..*");
 
     public static String getNameServerAddresses() {

common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java

@@ -25,6 +25,8 @@ import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.Map;
 
+import static org.apache.rocketmq.common.message.MessageDecoder.NAME_VALUE_SEPARATOR;
+import static org.apache.rocketmq.common.message.MessageDecoder.PROPERTY_SEPARATOR;
 import static org.apache.rocketmq.common.message.MessageDecoder.createMessageId;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -265,4 +267,110 @@ public class MessageDecoderTest {
         }
     }
 
+    @Test
+    public void testString2messageProperties() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("k1").append(NAME_VALUE_SEPARATOR).append("v1");
+        Map<String,String> m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("k1")).isEqualTo("v1");
+
+        m = MessageDecoder.string2messageProperties("");
+        assertThat(m).size().isEqualTo(0);
+
+        m = MessageDecoder.string2messageProperties(" ");
+        assertThat(m).size().isEqualTo(0);
+
+        m = MessageDecoder.string2messageProperties("aaa");
+        assertThat(m).size().isEqualTo(0);
+
+        sb.setLength(0);
+        sb.append("k1").append(NAME_VALUE_SEPARATOR);
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(0);
+
+        sb.setLength(0);
+        sb.append(NAME_VALUE_SEPARATOR).append("v1");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(0);
+
+        sb.setLength(0);
+        sb.append("k1").append(NAME_VALUE_SEPARATOR).append("v1").append(PROPERTY_SEPARATOR);
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("k1")).isEqualTo("v1");
+
+        sb.setLength(0);
+        sb.append("k1").append(NAME_VALUE_SEPARATOR).append("v1").append(PROPERTY_SEPARATOR)
+                .append("k2").append(NAME_VALUE_SEPARATOR).append("v2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(2);
+        assertThat(m.get("k1")).isEqualTo("v1");
+        assertThat(m.get("k2")).isEqualTo("v2");
+
+        sb.setLength(0);
+        sb.append("k1").append(NAME_VALUE_SEPARATOR).append("v1").append(PROPERTY_SEPARATOR)
+                .append(NAME_VALUE_SEPARATOR).append("v2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("k1")).isEqualTo("v1");
+
+        sb.setLength(0);
+        sb.append("k1").append(NAME_VALUE_SEPARATOR).append("v1").append(PROPERTY_SEPARATOR)
+                .append("k2").append(NAME_VALUE_SEPARATOR);
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("k1")).isEqualTo("v1");
+
+        sb.setLength(0);
+        sb.append(NAME_VALUE_SEPARATOR).append("v1").append(PROPERTY_SEPARATOR)
+                .append("k2").append(NAME_VALUE_SEPARATOR).append("v2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("k2")).isEqualTo("v2");
+
+        sb.setLength(0);
+        sb.append("k1").append(NAME_VALUE_SEPARATOR).append(PROPERTY_SEPARATOR)
+                .append("k2").append(NAME_VALUE_SEPARATOR).append("v2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("k2")).isEqualTo("v2");
+
+        sb.setLength(0);
+        sb.append("1").append(NAME_VALUE_SEPARATOR).append("1").append(PROPERTY_SEPARATOR)
+                .append("2").append(NAME_VALUE_SEPARATOR).append("2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(2);
+        assertThat(m.get("1")).isEqualTo("1");
+        assertThat(m.get("2")).isEqualTo("2");
+
+        sb.setLength(0);
+        sb.append("1").append(NAME_VALUE_SEPARATOR).append(PROPERTY_SEPARATOR)
+                .append("2").append(NAME_VALUE_SEPARATOR).append("2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("2")).isEqualTo("2");
+
+        sb.setLength(0);
+        sb.append(NAME_VALUE_SEPARATOR).append("1").append(PROPERTY_SEPARATOR)
+                .append("2").append(NAME_VALUE_SEPARATOR).append("2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("2")).isEqualTo("2");
+
+        sb.setLength(0);
+        sb.append("1").append(NAME_VALUE_SEPARATOR).append("1").append(PROPERTY_SEPARATOR)
+                .append("2").append(NAME_VALUE_SEPARATOR);
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("1")).isEqualTo("1");
+
+        sb.setLength(0);
+        sb.append("1").append(NAME_VALUE_SEPARATOR).append("1").append(PROPERTY_SEPARATOR)
+                .append(NAME_VALUE_SEPARATOR).append("2");
+        m = MessageDecoder.string2messageProperties(sb.toString());
+        assertThat(m).size().isEqualTo(1);
+        assertThat(m.get("1")).isEqualTo("1");
+    }
+
 }

distribution/benchmark/batchproducer.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sh ./runclass.sh org.apache.rocketmq.example.benchmark.BatchProducer $@ &

distribution/bin/runbroker.cmd

@@ -28,7 +28,7 @@ set CLASSPATH=.;%BASE_DIR%conf;%CLASSPATH%
 rem ===========================================================================================
 rem  JVM Configuration
 rem ===========================================================================================
-set "JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g -Xmn1g"
+set "JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g"
 set "JAVA_OPT=%JAVA_OPT% -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8"
 set "JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:%USERPROFILE%\mq_gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy"
 set "JAVA_OPT=%JAVA_OPT% -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m"

distribution/bin/runbroker.sh

@@ -64,7 +64,7 @@ choose_gc_log_directory()
 
 choose_gc_log_directory
 
-JAVA_OPT="${JAVA_OPT} -server -Xms8g -Xmx8g -Xmn4g"
+JAVA_OPT="${JAVA_OPT} -server -Xms8g -Xmx8g"
 JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0"
 JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_broker_gc_%p_%t.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy"
 JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m"

distribution/bin/runserver.cmd

@@ -28,7 +28,7 @@ set CLASSPATH=.;%BASE_DIR%conf;%CLASSPATH%
 
 set "JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m"
 set "JAVA_OPT=%JAVA_OPT% -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC"
-set "JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:"%USERPROFILE%\rmq_srv_gc.log" -XX:+PrintGCDetails"
+set "JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:"%USERPROFILE%\rmq_srv_gc.log" -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
 set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow"
 set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages"
 set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext"

distribution/bin/runserver.sh

@@ -68,18 +68,18 @@ choose_gc_options()
     # '1' means releases befor Java 9
     JAVA_MAJOR_VERSION=$("$JAVA" -version 2>&1 | sed -r -n 's/.* version "([0-9]*).*$/\1/p')
     if [ -z "$JAVA_MAJOR_VERSION" ] || [ "$JAVA_MAJOR_VERSION" -lt "9" ] ; then
+      JAVA_OPT="${JAVA_OPT} -server -Xms4g -Xmx4g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m"
       JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC"
-      JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log -XX:+PrintGCDetails"
+      JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
       JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m"
     else
+      JAVA_OPT="${JAVA_OPT} -server -Xms4g -Xmx4g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m"
       JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0"
       JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log:time,tags:filecount=5,filesize=30M"
     fi
 }
 
 choose_gc_log_directory
-
-JAVA_OPT="${JAVA_OPT} -server -Xms4g -Xmx4g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m"
 choose_gc_options
 JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow"
 JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages"

distribution/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
     <artifactId>rocketmq-distribution</artifactId>
     <name>rocketmq-distribution ${project.version}</name>

distribution/release.xml

@@ -74,6 +74,10 @@
                 <dependencySets>
                     <dependencySet>
                         <outputDirectory>lib/</outputDirectory>
+                        <excludes>
+                            <exclude>io.jaegertracing:jaeger-core</exclude>
+                            <exclude>io.jaegertracing:jaeger-client</exclude>
+                        </excludes>
                     </dependencySet>
                 </dependencySets>
             </binaries>

docs/cn/Configuration_System.md

@@ -0,0 +1,70 @@
+# 系统配置
+
+本节重点介绍系统（JVM/OS）的配置 
+---
+
+## **1 JVM 选项** ##
+
+建议使用最新发布的 JDK 1.8 版本。设置相同的 Xms 和 Xmx 值以防止 JVM 调整堆大小，并获得更好的性能。一种通用的JVM配置如下： 
+
+    -server -Xms8g -Xmx8g -Xmn4g
+
+设置 Direct ByteBuffer 内存大小。当 Direct ByteBuffer 达到指定大小时，将触发 Full GC：
+
+    -XXMaxDirectMemorySize=15g
+
+如果你不在乎 RocketMQ broker 的启动时间，建议启用预分配 Java 堆以确保在 JVM 初始化期间为每个页面分配内存。你可以通过以下方式启用它： 
+    
+    -XX+AlwaysPreTouch
+
+禁用偏向锁定可以减少 JVM 停顿： 
+
+    -XX-UseBiasedLocking
+
+关于垃圾收集器，推荐使用 JDK 1.8 的 G1 收集器： 
+
+    -XX+UseG1GC -XXG1HeapRegionSize=16m 
+    -XXG1ReservePercent=25
+    -XXInitiatingHeapOccupancyPercent=30
+
+这些 GC 选项看起来有点激进，但事实证明它在生产环境中具有良好的性能 
+
+不要把-XXMaxGCPauseMillis 的值设置太小，否则JVM会使用一个小的新生代来实现这个目标，从而导致频繁发生minor GC。因此，建议使用滚动 GC 日志文件：
+    
+    -XX+UseGCLogFileRotation 
+    -XXNumberOfGCLogFiles=5 
+    -XXGCLogFileSize=30m
+    
+写 GC 文件会增加 broker 的延迟，因此可以考虑将 GC 日志文件重定向到内存文件系统：
+    
+    -Xloggcdevshmmq_gc_%p.log123
+
+## 2 Linux 内核参数 ##
+
+在 bin 文件夹里，有一个 os.sh 脚本，里面列出了许多的内核参数，只需稍作更改即可用于生产用途。需特别关注以下参数，如想了解更多细节，请参考文档/proc/sys/vm/*。 
+
+
+
+- **vm.extra_free_kbytes**, 控制VM在后台回收（kswapd）开始的阈值和直接回收（通过分配进程）开始的阈值之间保留额外的空闲内存。通过使用这个参数，RocketMQ 可以避免在内存分配过程中出现高延迟。（与内核版本版本有关）
+
+
+
+- **vm.min_free_kbytes**, 该值不应设置低于1024KB，否则系统将遭到破坏，并且在高负载环境下容易出现死锁。 
+
+
+
+
+
+- **vm.max_map_count**, 规定进程可以拥有的最大内存映射区域数。 RocketMQ 使用 mmap 来加载 CommitLog 和 ConsumeQueue，因此建议将此参数设置为较大的值。 
+
+
+
+- **vm.swappiness**, 定义内核交换内存页的频率。该值若较大，则会导致频繁交换，较小则会减少交换量。为了避免交换延迟，建议将此值设为 10。 
+
+
+
+- **File descriptor limits**, RocketMQ 需要给文件（CommitLog 和 ConsumeQueue）和网络连接分配文件描述符。因此建议将该值设置为 655350。 
+
+
+
+- **Disk scheduler**, 推荐使用deadline IO 调度器，它可以为请求提供有保证的延迟。 

docs/cn/Deployment.md

@@ -0,0 +1,159 @@
+# 部署架构和设置步骤
+
+## 集群的设置
+
+### 1 单master模式
+
+这是最简单但也是最危险的模式，一旦broker服务器重启或宕机，整个服务将不可用。 建议在生产环境中不要使用这种部署方式，在本地测试和开发可以选择这种模式。 以下是构建的步骤。
+
+**1）启动NameServer**
+
+```shell
+### 第一步启动namesrv
+$ nohup sh mqnamesrv &
+ 
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+我们可以在namesrv.log 中看到'The Name Server boot success..'，表示NameServer 已成功启动。
+
+**2）启动Broker**
+
+```shell
+### 第一步先启动broker
+$ nohup sh bin/mqbroker -n localhost:9876 &
+
+### 验证broker是否启动成功, 比如, broker的ip是192.168.1.2 然后名字是broker-a
+$ tail -f ~/logs/rocketmqlogs/Broker.log 
+The broker[broker-a,192.169.1.2:10911] boot success...
+```
+
+我们可以在 Broker.log 中看到“The broker[brokerName,ip:port] boot success..”，这表明 broker 已成功启动。
+
+### 2 多Master模式
+
+该模式是指所有节点都是master主节点（比如2个或3个主节点），没有slave从节点的模式。 这种模式的优缺点如下：
+
+- 优点: 
+  1. 配置简单。
+  2. 一个master节点的宕机或者重启（维护）对应用程序没有影响。
+  3. 当磁盘配置为RAID10时，消息不会丢失，因为RAID10磁盘非常可靠，即使机器不可恢复（消息异步刷盘模式的情况下，会丢失少量消息；如果消息是同步刷盘模式，不会丢失任何消息）。
+  4. 在这种模式下，性能是最高的。
+- 缺点:
+  1. 单台机器宕机时，本机未消费的消息，直到机器恢复后才会订阅，影响消息实时性。
+
+多Master模式的启动步骤如下：
+
+**1）启动 NameServer**
+
+```shell
+### 第一步先启动broker
+$ nohup sh mqnamesrv &
+ 
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+**2）启动 Broker 集群**
+
+```shell
+### 比如在A机器上启动第一个Master，假设配置的NameServer IP为：192.168.1.1
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties &
+ 
+### 然后在机器B上启动第二个Master，假设配置的NameServer IP是：192.168.1.1
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties &
+
+...
+```
+
+上面显示的boot命令用于单个NameServer的情况。对于多个NameServer的集群，broker boot命令中-n参数后面的地址列表用分号隔开，例如 192.168.1.1 : 9876; 192.161.2 : 9876
+
+### 3 多Master多Slave模式-异步复制
+
+每个主节点配置多个从节点，多对主从。HA采用异步复制，主节点和从节点之间有短消息延迟（毫秒）。这种模式的优缺点如下：
+
+- 优点: 
+  1. 即使磁盘损坏，也不会丢失极少的消息，不影响消息的实时性能。
+  2. 同时，当主节点宕机时，消费者仍然可以消费从节点的消息，这个过程对应用本身是透明的，不需要人为干预。
+  3. 性能几乎与多Master模式一样高。
+- 缺点:
+  1. 主节点宕机、磁盘损坏时，会丢失少量消息。
+
+多主多从模式的启动步骤如下：
+
+**1）启动 NameServer**
+
+```shell
+### 第一步先启动broker
+$ nohup sh mqnamesrv &
+ 
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+**2）启动 Broker 集群**
+
+```shell
+### 例如在A机器上启动第一个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties &
+ 
+### 然后在机器B上启动第二个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties &
+ 
+### 然后在C机器上启动第一个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties &
+ 
+### 最后在D机启动第二个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties &
+```
+
+上图显示了 2M-2S-Async 模式的启动命令，类似于其他 nM-nS-Async 模式。
+
+### 4 多Master多Slave模式-同步双写
+
+这种模式下，每个master节点配置多个slave节点，有多对Master-Slave。HA采用同步双写，即只有消息成功写入到主节点并复制到多个从节点，才会返回成功响应给应用程序。
+
+这种模式的优缺点如下：
+
+- 优点: 
+  1. 数据和服务都没有单点故障。
+  2. 在master节点关闭的情况下，消息也没有延迟。
+  3. 服务可用性和数据可用性非常高；
+- 缺点:
+  1. 这种模式下的性能略低于异步复制模式（大约低 10%）。
+  2. 发送单条消息的RT略高，目前版本，master节点宕机后，slave节点无法自动切换到master。
+
+启动步骤如下：
+
+**1）启动NameServer**
+
+```shell
+### 第一步启动broker
+$ nohup sh mqnamesrv &
+ 
+### 验证namesrv是否启动成功
+$ tail -f ~/logs/rocketmqlogs/namesrv.log
+The Name Server boot success...
+```
+
+**2）启动 Broker 集群**
+
+```shell
+### 例如在A机器上启动第一个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties &
+ 
+### 然后在B机器上启动第二个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties &
+ 
+### 然后在C机器上启动第一个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties &
+ 
+### 最后在D机启动第二个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。
+$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties &
+```
+
+上述Master和Slave是通过指定相同的config命名为“brokerName”来配对的，master节点的brokerId必须为0，slave节点的brokerId必须大于0。

docs/cn/Example_Batch.md

@@ -0,0 +1,82 @@
+# 批量消息发送
+批量消息发送能够提高发送效率，提升系统吞吐量。同一批批量消息的topic、waitStoreMsgOK属性必须保持一致，批量消息不支持延迟消息。批量消息发送一次最多可以发送 4MiB 的消息，但是如果需要发送更大的消息，建议将较大的消息分成多个不超过 1MiB 的小消息。
+
+### 1 发送批量消息
+如果你一次只发送不超过 4MiB 的消息，使用批处理很容易：
+```java
+String topic = "BatchTest";
+List<Message> messages = new ArrayList<>();
+messages.add(new Message(topic, "TagA", "OrderID001", "Hello world 0".getBytes()));
+messages.add(new Message(topic, "TagA", "OrderID002", "Hello world 1".getBytes()));
+messages.add(new Message(topic, "TagA", "OrderID003", "Hello world 2".getBytes()));
+try {
+    producer.send(messages);
+} catch (Exception e) {
+    e.printStackTrace();
+    //handle the error
+}
+```
+### 2 拆分
+当您发送较大的消息时，复杂性会增加，如果您不确定它是否超过 4MiB的限制。 这时候，您最好将较大的消息分成多个不超过 1MiB 的小消息：
+
+```java
+public class ListSplitter implements Iterator<List<Message>> { 
+    private final int SIZE_LIMIT = 1024 * 1024 * 4;
+    private final List<Message> messages;
+    private int currIndex;
+    public ListSplitter(List<Message> messages) { 
+        this.messages = messages;
+    }
+    @Override public boolean hasNext() {
+        return currIndex < messages.size(); 
+    }
+    @Override public List<Message> next() { 
+        int startIndex = getStartIndex();
+        int nextIndex = startIndex;
+        int totalSize = 0;
+        for (; nextIndex < messages.size(); nextIndex++) {
+            Message message = messages.get(nextIndex); 
+            int tmpSize = calcMessageSize(message);
+            if (tmpSize + totalSize > SIZE_LIMIT) {
+                break; 
+            } else {
+                totalSize += tmpSize; 
+            }
+        }
+        List<Message> subList = messages.subList(startIndex, nextIndex); 
+        currIndex = nextIndex;
+        return subList;
+    }
+    private int getStartIndex() {
+        Message currMessage = messages.get(currIndex); 
+        int tmpSize = calcMessageSize(currMessage); 
+        while(tmpSize > SIZE_LIMIT) {
+            currIndex += 1;
+            Message message = messages.get(curIndex); 
+            tmpSize = calcMessageSize(message);
+        }
+        return currIndex; 
+    }
+    private int calcMessageSize(Message message) {
+        int tmpSize = message.getTopic().length() + message.getBody().length(); 
+        Map<String, String> properties = message.getProperties();
+        for (Map.Entry<String, String> entry : properties.entrySet()) {
+            tmpSize += entry.getKey().length() + entry.getValue().length(); 
+        }
+        tmpSize = tmpSize + 20; // Increase the log overhead by 20 bytes
+        return tmpSize; 
+    }
+}
+
+// then you could split the large list into small ones:
+ListSplitter splitter = new ListSplitter(messages);
+while (splitter.hasNext()) {
+   try {
+       List<Message>  listItem = splitter.next();
+       producer.send(listItem);
+   } catch (Exception e) {
+       e.printStackTrace();
+       // handle the error
+   }
+}
+```

docs/cn/Example_Delay.md

@@ -0,0 +1,85 @@
+# Schedule example
+
+### 1 启动消费者等待传入的订阅消息
+
+```java
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.message.MessageExt;
+import java.util.List;
+
+public class ScheduledMessageConsumer {
+
+    public static void main(String[] args) throws Exception {
+        // Instantiate message consumer
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ExampleConsumer");
+        // Subscribe topics
+        consumer.subscribe("TestTopic", "*");
+        // Register message listener
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> messages, ConsumeConcurrentlyContext context) {
+                for (MessageExt message : messages) {
+                    // Print approximate delay time period
+                    System.out.println("Receive message[msgId=" + message.getMsgId() + "] "
+                                       + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later");
+                }
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        // Launch consumer
+        consumer.start();
+    }
+}
+```
+
+### 2 发送延迟消息 
+
+```java
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.message.Message;
+
+public class ScheduledMessageProducer {
+
+    public static void main(String[] args) throws Exception {
+        // Instantiate a producer to send scheduled messages
+        DefaultMQProducer producer = new DefaultMQProducer("ExampleProducerGroup");
+        // Launch producer
+        producer.start();
+        int totalMessagesToSend = 100;
+        for (int i = 0; i < totalMessagesToSend; i++) {
+            Message message = new Message("TestTopic", ("Hello scheduled message " + i).getBytes());
+            // This message will be delivered to consumer 10 seconds later.
+            message.setDelayTimeLevel(3);
+            // Send the message
+            producer.send(message);
+        }
+
+        // Shutdown producer after use.
+        producer.shutdown();
+    }
+
+}
+```
+
+### 3 确认
+
+您应该会看到消息在其存储时间后大约 10 秒被消耗。
+
+### 4 延迟消息的使用场景
+
+例如在电子商务中，如果提交订单，可以发送延迟消息，1小时后可以查看订单状态。 如果订单仍未付款，则可以取消订单并释放库存。
+
+### 5 使用延迟消息的限制
+
+```java 
+// org/apache/rocketmq/store/config/MessageStoreConfig.java
+
+private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h";
+```
+
+当前 RocketMQ 不支持任意时间的延迟。 生产者发送延迟消息前需要设置几个固定的延迟级别，分别对应1s到2h的1到18个延迟级，消息消费失败会进入延迟消息队列，消息发送时间与设置的延迟级别和重试次数有关。
+
+ See `SendMessageProcessor.java` 

docs/cn/Example_Simple_cn.md

@@ -0,0 +1,136 @@
+# Basic Sample 
+------
+基本示例中提供了以下两个功能
+* RocketMQ可用于以三种方式发送消息：可靠的同步、可靠的异步和单向传输。前两种消息类型是可靠的，因为无论它们是否成功发送都有响应。
+* RocketMQ可以用来消费消息。
+### 1 添加依赖
+maven:
+``` java
+<dependency>
+  <groupId>org.apache.rocketmq</groupId>
+  <artifactId>rocketmq-client</artifactId>
+  <version>4.3.0</version>
+</dependency>
+```
+gradle: 
+``` java 
+compile 'org.apache.rocketmq:rocketmq-client:4.3.0'
+```
+### 2 发送消息
+##### 2.1 使用Producer发送同步消息
+可靠的同步传输被广泛应用于各种场景，如重要的通知消息、短消息通知等。
+``` java
+public class SyncProducer {
+  public static void main(String[] args) throws Exception {
+    // Instantiate with a producer group name
+    DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+    // Specify name server addresses
+    producer.setNamesrvAddr("localhost:9876");
+    // Launch the producer instance
+    producer.start();
+    for (int i = 0; i < 100; i++) {
+      // Create a message instance with specifying topic, tag and message body
+      Message msg = new Message("TopicTest" /* Topic */,
+        "TagA" /* Tag */,
+        ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
+        );
+      // Send message to one of brokers
+      SendResult sendResult = producer.send(msg);
+      // Check whether the message has been delivered by the callback of sendResult
+      System.out.printf("%s%n", sendResult);
+    }
+    // Shut down once the producer instance is not longer in use
+    producer.shutdown();
+  }
+}
+```
+##### 2.2 发送异步消息
+异步传输通常用于响应时间敏感的业务场景。这意味着发送方无法等待代理的响应太长时间。
+``` java
+public class AsyncProducer {
+  public static void main(String[] args) throws Exception {
+    // Instantiate with a producer group name
+    DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+    // Specify name server addresses
+    producer.setNamesrvAddr("localhost:9876");
+    // Launch the producer instance
+    producer.start();
+    producer.setRetryTimesWhenSendAsyncFailed(0);
+    for (int i = 0; i < 100; i++) {
+      final int index = i;
+      // Create a message instance with specifying topic, tag and message body
+      Message msg = new Message("TopicTest",
+        "TagA",
+        "OrderID188",
+        "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+      // SendCallback: receive the callback of the asynchronous return result.
+      producer.send(msg, new SendCallback() {
+        @Override
+        public void onSuccess(SendResult sendResult) {
+          System.out.printf("%-10d OK %s %n", index,
+            sendResult.getMsgId());
+        }
+        @Override
+        public void onException(Throwable e) {
+          System.out.printf("%-10d Exception %s %n", index, e);
+          e.printStackTrace();
+        }
+      });
+    }
+    // Shut down once the producer instance is not longer in use
+    producer.shutdown();
+  }
+}
+```
+##### 2.3 以单向模式发送消息
+单向传输用于需要中等可靠性的情况，如日志收集。
+``` java
+public class OnewayProducer {
+  public static void main(String[] args) throws Exception{
+    // Instantiate with a producer group name
+    DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+    // Specify name server addresses
+    producer.setNamesrvAddr("localhost:9876");
+    // Launch the producer instance
+    producer.start();
+    for (int i = 0; i < 100; i++) {
+      // Create a message instance with specifying topic, tag and message body
+      Message msg = new Message("TopicTest" /* Topic */,
+        "TagA" /* Tag */,
+        ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
+      );
+      // Send in one-way mode, no return result
+      producer.sendOneway(msg);
+    }
+    // Shut down once the producer instance is not longer in use
+     producer.shutdown();
+  }
+}
+```
+### 3 消费消息
+``` java
+public class Consumer {
+  public static void main(String[] args) throws InterruptedException, MQClientException {
+    // Instantiate with specified consumer group name
+    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name");
+    
+    // Specify name server addresses
+    consumer.setNamesrvAddr("localhost:9876");
+
+    // Subscribe one or more topics and tags for finding those messages need to be consumed
+    consumer.subscribe("TopicTest", "*");
+    // Register callback to execute on arrival of messages fetched from brokers
+    consumer.registerMessageListener(new MessageListenerConcurrently() {
+      @Override
+      public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+        System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+        // Mark the message that have been consumed successfully
+        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+      }
+    });
+    // Launch the consumer instance
+    consumer.start();
+    System.out.printf("Consumer Started.%n");
+  }
+}
+```

docs/cn/FAQ.md

@@ -0,0 +1,110 @@
+# 经常被问到的问题
+
+以下是关于RocketMQ项目的常见问题
+
+## 1 基本
+
+1. **为什么我们要使用RocketMQ而不是选择其他的产品？**
+
+   请参考[为什么要选择RocketMQ](http://rocketmq.apache.org/docs/motivation/)
+
+2. **我是否需要安装其他的软件才能使用RocketMQ，例如zookeeper？**
+
+   不需要，RocketMQ可以独立的运行。
+
+## 2 使用
+
+1. **新创建的Consumer ID从哪里开始消费消息？**
+
+   1）如果发送的消息在三天之内，那么消费者会从服务器中保存的第一条消息开始消费。
+
+   2）如果发送的消息已经超过三天，则消费者会从服务器中的最新消息开始消费，也就是从队列的尾部开始消费。
+
+   3）如果消费者重新启动，那么它会从最后一个消费位置开始消费消息。
+
+2. **当消费失败的时候如何重新消费消息？**
+
+   1）在集群模式下，消费的业务逻辑代码会返回Action.ReconsumerLater，NULL，或者抛出异常，如果一条消息消费失败，最多会重试16次，之后该消息会被丢弃。
+
+   2）在广播消费模式下，广播消费仍然保证消息至少被消费一次，但不提供重发的选项。
+
+3. **当消费失败的时候如何找到失败的消息？**
+
+   1）使用按时间的主题查询，可以查询到一段时间内的消息。
+
+   2）使用主题和消息ID来准确查询消息。
+
+   3）使用主题和消息的Key来准确查询所有消息Key相同的消息。
+
+4. **消息只会被传递一次吗？**
+
+   RocketMQ 确保所有消息至少传递一次。 在大多数情况下，消息不会重复。
+
+5. **如何增加一个新的Broker？**
+
+   1）启动一个新的Broker并将其注册到name server中的Broker列表里。
+
+   2）默认只自动创建内部系统topic和consumer group。 如果您希望在新节点上拥有您的业务主题和消费者组，请从现有的Broker中复制它们。 我们提供了管理工具和命令行来处理此问题。
+
+## 3 配置相关
+
+以下回答均为默认值，可通过配置修改。
+
+1. **消息在服务器上可以保存多长时间？**
+
+   存储的消息将最多保存 3 天，超过 3 天未使用的消息将被删除。
+
+2. **消息体的大小限制是多少？**
+
+   通常是256KB
+
+3. **怎么设置消费者线程数？**
+
+   当你启动消费者的时候，可以设置 ConsumeThreadNums属性的值，举例如下：
+
+   ```java
+   consumer.setConsumeThreadMin(20);
+   consumer.setConsumeThreadMax(20);
+   ```
+
+## 4 错误
+
+1. **当你启动一个生产者或消费者的过程失败了并且错误信息是生产者组或消费者重复**
+
+   原因：使用同一个Producer/Consumer Group在同一个JVM中启动多个Producer/Consumer实例可能会导致客户端无法启动。
+
+   解决方案：确保一个 Producer/Consumer Group 对应的 JVM 只启动一个 Producer/Consumer 实例。
+
+2. **消费者无法在广播模式下开始加载 json 文件**
+
+   原因：fastjson 版本太低，无法让广播消费者加载本地 offsets.json，导致消费者启动失败。 损坏的 fastjson 文件也会导致同样的问题。
+
+   解决方案：Fastjson 版本必须升级到 RocketMQ 客户端依赖版本，以确保可以加载本地 offsets.json。 默认情况下，offsets.json 文件在 /home/{user}/.rocketmq_offsets 中。 或者检查fastjson的完整性。
+
+3. **Broker崩溃以后有什么影响？**
+
+   1）Master节点崩溃
+
+   消息不能再发送到该Broker集群，但是如果您有另一个可用的Broker集群，那么在主题存在的条件下仍然可以发送消息。消息仍然可以从Slave节点消费。
+
+   2）一些Slave节点崩溃
+
+   只要有另一个工作的slave，就不会影响发送消息。 对消费消息也不会产生影响，除非消费者组设置为优先从该Slave消费。 默认情况下，消费者组从 master 消费。
+
+   3）所有Slave节点崩溃
+
+   向master发送消息不会有任何影响，但是，如果master是SYNC_MASTER，producer会得到一个SLAVE_NOT_AVAILABLE，表示消息没有发送给任何slave。 对消费消息也没有影响，除非消费者组设置为优先从slave消费。 默认情况下，消费者组从master消费。
+
+4. **Producer提示“No Topic Route Info”，如何诊断？**
+
+   当您尝试将消息发送到一个路由信息对生产者不可用的主题时，就会发生这种情况。
+
+   1）确保生产者可以连接到名称服务器并且能够从中获取路由元信息。
+
+   2）确保名称服务器确实包含主题的路由元信息。 您可以使用管理工具或 Web 控制台通过 topicRoute 从名称服务器查询路由元信息。
+
+   3）确保您的Broker将心跳发送到您的生产者正在连接的同一name server列表。
+
+   4）确保主题的权限为6(rw-)，或至少为2(-w-)。
+
+   如果找不到此主题，请通过管理工具命令updateTopic或Web控制台在Broker上创建它。

docs/cn/best_practice.md

@@ -186,8 +186,8 @@ msgId一定是全局唯一标识符，但是实际使用中，可能会存在相
 | brokerName        | null                         | broker 的名称                           |
 | brokerClusterName                     | DefaultCluster                  | 本 broker 所属的 Cluser 名称           |
 | brokerId             | 0                              | broker id, 0 表示 master, 其他的正整数表示 slave                                                 |
+| storePathRootDir                         | $HOME/store/                   | 存储根路径                                            |
 | storePathCommitLog                      | $HOME/store/commitlog/                              | 存储 commit log 的路径                                                |
-| storePathConsumerQueue                   | $HOME/store/consumequeue/                              | 存储 consume queue 的路径                                              |
 | mappedFileSizeCommitLog     | 1024 * 1024 * 1024(1G) | commit log 的映射文件大小                                       |​ 
 | deleteWhen     | 04 | 在每天的什么时间删除已经超过文件保留时间的 commit log                                        |​ 
 | fileReservedTime     | 72 | 以小时计算的文件保留时间                                        |​ 

docs/cn/design.md

@@ -10,7 +10,7 @@
 #### 1.1 消息存储整体架构
 消息存储架构图中主要有下面三个跟消息存储相关的文件构成。
 
-(1) CommitLog：消息主体以及元数据的存储主体，存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G ，文件名长度为20位，左边补零，剩余为起始偏移量，比如00000000000000000000代表了第一个文件，起始偏移量为0，文件大小为1G=1073741824；当第一个文件写满了，第二个文件为00000000001073741824，起始偏移量为1073741824，以此类推。消息主要是顺序写入日志文件，当文件满了，写入下一个文件；
+(1) CommitLog：消息主体以及元数据的存储主体，存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G, 文件名长度为20位，左边补零，剩余为起始偏移量，比如00000000000000000000代表了第一个文件，起始偏移量为0，文件大小为1G=1073741824；当第一个文件写满了，第二个文件为00000000001073741824，起始偏移量为1073741824，以此类推。消息主要是顺序写入日志文件，当文件满了，写入下一个文件；
 
 (2) ConsumeQueue：消息消费队列，引入的目的主要是提高消息消费的性能，由于RocketMQ是基于主题topic的订阅模式，消息消费是针对主题进行的，如果要遍历commitlog文件中根据topic检索消息是非常低效的。Consumer即可根据ConsumeQueue来查找待消费的消息。其中，ConsumeQueue（逻辑消费队列）作为消费消息的索引，保存了指定Topic下的队列消息在CommitLog中的起始物理偏移量offset，消息大小size和消息Tag的HashCode值。consumequeue文件可以看成是基于topic的commitlog索引文件，故consumequeue文件夹的组织方式如下：topic/queue/file三层组织结构，具体存储路径为：$HOME/store/consumequeue/{topic}/{queueId}/{fileName}。同样consumequeue文件采取定长设计，每一个条目共20个字节，分别为8字节的commitlog物理偏移量、4字节的消息长度、8字节tag hashcode，单个文件由30W个条目组成，可以像数组一样随机访问每一个条目，每个ConsumeQueue文件大小约5.72M；
 
@@ -95,7 +95,7 @@ M1 | NettyServerCodecThread_%d | Worker线程池
 M2 | RemotingExecutorThread_%d | 业务processor处理线程池
 
 ### 3 消息过滤
-RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正式基于这个字段值的。
+RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正是基于这个字段值的。
 
 ![](image/rocketmq_design_7.png)
 

docs/cn/msg_trace/user_guide.md

@@ -100,6 +100,19 @@ RocketMQ的消息轨迹特性支持两种存储轨迹数据的方式：
         ......
 ```
 
-
-
-
+### 4.4 使用mqadmin命令发送和查看轨迹
+- 发送消息
+```shell
+./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p "your meesgae content"
+```
+- 查询轨迹
+```shell
+./mqadmin QueryMsgTraceById -n 127.0.0.1:9876 -i "some-message-id"
+```
+- 查询轨迹结果
+```
+RocketMQLog:WARN No appenders could be found for logger (io.netty.util.internal.PlatformDependent0).
+RocketMQLog:WARN Please initialize the logger system properly.
+#Type      #ProducerGroup       #ClientHost          #SendTime            #CostTimes #Status
+Pub        1623305799667        xxx.xxx.xxx.xxx       2021-06-10 14:16:40  131ms      success
+```

docs/en/Example_Transaction.md

@@ -89,7 +89,7 @@ public class TransactionListenerImpl implements TransactionListener {
 ## 3 Usage Constraint  
 1. Messages of the transactional have no schedule and batch support.
 2. In order to avoid a single message being checked too many times and lead to half queue message accumulation,  we limited the number of checks for a single message to 15 times by default, but users can change this limit by change the ```transactionCheckMax``` parameter in the configuration of the broker,  if one message has been checked over ```transactionCheckMax``` times,  broker will discard this message and print an error log at the same time by default. Users can change this behavior by override the ```AbstractTransactionalMessageCheckListener``` class.
-3. A transactional message will be checked after a certain period of time that determined by parameter g```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter. 
+3. A transactional message will be checked after a certain period of time that determined by parameter ```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter. 
 4. A transactional message maybe checked or consumed more than once. 
 5. Committed message reput to the user’s target topic may fail. Currently, it depends on the log record. High availability is ensured by the high availability mechanism of RocketMQ itself. If you want to ensure that the transactional message isn’t lost and the transaction integrity is guaranteed, it is recommended to use synchronous double write. mechanism. 
 6. Producer IDs of transactional messages cannot be shared with producer IDs of other types of messages. Unlike other types of message, transactional messages allow backward queries. MQ Server query clients by their Producer IDs.

docs/en/best_practice.md

@@ -20,8 +20,8 @@
 | brokerName        | null                         | broker name                           |
 | brokerClusterName                     | DefaultCluster                  | this broker belongs to which cluster           |
 | brokerId             | 0                              | broker id, 0 means master, positive integers mean slave                                                 |
+| storePathRootDir                         | $HOME/store/                   | file path for root store                                            |
 | storePathCommitLog                      | $HOME/store/commitlog/                              | file path for commit log                                                 |
-| storePathConsumerQueue                   | $HOME/store/consumequeue/                              | file path for consume queue                                              |
 | mappedFileSizeCommitLog     | 1024 * 1024 * 1024(1G) | mapped file size for commit log                                        |​ 
 | deleteWhen     | 04 | When to delete the commitlog which is out of the reserve time                                        |​ 
 | fileReserverdTime     | 72 | The number of hours to keep a commitlog before deleting it                                        |​ 

docs/en/client/java/API_Reference_DefaultMQProducer.md

@@ -0,0 +1,71 @@
+## DefaultMQProducer
+---
+### Class introduction
+
+`public class DefaultMQProducer 
+extends ClientConfig 
+implements MQProducer`
+
+>`DefaultMQProducer` is the entry point for an application to post messages, out of the box，ca  quickly create a producer with a no-argument construction. it is mainly responsible for message sending, support synchronous、asynchronous、one-way send. All of these send methods support batch send.  The parameters of the sender can be adjusted through the getter/setter methods , provided by this class. `DefaultMQProducer` has multi send method and each method is slightly different. Make  sure you know the usage before you use it . Blow is a producer example . [see more examples](https://github.com/apache/rocketmq/blob/master/example/src/main/java/org/apache/rocketmq/example/)。
+
+``` java 
+public class Producer {
+    public static void main(String[] args) throws MQClientException {
+        // create a produce with producer_group_name
+        DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName");
+
+        // start the producer
+        producer.start();
+
+        for (int i = 0; i < 128; i++)
+            try {
+            	// construct the msg
+                Message msg = new Message("TopicTest",
+                        "TagA",
+                        "OrderID188",
+                        "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+
+                // send sync
+                SendResult sendResult = producer.send(msg);
+
+                // print the result
+                System.out.printf("%s%n", sendResult);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+
+        producer.shutdown();
+    }
+}
+```
+
+**Note** : This class is thread safe. It can be safely shared between multiple threads after configuration and startup is complete.
+
+###  Variable 
+
+|Type|Name| description                                                  |
+|------|-------|-------|
+|DefaultMQProducerImpl|defaultMQProducerImpl|The producer's internal default implementation|
+|String|producerGroup|The producer's group|
+|String|createTopicKey| Topics that do not exist on the server are automatically created when the message is sent |
+|int|defaultTopicQueueNums|The default number of queues to create a topic|
+|int|sendMsgTimeout|The timeout for the message to be sent|
+|int|compressMsgBodyOverHowmuch|the threshold of the compress of  message body|
+|int|retryTimesWhenSendFailed|Maximum number of internal attempts to send a message in synchronous mode|
+|int|retryTimesWhenSendAsyncFailed|Maximum number of internal attempts to send a message in asynchronous mode|
+|boolean|retryAnotherBrokerWhenNotStoreOK|Whether to retry another broker if an internal send fails|
+|int|maxMessageSize| Maximum length of message                                    |
+|TraceDispatcher|traceDispatcher| Message trackers. Use rcpHook to track messages              |
+
+### construction method 
+
+|方法名称|方法描述|
+|-------|------------|
+|DefaultMQProducer()| creates a producer with default parameter values             |
+|DefaultMQProducer(final String producerGroup)| creates a producer with producer group name.                 |
+|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace)|creates a producer with producer group name and set whether to enable message tracking|
+|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic)|creates a producer with producer group name and set whether to enable message tracking、the trace topic.|
+|DefaultMQProducer(RPCHook rpcHook)|creates a producer with  a rpc hook.|
+|DefaultMQProducer(final String producerGroup, RPCHook rpcHook)|creates a producer with  a rpc hook and producer group.|
+|DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic)|all of above.|
+

docs/en/msg_trace/user_guide.md

@@ -101,5 +101,21 @@ Adjusting instantiation of DefaultMQProducer and DefaultMQPushConsumer as follow
 ```
 
 
+### 4.4 Send and query message trace by mqadmin command
+- send message
+```shell
+./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p "your meesgae content"
+```
+- query trace
+```shell
+./mqadmin QueryMsgTraceById -n 127.0.0.1:9876 -i "some-message-id"
+```
+- query trace result
+```
+RocketMQLog:WARN No appenders could be found for logger (io.netty.util.internal.PlatformDependent0).
+RocketMQLog:WARN Please initialize the logger system properly.
+#Type      #ProducerGroup       #ClientHost          #SendTime            #CostTimes #Status
+Pub        1623305799667        xxx.xxx.xxx.xxx       2021-06-10 14:16:40  131ms      success
+```
 
 

example/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java

@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.benchmark;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.srvutil.ServerUtil;
+
+public class BatchProducer {
+
+    public static void main(String[] args) throws MQClientException, UnsupportedEncodingException {
+
+        Options options = ServerUtil.buildCommandlineOptions(new Options());
+        CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkBatchProducer", args, buildCommandlineOptions(options), new PosixParser());
+        if (null == commandLine) {
+            System.exit(-1);
+        }
+
+        final String namesrv = getOptionValue(commandLine, 'n', "127.0.0.1:9876");
+        final String topic = getOptionValue(commandLine, 't', "BenchmarkTest");
+        final int threadCount = getOptionValue(commandLine, 'w', 64);
+        final int messageSize = getOptionValue(commandLine, 's', 128);
+        final int batchSize = getOptionValue(commandLine, 'b', 16);
+        final boolean keyEnable = getOptionValue(commandLine, 'k', false);
+        final int propertySize = getOptionValue(commandLine, 'p', 0);
+        final int tagCount = getOptionValue(commandLine, 'l', 0);
+        final boolean msgTraceEnable = getOptionValue(commandLine, 'm', false);
+        final boolean aclEnable = getOptionValue(commandLine, 'a', false);
+        final String ak = getOptionValue(commandLine, 'c', "rocketmq2");
+        final String sk = getOptionValue(commandLine, 'e', "12346789");
+
+        System.out.printf("topic: %s threadCount: %d messageSize: %d batchSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s%n",
+                topic, threadCount, messageSize, batchSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable);
+
+        final StatsBenchmarkBatchProducer statsBenchmark = new StatsBenchmarkBatchProducer();
+        statsBenchmark.start();
+
+        final DefaultMQProducer producer = initInstance(namesrv, msgTraceEnable, aclEnable, ak, sk);
+        producer.start();
+
+        final InternalLogger log = ClientLogger.getLog();
+        final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount);
+        for (int i = 0; i < threadCount; i++) {
+            sendThreadPool.execute(new Runnable() {
+                @Override
+                public void run() {
+                    while (true) {
+                        List<Message> msgs;
+
+                        try {
+                            msgs = buildBathMessage(batchSize, messageSize, topic);
+                        } catch (UnsupportedEncodingException e) {
+                            e.printStackTrace();
+                            return;
+                        }
+
+                        if (CollectionUtils.isEmpty(msgs)) {
+                            return;
+                        }
+
+                        try {
+                            long beginTimestamp = System.currentTimeMillis();
+                            long sendSucCount = statsBenchmark.getSendMessageSuccessCount().get();
+
+                            setKeys(keyEnable, msgs, String.valueOf(beginTimestamp / 1000));
+                            setTags(tagCount, msgs, sendSucCount);
+                            setProperties(propertySize, msgs);
+                            SendResult sendResult = producer.send(msgs);
+                            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {
+                                statsBenchmark.getSendRequestSuccessCount().incrementAndGet();
+                                statsBenchmark.getSendMessageSuccessCount().addAndGet(msgs.size());
+                            } else {
+                                statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                                statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            }
+                            long currentRT = System.currentTimeMillis() - beginTimestamp;
+                            statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT);
+                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+                            while (currentRT > prevMaxRT) {
+                                boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT);
+                                if (updated) {
+                                    break;
+                                }
+
+                                prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+                            }
+                        } catch (RemotingException e) {
+                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+
+                            try {
+                                Thread.sleep(3000);
+                            } catch (InterruptedException ignored) {
+                            }
+                        } catch (InterruptedException e) {
+                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            try {
+                                Thread.sleep(3000);
+                            } catch (InterruptedException e1) {
+                            }
+                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+                        } catch (MQClientException e) {
+                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+                        } catch (MQBrokerException e) {
+                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            log.error("[BENCHMARK_PRODUCER] Send Exception", e);
+                            try {
+                                Thread.sleep(3000);
+                            } catch (InterruptedException ignored) {
+                            }
+                        }
+                    }
+                }
+            });
+        }
+    }
+
+    public static Options buildCommandlineOptions(final Options options) {
+        Option opt = new Option("w", "threadCount", true, "Thread count, Default: 64");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("s", "messageSize", true, "Message Size, Default: 128");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("b", "batchSize", true, "Batch Size, Default: 16");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("k", "keyEnable", true, "Message Key Enable, Default: false");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("l", "tagCount", true, "Tag count, Default: 0");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("a", "aclEnable", true, "Acl Enable, Default: false");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("c", "accessKey", true, "Acl Access Key, Default: rocketmq2");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("e", "secretKey", true, "Acl Secret Key, Default: 123456789");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("p", "propertySize", true, "Property Size, Default: 0");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("n", "namesrv", true, "name server, Default: 127.0.0.1:9876");
+        opt.setRequired(false);
+        options.addOption(opt);
+        return options;
+    }
+
+    private static String getOptionValue(CommandLine commandLine, char key, String defaultValue) {
+        if (commandLine.hasOption(key)) {
+            return commandLine.getOptionValue(key).trim();
+        }
+        return defaultValue;
+    }
+
+    private static int getOptionValue(CommandLine commandLine, char key, int defaultValue) {
+        if (commandLine.hasOption(key)) {
+            return Integer.parseInt(commandLine.getOptionValue(key).trim());
+        }
+        return defaultValue;
+    }
+
+    private static boolean getOptionValue(CommandLine commandLine, char key, boolean defaultValue) {
+        if (commandLine.hasOption(key)) {
+            return Boolean.parseBoolean(commandLine.getOptionValue(key).trim());
+        }
+        return defaultValue;
+    }
+
+    private static List<Message> buildBathMessage(int batchSize, int messageSize,
+                                                  String topic) throws UnsupportedEncodingException {
+        List<Message> batchMessage = new ArrayList<>(batchSize);
+
+        for (int i = 0; i < batchSize; i++) {
+            Message msg = new Message();
+            msg.setTopic(topic);
+
+            StringBuilder sb = new StringBuilder();
+            for (int j = 0; j < messageSize; j += 10) {
+                sb.append("hello baby");
+            }
+
+            msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET));
+            batchMessage.add(msg);
+        }
+
+        return batchMessage;
+    }
+
+    private static void setKeys(boolean keyEnable, List<Message> msgs, String keys) {
+        if (!keyEnable) {
+            return;
+        }
+
+        for (Message msg : msgs) {
+            msg.setKeys(keys);
+        }
+    }
+
+    private static void setTags(int tagCount, List<Message> msgs, long startTagId) {
+        if (tagCount <= 0) {
+            return;
+        }
+
+        long tagId = startTagId % tagCount;
+        for (Message msg : msgs) {
+            msg.setTags(String.format("tag%d", tagId++));
+        }
+    }
+
+    private static void setProperties(int propertySize, List<Message> msgs) {
+        if (propertySize <= 0) {
+            return;
+        }
+
+        for (Message msg : msgs) {
+            if (msg.getProperties() != null) {
+                msg.getProperties().clear();
+            }
+
+            int startValue = (new Random(System.currentTimeMillis())).nextInt(100);
+            int size = 0;
+            for (int i = 0; ; i++) {
+                String prop1 = "prop" + i, prop1V = "hello" + startValue;
+                msg.putUserProperty(prop1, prop1V);
+                size += prop1.length() + prop1V.length();
+                if (size > propertySize) {
+                    break;
+                }
+                startValue++;
+            }
+        }
+    }
+
+    private static DefaultMQProducer initInstance(String namesrv, boolean traceEnable, boolean aclEnable, String ak,
+                                                  String sk) {
+        RPCHook rpcHook = aclEnable ? new AclClientRPCHook(new SessionCredentials(ak, sk)) : null;
+        final DefaultMQProducer producer = new DefaultMQProducer("benchmark_batch_producer", rpcHook, traceEnable, null);
+        producer.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        producer.setNamesrvAddr(namesrv);
+        producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);
+        return producer;
+    }
+}
+
+class StatsBenchmarkBatchProducer {
+
+    private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L);
+
+    private final AtomicLong sendRequestFailedCount = new AtomicLong(0L);
+
+    private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L);
+
+    private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);
+
+    private final AtomicLong sendMessageSuccessCount = new AtomicLong(0L);
+
+    private final AtomicLong sendMessageFailedCount = new AtomicLong(0L);
+
+    private final Timer timer = new Timer("BenchmarkTimerThread", true);
+
+    private final LinkedList<Long[]> snapshotList = new LinkedList<>();
+
+    public Long[] createSnapshot() {
+        Long[] snap = new Long[] {
+                System.currentTimeMillis(),
+                this.sendRequestSuccessCount.get(),
+                this.sendRequestFailedCount.get(),
+                this.sendMessageSuccessCount.get(),
+                this.sendMessageFailedCount.get(),
+                this.sendMessageSuccessTimeTotal.get(),
+        };
+
+        return snap;
+    }
+
+    public AtomicLong getSendRequestSuccessCount() {
+        return sendRequestSuccessCount;
+    }
+
+    public AtomicLong getSendRequestFailedCount() {
+        return sendRequestFailedCount;
+    }
+
+    public AtomicLong getSendMessageSuccessTimeTotal() {
+        return sendMessageSuccessTimeTotal;
+    }
+
+    public AtomicLong getSendMessageMaxRT() {
+        return sendMessageMaxRT;
+    }
+
+    public AtomicLong getSendMessageSuccessCount() {
+        return sendMessageSuccessCount;
+    }
+
+    public AtomicLong getSendMessageFailedCount() {
+        return sendMessageFailedCount;
+    }
+
+    public void start() {
+
+        timer.scheduleAtFixedRate(new TimerTask() {
+            @Override
+            public void run() {
+                snapshotList.addLast(createSnapshot());
+                if (snapshotList.size() > 10) {
+                    snapshotList.removeFirst();
+                }
+            }
+        }, 1000, 1000);
+
+        timer.scheduleAtFixedRate(new TimerTask() {
+            private void printStats() {
+                if (snapshotList.size() >= 10) {
+                    Long[] begin = snapshotList.getFirst();
+                    Long[] end = snapshotList.getLast();
+
+                    final long sendTps = (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L);
+                    final long sendMps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L);
+                    final double averageRT = (end[5] - begin[5]) / (double) (end[1] - begin[1]);
+                    final double averageMsgRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]);
+
+                    System.out.printf("Current Time: %s Send TPS: %d Send MPS: %d Max RT(ms): %d Average RT(ms): %7.3f Average Message RT(ms): %7.3f Send Failed: %d Send Message Failed: %d%n",
+                            System.currentTimeMillis(), sendTps, sendMps, getSendMessageMaxRT().get(), averageRT, averageMsgRT, end[2], end[4]);
+                }
+            }
+
+            @Override
+            public void run() {
+                try {
+                    this.printStats();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }, 10000, 10000);
+    }
+
+    public void shutdown() {
+        timer.cancel();
+    }
+}

example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java

@@ -189,8 +189,7 @@ public class Consumer {
         opt = new Option("g", "group", true, "Consumer group name, Default: benchmark_consumer");
         opt.setRequired(false);
         options.addOption(opt);
-
-        opt = new Option("p", "group suffix enable", true, "Consumer group suffix enable, Default: false");
+        opt = new Option("p", "group prefix enable", true, "Is group prefix enable, Default: false");
         opt.setRequired(false);
         options.addOption(opt);
 

example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java

@@ -61,9 +61,11 @@ public class Producer {
         final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m'));
         final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a'));
         final long messageNum = commandLine.hasOption('q') ? Long.parseLong(commandLine.getOptionValue('q')) : 0;
+        final boolean delayEnable = commandLine.hasOption('d') && Boolean.parseBoolean(commandLine.getOptionValue('d'));
+        final int delayLevel = commandLine.hasOption('e') ? Integer.parseInt(commandLine.getOptionValue('e')) : 1;
 
-        System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n",
-            topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum);
+        System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n delayEnable: %s%n delayLevel: %s%n",
+            topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum, delayEnable, delayLevel);
 
         final InternalLogger log = ClientLogger.getLog();
 
@@ -147,6 +149,9 @@ public class Producer {
                             if (keyEnable) {
                                 msg.setKeys(String.valueOf(beginTimestamp / 1000));
                             }
+                            if (delayEnable) {
+                                msg.setDelayTimeLevel(delayLevel);
+                            }
                             if (tagCount > 0) {
                                 long sendSucCount = statsBenchmark.getReceiveResponseSuccessCount().get();
                                 msg.setTags(String.format("tag%d", sendSucCount % tagCount));
@@ -266,6 +271,14 @@ public class Producer {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt = new Option("d", "delayEnable", true, "Delay message Enable, Default: false");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("e", "delayLevel", true, "Delay message level, Default: 1");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 

example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java

@@ -69,6 +69,7 @@ public class TransactionProducer {
         config.batchId = commandLine.hasOption("b") ? Long.parseLong(commandLine.getOptionValue("b")) : System.currentTimeMillis();
         config.sendInterval = commandLine.hasOption("i") ? Integer.parseInt(commandLine.getOptionValue("i")) : 0;
         config.aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a'));
+        config.msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m'));
 
         final ExecutorService sendThreadPool = Executors.newFixedThreadPool(config.threadCount);
 
@@ -123,8 +124,12 @@ public class TransactionProducer {
         }, 10000, 10000);
 
         final TransactionListener transactionCheckListener = new TransactionListenerImpl(statsBenchmark, config);
-        final TransactionMQProducer producer =
-            new TransactionMQProducer("benchmark_transaction_producer", config.aclEnable ? AclClient.getAclRPCHook() : null);
+        final TransactionMQProducer producer = new TransactionMQProducer(
+                null,
+                "benchmark_transaction_producer",
+                config.aclEnable ? AclClient.getAclRPCHook() : null,
+                config.msgTraceEnable,
+                null);
         producer.setInstanceName(Long.toString(System.currentTimeMillis()));
         producer.setTransactionListener(transactionCheckListener);
         producer.setDefaultTopicQueueNums(1000);
@@ -256,6 +261,10 @@ public class TransactionProducer {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 }
@@ -439,6 +448,7 @@ class TxSendConfig {
     long batchId;
     int sendInterval;
     boolean aclEnable;
+    boolean msgTraceEnable;
 }
 
 class LRUMap<K, V> extends LinkedHashMap<K, V> {

example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java

@@ -65,6 +65,39 @@ public class Producer {
                  * Call send message to deliver message to one of brokers.
                  */
                 SendResult sendResult = producer.send(msg);
+                /*
+                 * There are different ways to send message, if you don't care about the send result,you can use this way
+                 * {@code
+                 * producer.sendOneway(msg);
+                 * }
+                 */
+
+                /*
+                 * if you want to get the send result in a synchronize way, you can use this send method
+                 * {@code
+                 * SendResult sendResult = producer.send(msg);
+                 * System.out.printf("%s%n", sendResult);
+                 * }
+                 */
+
+                /*
+                 * if you want to get the send result in a asynchronize way, you can use this send method
+                 * {@code
+                 *
+                 *  producer.send(msg, new SendCallback() {
+                 *  @Override
+                 *  public void onSuccess(SendResult sendResult) {
+                 *      // do something
+                 *  }
+                 *
+                 *  @Override
+                 *  public void onException(Throwable e) {
+                 *      // do something
+                 *  }
+                 *});
+                 *
+                 *}
+                 */
 
                 System.out.printf("%s%n", sendResult);
             } catch (Exception e) {

filter/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

logappender/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>rocketmq-logappender</artifactId>

logging/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

namesrv/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

openmessaging/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

pom.xml

@@ -29,7 +29,7 @@
     <inceptionYear>2012</inceptionYear>
     <groupId>org.apache.rocketmq</groupId>
     <artifactId>rocketmq-all</artifactId>
-    <version>4.9.1-SNAPSHOT</version>
+    <version>4.9.2-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>Apache RocketMQ ${project.version}</name>
     <url>http://rocketmq.apache.org/</url>
@@ -537,7 +537,7 @@
             <dependency>
                 <groupId>io.netty</groupId>
                 <artifactId>netty-all</artifactId>
-                <version>4.0.42.Final</version>
+                <version>4.1.65.Final</version>
             </dependency>
             <dependency>
                 <groupId>com.alibaba</groupId>

remoting/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java

@@ -20,23 +20,23 @@ public class NettyClientConfig {
     /**
      * Worker thread number
      */
-    private int clientWorkerThreads = 4;
+    private int clientWorkerThreads = NettySystemConfig.clientWorkerSize;
     private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors();
     private int clientOnewaySemaphoreValue = NettySystemConfig.CLIENT_ONEWAY_SEMAPHORE_VALUE;
     private int clientAsyncSemaphoreValue = NettySystemConfig.CLIENT_ASYNC_SEMAPHORE_VALUE;
-    private int connectTimeoutMillis = 3000;
+    private int connectTimeoutMillis = NettySystemConfig.connectTimeoutMillis;
     private long channelNotActiveInterval = 1000 * 60;
 
     /**
      * IdleStateEvent will be triggered when neither read nor write was performed for
      * the specified period of this time. Specify {@code 0} to disable
      */
-    private int clientChannelMaxIdleTimeSeconds = 120;
+    private int clientChannelMaxIdleTimeSeconds = NettySystemConfig.clientChannelMaxIdleTimeSeconds;
 
     private int clientSocketSndBufSize = NettySystemConfig.socketSndbufSize;
     private int clientSocketRcvBufSize = NettySystemConfig.socketRcvbufSize;
     private boolean clientPooledByteBufAllocatorEnable = false;
-    private boolean clientCloseSocketIfTimeout = false;
+    private boolean clientCloseSocketIfTimeout = NettySystemConfig.clientCloseSocketIfTimeout;
 
     private boolean useTLS;
 

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java

@@ -52,6 +52,8 @@ public class NettyLogger {
 
         private InternalLogger logger = null;
 
+        private static final String EXCEPTION_MESSAGE = "Unexpected exception:";
+
         public NettyBridgeLogger(String name) {
             logger = InternalLoggerFactory.getLogger(name);
         }
@@ -161,6 +163,25 @@ public class NettyLogger {
             }
         }
 
+        @Override
+        public void log(InternalLogLevel internalLogLevel, Throwable throwable) {
+            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {
+                logger.debug(EXCEPTION_MESSAGE, throwable);
+            }
+            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
+                logger.info(EXCEPTION_MESSAGE, throwable);
+            }
+            if (internalLogLevel.equals(InternalLogLevel.INFO)) {
+                logger.info(EXCEPTION_MESSAGE, throwable);
+            }
+            if (internalLogLevel.equals(InternalLogLevel.WARN)) {
+                logger.warn(EXCEPTION_MESSAGE, throwable);
+            }
+            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {
+                logger.error(EXCEPTION_MESSAGE, throwable);
+            }
+        }
+
         @Override
         public boolean isTraceEnabled() {
             return isEnabled(InternalLogLevel.TRACE);
@@ -191,6 +212,11 @@ public class NettyLogger {
             logger.info(var1, var2);
         }
 
+        @Override
+        public void trace(Throwable var1) {
+            logger.info(EXCEPTION_MESSAGE, var1);
+        }
+
         @Override
         public boolean isDebugEnabled() {
             return isEnabled(InternalLogLevel.DEBUG);
@@ -221,6 +247,11 @@ public class NettyLogger {
             logger.debug(var1, var2);
         }
 
+        @Override
+        public void debug(Throwable var1) {
+            logger.debug(EXCEPTION_MESSAGE, var1);
+        }
+
         @Override
         public boolean isInfoEnabled() {
             return isEnabled(InternalLogLevel.INFO);
@@ -251,6 +282,11 @@ public class NettyLogger {
             logger.info(var1, var2);
         }
 
+        @Override
+        public void info(Throwable var1) {
+            logger.info(EXCEPTION_MESSAGE, var1);
+        }
+
         @Override
         public boolean isWarnEnabled() {
             return isEnabled(InternalLogLevel.WARN);
@@ -281,6 +317,11 @@ public class NettyLogger {
             logger.warn(var1, var2);
         }
 
+        @Override
+        public void warn(Throwable var1) {
+            logger.warn(EXCEPTION_MESSAGE, var1);
+        }
+
         @Override
         public boolean isErrorEnabled() {
             return isEnabled(InternalLogLevel.ERROR);
@@ -310,6 +351,11 @@ public class NettyLogger {
         public void error(String var1, Throwable var2) {
             logger.error(var1, var2);
         }
+
+        @Override
+        public void error(Throwable var1) {
+            logger.error(EXCEPTION_MESSAGE, var1);
+        }
     }
 
 }

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java

@@ -553,7 +553,7 @@ public abstract class NettyRemotingAbstract {
                 throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast");
             } else {
                 String info = String.format(
-                    "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d",
+                    "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreOnewayValue: %d",
                     timeoutMillis,
                     this.semaphoreOneway.getQueueLength(),
                     this.semaphoreOneway.availablePermits()

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettySystemConfig.java

@@ -28,6 +28,14 @@ public class NettySystemConfig {
         "com.rocketmq.remoting.clientAsyncSemaphoreValue";
     public static final String COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE =
         "com.rocketmq.remoting.clientOnewaySemaphoreValue";
+    public static final String COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE =
+        "com.rocketmq.remoting.client.worker.size";
+    public static final String COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT =
+        "com.rocketmq.remoting.client.connect.timeout";
+    public static final String COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS =
+        "com.rocketmq.remoting.client.channel.maxIdleTimeSeconds";
+    public static final String COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT =
+        "com.rocketmq.remoting.client.closeSocketIfTimeout";
 
     public static final boolean NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE = //
         Boolean.parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE, "false"));
@@ -39,4 +47,12 @@ public class NettySystemConfig {
         Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "65535"));
     public static int socketRcvbufSize =
         Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "65535"));
+    public static int clientWorkerSize =
+        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "4"));
+    public static int connectTimeoutMillis =
+        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "3000"));
+    public static int clientChannelMaxIdleTimeSeconds =
+        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "120"));
+    public static boolean clientCloseSocketIfTimeout =
+        Boolean.parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "true"));
 }

remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.remoting.netty;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@RunWith(MockitoJUnitRunner.class)
+public class NettyClientConfigTest {
+
+  @Test
+  public void testChangeConfigBySystemProperty() throws NoSuchFieldException, IllegalAccessException {
+    
+
+    System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "1");
+    System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "2000");
+    System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "60");
+    System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "16383");
+    System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "16384");
+    System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "false");
+
+
+    NettySystemConfig.socketSndbufSize =
+        Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "65535"));
+    NettySystemConfig.socketRcvbufSize =
+        Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "65535"));
+    NettySystemConfig.clientWorkerSize =
+        Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, "4"));
+    NettySystemConfig.connectTimeoutMillis =
+        Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, "3000"));
+    NettySystemConfig.clientChannelMaxIdleTimeSeconds =
+        Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, "120"));
+    NettySystemConfig.clientCloseSocketIfTimeout =
+        Boolean.parseBoolean(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, "true"));
+
+    NettyClientConfig changedConfig = new NettyClientConfig();
+    assertThat(changedConfig.getClientWorkerThreads()).isEqualTo(1);
+    assertThat(changedConfig.getClientOnewaySemaphoreValue()).isEqualTo(65535);
+    assertThat(changedConfig.getClientAsyncSemaphoreValue()).isEqualTo(65535);
+    assertThat(changedConfig.getConnectTimeoutMillis()).isEqualTo(2000);
+    assertThat(changedConfig.getClientChannelMaxIdleTimeSeconds()).isEqualTo(60);
+    assertThat(changedConfig.getClientSocketSndBufSize()).isEqualTo(16383);
+    assertThat(changedConfig.getClientSocketRcvBufSize()).isEqualTo(16384);
+    assertThat(changedConfig.isClientCloseSocketIfTimeout()).isEqualTo(false);
+  }
+}

srvutil/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

store/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

store/src/main/java/org/apache/rocketmq/store/AppendMessageCallback.java

@@ -18,6 +18,7 @@ package org.apache.rocketmq.store;
 
 import java.nio.ByteBuffer;
 import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.store.CommitLog.PutMessageContext;
 
 /**
  * Write messages callback interface
@@ -30,7 +31,7 @@ public interface AppendMessageCallback {
      * @return How many bytes to write
      */
     AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer,
-        final int maxBlank, final MessageExtBrokerInner msg);
+        final int maxBlank, final MessageExtBrokerInner msg, PutMessageContext putMessageContext);
 
     /**
      * After batched message serialization, write MapedByteBuffer
@@ -39,5 +40,5 @@ public interface AppendMessageCallback {
      * @return How many bytes to write
      */
     AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer,
-        final int maxBlank, final MessageExtBatch messageExtBatch);
+        final int maxBlank, final MessageExtBatch messageExtBatch, PutMessageContext putMessageContext);
 }

store/src/main/java/org/apache/rocketmq/store/AppendMessageResult.java

@@ -16,6 +16,8 @@
  */
 package org.apache.rocketmq.store;
 
+import java.util.function.Supplier;
+
 /**
  * When write a message to the commit log, returns results
  */
@@ -28,6 +30,7 @@ public class AppendMessageResult {
     private int wroteBytes;
     // Message ID
     private String msgId;
+    private Supplier<String> msgIdSupplier;
     // Message storage timestamp
     private long storeTimestamp;
     // Consume queue's offset(step by one)
@@ -51,6 +54,17 @@ public class AppendMessageResult {
         this.pagecacheRT = pagecacheRT;
     }
 
+    public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, Supplier<String> msgIdSupplier,
+            long storeTimestamp, long logicsOffset, long pagecacheRT) {
+        this.status = status;
+        this.wroteOffset = wroteOffset;
+        this.wroteBytes = wroteBytes;
+        this.msgIdSupplier = msgIdSupplier;
+        this.storeTimestamp = storeTimestamp;
+        this.logicsOffset = logicsOffset;
+        this.pagecacheRT = pagecacheRT;
+    }
+
     public long getPagecacheRT() {
         return pagecacheRT;
     }
@@ -88,6 +102,9 @@ public class AppendMessageResult {
     }
 
     public String getMsgId() {
+        if (msgId == null && msgIdSupplier != null) {
+            msgId = msgIdSupplier.get();
+        }
         return msgId;
     }
 

store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java

@@ -34,6 +34,7 @@ import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -476,58 +477,20 @@ public class DefaultMessageStore implements MessageStore {
 
     @Override
     public PutMessageResult putMessage(MessageExtBrokerInner msg) {
-        PutMessageStatus checkStoreStatus = this.checkStoreStatus();
-        if (checkStoreStatus != PutMessageStatus.PUT_OK) {
-            return new PutMessageResult(checkStoreStatus, null);
+        try {
+            return asyncPutMessage(msg).get();
+        } catch (InterruptedException | ExecutionException e) {
+            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);
         }
-
-        PutMessageStatus msgCheckStatus = this.checkMessage(msg);
-        if (msgCheckStatus == PutMessageStatus.MESSAGE_ILLEGAL) {
-            return new PutMessageResult(msgCheckStatus, null);
-        }
-
-        long beginTime = this.getSystemClock().now();
-        PutMessageResult result = this.commitLog.putMessage(msg);
-        long elapsedTime = this.getSystemClock().now() - beginTime;
-        if (elapsedTime > 500) {
-            log.warn("not in lock elapsed time(ms)={}, bodyLength={}", elapsedTime, msg.getBody().length);
-        }
-
-        this.storeStatsService.setPutMessageEntireTimeMax(elapsedTime);
-
-        if (null == result || !result.isOk()) {
-            this.storeStatsService.getPutMessageFailedTimes().incrementAndGet();
-        }
-
-        return result;
     }
 
     @Override
     public PutMessageResult putMessages(MessageExtBatch messageExtBatch) {
-        PutMessageStatus checkStoreStatus = this.checkStoreStatus();
-        if (checkStoreStatus != PutMessageStatus.PUT_OK) {
-            return new PutMessageResult(checkStoreStatus, null);
+        try {
+            return asyncPutMessages(messageExtBatch).get();
+        } catch (InterruptedException | ExecutionException e) {
+            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);
         }
-
-        PutMessageStatus msgCheckStatus = this.checkMessages(messageExtBatch);
-        if (msgCheckStatus == PutMessageStatus.MESSAGE_ILLEGAL) {
-            return new PutMessageResult(msgCheckStatus, null);
-        }
-
-        long beginTime = this.getSystemClock().now();
-        PutMessageResult result = this.commitLog.putMessages(messageExtBatch);
-        long elapsedTime = this.getSystemClock().now() - beginTime;
-        if (elapsedTime > 500) {
-            log.warn("not in lock elapsed time(ms)={}, bodyLength={}", elapsedTime, messageExtBatch.getBody().length);
-        }
-
-        this.storeStatsService.setPutMessageEntireTimeMax(elapsedTime);
-
-        if (null == result || !result.isOk()) {
-            this.storeStatsService.getPutMessageFailedTimes().incrementAndGet();
-        }
-
-        return result;
     }
 
     @Override
@@ -921,13 +884,13 @@ public class DefaultMessageStore implements MessageStore {
     }
 
     @Override
-    public boolean appendToCommitLog(long startOffset, byte[] data) {
+    public boolean appendToCommitLog(long startOffset, byte[] data, int dataStart, int dataLength) {
         if (this.shutdown) {
             log.warn("message store has shutdown, so appendToPhyQueue is forbidden");
             return false;
         }
 
-        boolean result = this.commitLog.appendData(startOffset, data);
+        boolean result = this.commitLog.appendData(startOffset, data, dataStart, dataLength);
         if (result) {
             this.reputMessageService.wakeup();
         } else {

store/src/main/java/org/apache/rocketmq/store/MappedFile.java

@@ -37,6 +37,7 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.store.CommitLog.PutMessageContext;
 import org.apache.rocketmq.store.config.FlushDiskType;
 import org.apache.rocketmq.store.util.LibC;
 import sun.nio.ch.DirectBuffer;
@@ -188,15 +189,18 @@ public class MappedFile extends ReferenceResource {
         return fileChannel;
     }
 
-    public AppendMessageResult appendMessage(final MessageExtBrokerInner msg, final AppendMessageCallback cb) {
-        return appendMessagesInner(msg, cb);
+    public AppendMessageResult appendMessage(final MessageExtBrokerInner msg, final AppendMessageCallback cb,
+            PutMessageContext putMessageContext) {
+        return appendMessagesInner(msg, cb, putMessageContext);
     }
 
-    public AppendMessageResult appendMessages(final MessageExtBatch messageExtBatch, final AppendMessageCallback cb) {
-        return appendMessagesInner(messageExtBatch, cb);
+    public AppendMessageResult appendMessages(final MessageExtBatch messageExtBatch, final AppendMessageCallback cb,
+            PutMessageContext putMessageContext) {
+        return appendMessagesInner(messageExtBatch, cb, putMessageContext);
     }
 
-    public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb) {
+    public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb,
+            PutMessageContext putMessageContext) {
         assert messageExt != null;
         assert cb != null;
 
@@ -207,9 +211,11 @@ public class MappedFile extends ReferenceResource {
             byteBuffer.position(currentPos);
             AppendMessageResult result;
             if (messageExt instanceof MessageExtBrokerInner) {
-                result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBrokerInner) messageExt);
+                result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos,
+                        (MessageExtBrokerInner) messageExt, putMessageContext);
             } else if (messageExt instanceof MessageExtBatch) {
-                result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBatch) messageExt);
+                result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos,
+                        (MessageExtBatch) messageExt, putMessageContext);
             } else {
                 return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
             }
@@ -301,7 +307,7 @@ public class MappedFile extends ReferenceResource {
         }
         if (this.isAbleToCommit(commitLeastPages)) {
             if (this.hold()) {
-                commit0(commitLeastPages);
+                commit0();
                 this.release();
             } else {
                 log.warn("in commit, hold failed, commit offset = " + this.committedPosition.get());
@@ -317,11 +323,11 @@ public class MappedFile extends ReferenceResource {
         return this.committedPosition.get();
     }
 
-    protected void commit0(final int commitLeastPages) {
+    protected void commit0() {
         int writePos = this.wrotePosition.get();
         int lastCommittedPosition = this.committedPosition.get();
 
-        if (writePos - lastCommittedPosition > commitLeastPages) {
+        if (writePos - lastCommittedPosition > 0) {
             try {
                 ByteBuffer byteBuffer = writeBuffer.slice();
                 byteBuffer.position(lastCommittedPosition);

store/src/main/java/org/apache/rocketmq/store/MessageExtBrokerInner.java

@@ -16,6 +16,8 @@
  */
 package org.apache.rocketmq.store;
 
+import java.nio.ByteBuffer;
+
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.message.MessageExt;
 
@@ -24,6 +26,16 @@ public class MessageExtBrokerInner extends MessageExt {
     private String propertiesString;
     private long tagsCode;
 
+    private ByteBuffer encodedBuff;
+
+    public ByteBuffer getEncodedBuff() {
+        return encodedBuff;
+    }
+
+    public void setEncodedBuff(ByteBuffer encodedBuff) {
+        this.encodedBuff = encodedBuff;
+    }
+
     public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) {
         if (null == tags || tags.length() == 0) { return 0; }
 

store/src/main/java/org/apache/rocketmq/store/MessageStore.java

@@ -245,9 +245,11 @@ public interface MessageStore {
      *
      * @param startOffset starting offset.
      * @param data data to append.
+     * @param dataStart the start index of data array
+     * @param dataLength the length of data array
      * @return true if success; false otherwise.
      */
-    boolean appendToCommitLog(final long startOffset, final byte[] data);
+    boolean appendToCommitLog(final long startOffset, final byte[] data, int dataStart, int dataLength);
 
     /**
      * Execute file deletion manually.

store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java

@@ -48,13 +48,6 @@ public class SelectMappedBufferResult {
         this.byteBuffer.limit(this.size);
     }
 
-//    @Override
-//    protected void finalize() {
-//        if (this.mappedFile != null) {
-//            this.release();
-//        }
-//    }
-
     public synchronized void release() {
         if (this.mappedFile != null) {
             this.mappedFile.release();

store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java

@@ -57,6 +57,7 @@ public class StoreStatsService extends ServiceThread {
     private final LinkedList<CallSnapshot> getTimesMissList = new LinkedList<CallSnapshot>();
     private final LinkedList<CallSnapshot> transferedMsgCountList = new LinkedList<CallSnapshot>();
     private volatile AtomicLong[] putMessageDistributeTime;
+    private volatile AtomicLong[] lastPutMessageDistributeTime;
     private long messageStoreBootTimestamp = System.currentTimeMillis();
     private volatile long putMessageEntireTimeMax = 0;
     private volatile long getMessageEntireTimeMax = 0;
@@ -80,11 +81,11 @@ public class StoreStatsService extends ServiceThread {
             next[i] = new AtomicLong(0);
         }
 
-        AtomicLong[] old = this.putMessageDistributeTime;
+        this.lastPutMessageDistributeTime = this.putMessageDistributeTime;
 
         this.putMessageDistributeTime = next;
 
-        return old;
+        return lastPutMessageDistributeTime;
     }
 
     public long getPutMessageEntireTimeMax() {
@@ -298,7 +299,7 @@ public class StoreStatsService extends ServiceThread {
     }
 
     private String putMessageDistributeTimeToString() {
-        final AtomicLong[] times = this.putMessageDistributeTime;
+        final AtomicLong[] times = this.lastPutMessageDistributeTime;
         if (null == times)
             return null;
 

store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java

@@ -54,13 +54,12 @@ public class MessageStoreConfig {
 
     /**
      * introduced since 4.0.x. Determine whether to use mutex reentrantLock when putting message.<br/>
-     * By default it is set to false indicating using spin lock when putting message.
      */
-    private boolean useReentrantLockWhenPutMessage = false;
+    private boolean useReentrantLockWhenPutMessage = true;
 
-    // Whether schedule flush,default is real-time
+    // Whether schedule flush
     @ImportantField
-    private boolean flushCommitLogTimed = false;
+    private boolean flushCommitLogTimed = true;
     // ConsumeQueue flush interval
     private int flushIntervalConsumeQueue = 1000;
     // Resource reclaim interval

store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java

@@ -37,7 +37,6 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageAccessor;
 import org.apache.rocketmq.common.message.MessageConst;
@@ -413,234 +412,6 @@ public class DLedgerCommitLog extends CommitLog {
         }
     }
 
-    @Override
-    public PutMessageResult putMessage(final MessageExtBrokerInner msg) {
-
-        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();
-        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
-        String topic = msg.getTopic();
-        setMessageInfo(msg,tranType);
-
-        // Back to Results
-        AppendMessageResult appendResult;
-        AppendFuture<AppendEntryResponse> dledgerFuture;
-        EncodeResult encodeResult;
-
-        encodeResult = this.messageSerializer.serialize(msg);
-        if (encodeResult.status != AppendMessageStatus.PUT_OK) {
-            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status));
-        }
-
-        putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
-        long elapsedTimeInLock;
-        long queueOffset;
-        try {
-            beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();
-            queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType);
-            encodeResult.setQueueOffsetKey(queueOffset, false);
-            AppendEntryRequest request = new AppendEntryRequest();
-            request.setGroup(dLedgerConfig.getGroup());
-            request.setRemoteId(dLedgerServer.getMemberState().getSelfId());
-            request.setBody(encodeResult.getData());
-            dledgerFuture = (AppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);
-            if (dledgerFuture.getPos() == -1) {
-                return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR));
-            }
-            long wroteOffset = dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET;
-
-            int msgIdLength = (msg.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;
-            ByteBuffer buffer = ByteBuffer.allocate(msgIdLength);
-
-            String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset);
-            elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
-            appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
-            switch (tranType) {
-                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
-                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
-                    break;
-                case MessageSysFlag.TRANSACTION_NOT_TYPE:
-                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
-                    // The next update ConsumeQueue information
-                    DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1);
-                    break;
-                default:
-                    break;
-            }
-        } catch (Exception e) {
-            log.error("Put message error", e);
-            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR));
-        } finally {
-            beginTimeInDledgerLock = 0;
-            putMessageLock.unlock();
-        }
-
-        if (elapsedTimeInLock > 500) {
-            log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", elapsedTimeInLock, msg.getBody().length, appendResult);
-        }
-
-        PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR;
-        try {
-            AppendEntryResponse appendEntryResponse = dledgerFuture.get(3, TimeUnit.SECONDS);
-            switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) {
-                case SUCCESS:
-                    putMessageStatus = PutMessageStatus.PUT_OK;
-                    break;
-                case INCONSISTENT_LEADER:
-                case NOT_LEADER:
-                case LEADER_NOT_READY:
-                case DISK_FULL:
-                    putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE;
-                    break;
-                case WAIT_QUORUM_ACK_TIMEOUT:
-                    //Do not return flush_slave_timeout to the client, for the ons client will ignore it.
-                    putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY;
-                    break;
-                case LEADER_PENDING_FULL:
-                    putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY;
-                    break;
-            }
-        } catch (Throwable t) {
-            log.error("Failed to get dledger append result", t);
-        }
-
-        PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);
-        if (putMessageStatus == PutMessageStatus.PUT_OK) {
-            // Statistics
-            storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet();
-            storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(appendResult.getWroteBytes());
-        }
-        return putMessageResult;
-    }
-
-    @Override
-    public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) {
-        final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag());
-
-        if (tranType != MessageSysFlag.TRANSACTION_NOT_TYPE) {
-            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
-        }
-        if (messageExtBatch.getDelayTimeLevel() > 0) {
-            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
-        }
-
-        // Set the storage time
-        messageExtBatch.setStoreTimestamp(System.currentTimeMillis());
-
-        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();
-
-        InetSocketAddress bornSocketAddress = (InetSocketAddress) messageExtBatch.getBornHost();
-        if (bornSocketAddress.getAddress() instanceof Inet6Address) {
-            messageExtBatch.setBornHostV6Flag();
-        }
-
-        InetSocketAddress storeSocketAddress = (InetSocketAddress) messageExtBatch.getStoreHost();
-        if (storeSocketAddress.getAddress() instanceof Inet6Address) {
-            messageExtBatch.setStoreHostAddressV6Flag();
-        }
-
-        // Back to Results
-        AppendMessageResult appendResult;
-        BatchAppendFuture<AppendEntryResponse> dledgerFuture;
-        EncodeResult encodeResult;
-
-        encodeResult = this.messageSerializer.serialize(messageExtBatch);
-        if (encodeResult.status != AppendMessageStatus.PUT_OK) {
-            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult
-                    .status));
-        }
-
-        putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
-        msgIdBuilder.setLength(0);
-        long elapsedTimeInLock;
-        long queueOffset;
-        long msgNum = 0;
-        try {
-            beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();
-            queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType);
-            encodeResult.setQueueOffsetKey(queueOffset, true);
-            BatchAppendEntryRequest request = new BatchAppendEntryRequest();
-            request.setGroup(dLedgerConfig.getGroup());
-            request.setRemoteId(dLedgerServer.getMemberState().getSelfId());
-            request.setBatchMsgs(encodeResult.batchData);
-            dledgerFuture = (BatchAppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);
-            if (dledgerFuture.getPos() == -1) {
-                log.warn("HandleAppend return false due to error code {}", dledgerFuture.get().getCode());
-                return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR));
-            }
-            long wroteOffset = 0;
-
-            int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;
-            ByteBuffer buffer = ByteBuffer.allocate(msgIdLength);
-
-            boolean isFirstOffset = true;
-            long firstWroteOffset = 0;
-            for (long pos : dledgerFuture.getPositions()) {
-                wroteOffset = pos + DLedgerEntry.BODY_OFFSET;
-                if (isFirstOffset) {
-                    firstWroteOffset = wroteOffset;
-                    isFirstOffset = false;
-                }
-                String msgId = MessageDecoder.createMessageId(buffer, messageExtBatch.getStoreHostBytes(), wroteOffset);
-                if (msgIdBuilder.length() > 0) {
-                    msgIdBuilder.append(',').append(msgId);
-                } else {
-                    msgIdBuilder.append(msgId);
-                }
-                msgNum++;
-            }
-
-            elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
-            appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen,
-                    msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
-            DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + msgNum);
-        } catch (Exception e) {
-            log.error("Put message error", e);
-            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus
-                    .UNKNOWN_ERROR));
-        } finally {
-            beginTimeInDledgerLock = 0;
-            putMessageLock.unlock();
-        }
-
-        if (elapsedTimeInLock > 500) {
-            log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}",
-                    elapsedTimeInLock, messageExtBatch.getBody().length, appendResult);
-        }
-
-        PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR;
-        try {
-            AppendEntryResponse appendEntryResponse = dledgerFuture.get(3, TimeUnit.SECONDS);
-            switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) {
-                case SUCCESS:
-                    putMessageStatus = PutMessageStatus.PUT_OK;
-                    break;
-                case INCONSISTENT_LEADER:
-                case NOT_LEADER:
-                case LEADER_NOT_READY:
-                case DISK_FULL:
-                    putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE;
-                    break;
-                case WAIT_QUORUM_ACK_TIMEOUT:
-                    //Do not return flush_slave_timeout to the client, for the ons client will ignore it.
-                    putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY;
-                    break;
-                case LEADER_PENDING_FULL:
-                    putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY;
-                    break;
-            }
-        } catch (Throwable t) {
-            log.error("Failed to get dledger append result", t);
-        }
-
-        PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);
-        if (putMessageStatus == PutMessageStatus.PUT_OK) {
-            // Statistics
-            storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(msgNum);
-            storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(encodeResult.totalMsgLen);
-        }
-        return putMessageResult;
-    }
-
     @Override
     public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {
 
@@ -779,7 +550,7 @@ public class DLedgerCommitLog extends CommitLog {
         msgIdBuilder.setLength(0);
         long elapsedTimeInLock;
         long queueOffset;
-        long msgNum = 0;
+        int msgNum = 0;
         try {
             beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();
             queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType);
@@ -788,11 +559,13 @@ public class DLedgerCommitLog extends CommitLog {
             request.setGroup(dLedgerConfig.getGroup());
             request.setRemoteId(dLedgerServer.getMemberState().getSelfId());
             request.setBatchMsgs(encodeResult.batchData);
-            dledgerFuture = (BatchAppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);
-            if (dledgerFuture.getPos() == -1) {
-                log.warn("HandleAppend return false due to error code {}", dledgerFuture.get().getCode());
+            AppendFuture<AppendEntryResponse> appendFuture = (AppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);
+            if (appendFuture.getPos() == -1) {
+                log.warn("HandleAppend return false due to error code {}", appendFuture.get().getCode());
                 return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
             }
+            dledgerFuture = (BatchAppendFuture<AppendEntryResponse>) appendFuture;
+
             long wroteOffset = 0;
 
             int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;
@@ -818,6 +591,7 @@ public class DLedgerCommitLog extends CommitLog {
             elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
             appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen,
                     msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
+            appendResult.setMsgNum(msgNum);
             DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + msgNum);
         } catch (Exception e) {
             log.error("Put message error", e);
@@ -899,7 +673,7 @@ public class DLedgerCommitLog extends CommitLog {
     }
 
     @Override
-    public boolean appendData(long startOffset, byte[] data) {
+    public boolean appendData(long startOffset, byte[] data, int dataStart, int dataLength) {
         //the old ha service will invoke method, here to prevent it
         return false;
     }

store/src/main/java/org/apache/rocketmq/store/ha/HAService.java

@@ -25,7 +25,6 @@ import java.nio.channels.SelectionKey;
 import java.nio.channels.Selector;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
-import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
@@ -39,6 +38,7 @@ import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingUtil;
 import org.apache.rocketmq.store.CommitLog;
 import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.PutMessageSpinLock;
 import org.apache.rocketmq.store.PutMessageStatus;
 
 public class HAService {
@@ -254,12 +254,16 @@ public class HAService {
     class GroupTransferService extends ServiceThread {
 
         private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject();
-        private volatile List<CommitLog.GroupCommitRequest> requestsWrite = new ArrayList<>();
-        private volatile List<CommitLog.GroupCommitRequest> requestsRead = new ArrayList<>();
+        private final PutMessageSpinLock lock = new PutMessageSpinLock();
+        private volatile LinkedList<CommitLog.GroupCommitRequest> requestsWrite = new LinkedList<>();
+        private volatile LinkedList<CommitLog.GroupCommitRequest> requestsRead = new LinkedList<>();
 
-        public synchronized void putRequest(final CommitLog.GroupCommitRequest request) {
-            synchronized (this.requestsWrite) {
+        public void putRequest(final CommitLog.GroupCommitRequest request) {
+            lock.lock();
+            try {
                 this.requestsWrite.add(request);
+            } finally {
+                lock.unlock();
             }
             this.wakeup();
         }
@@ -269,32 +273,35 @@ public class HAService {
         }
 
         private void swapRequests() {
-            List<CommitLog.GroupCommitRequest> tmp = this.requestsWrite;
-            this.requestsWrite = this.requestsRead;
-            this.requestsRead = tmp;
+            lock.lock();
+            try {
+                LinkedList<CommitLog.GroupCommitRequest> tmp = this.requestsWrite;
+                this.requestsWrite = this.requestsRead;
+                this.requestsRead = tmp;
+            } finally {
+                lock.unlock();
+            }
         }
 
         private void doWaitTransfer() {
-            synchronized (this.requestsRead) {
-                if (!this.requestsRead.isEmpty()) {
-                    for (CommitLog.GroupCommitRequest req : this.requestsRead) {
-                        boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
-                        long waitUntilWhen = HAService.this.defaultMessageStore.getSystemClock().now()
+            if (!this.requestsRead.isEmpty()) {
+                for (CommitLog.GroupCommitRequest req : this.requestsRead) {
+                    boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
+                    long waitUntilWhen = HAService.this.defaultMessageStore.getSystemClock().now()
                             + HAService.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout();
-                        while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntilWhen) {
-                            this.notifyTransferObject.waitForRunning(1000);
-                            transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
-                        }
-
-                        if (!transferOK) {
-                            log.warn("transfer messsage to slave timeout, " + req.getNextOffset());
-                        }
-
-                        req.wakeupCustomer(transferOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
+                    while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntilWhen) {
+                        this.notifyTransferObject.waitForRunning(1000);
+                        transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
                     }
 
-                    this.requestsRead.clear();
+                    if (!transferOK) {
+                        log.warn("transfer messsage to slave timeout, " + req.getNextOffset());
+                    }
+
+                    req.wakeupCustomer(transferOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                 }
+
+                this.requestsRead = new LinkedList<>();
             }
         }
 
@@ -433,7 +440,6 @@ public class HAService {
 
         private boolean dispatchReadRequest() {
             final int msgHeaderSize = 8 + 4; // phyoffset + size
-            int readSocketPos = this.byteBufferRead.position();
 
             while (true) {
                 int diff = this.byteBufferRead.position() - this.dispatchPosition;
@@ -452,13 +458,12 @@ public class HAService {
                     }
 
                     if (diff >= (msgHeaderSize + bodySize)) {
-                        byte[] bodyData = new byte[bodySize];
-                        this.byteBufferRead.position(this.dispatchPosition + msgHeaderSize);
-                        this.byteBufferRead.get(bodyData);
+                        byte[] bodyData = byteBufferRead.array();
+                        int dataStart = this.dispatchPosition + msgHeaderSize;
 
-                        HAService.this.defaultMessageStore.appendToCommitLog(masterPhyOffset, bodyData);
+                        HAService.this.defaultMessageStore.appendToCommitLog(
+                                masterPhyOffset, bodyData, dataStart, bodySize);
 
-                        this.byteBufferRead.position(readSocketPos);
                         this.dispatchPosition += msgHeaderSize + bodySize;
 
                         if (!reportSlaveMaxOffsetPlus()) {

store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java

@@ -20,40 +20,43 @@ import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 public class WaitNotifyObject {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
 
-    protected final HashMap<Long/* thread id */, Boolean/* notified */> waitingThreadTable =
-        new HashMap<Long, Boolean>(16);
+    protected final ConcurrentHashMap<Long/* thread id */, AtomicBoolean/* notified */> waitingThreadTable =
+        new ConcurrentHashMap<Long, AtomicBoolean>(16);
 
-    protected volatile boolean hasNotified = false;
+    protected AtomicBoolean hasNotified = new AtomicBoolean(false);
 
     public void wakeup() {
-        synchronized (this) {
-            if (!this.hasNotified) {
-                this.hasNotified = true;
+        boolean needNotify = hasNotified.compareAndSet(false, true);
+        if (needNotify) {
+            synchronized (this) {
                 this.notify();
             }
         }
     }
 
     protected void waitForRunning(long interval) {
+        if (this.hasNotified.compareAndSet(true, false)) {
+            this.onWaitEnd();
+            return;
+        }
         synchronized (this) {
-            if (this.hasNotified) {
-                this.hasNotified = false;
-                this.onWaitEnd();
-                return;
-            }
-
             try {
+                if (this.hasNotified.compareAndSet(true, false)) {
+                    this.onWaitEnd();
+                    return;
+                }
                 this.wait(interval);
             } catch (InterruptedException e) {
                 log.error("Interrupted", e);
             } finally {
-                this.hasNotified = false;
+                this.hasNotified.set(false);
                 this.onWaitEnd();
             }
         }
@@ -63,15 +66,14 @@ public class WaitNotifyObject {
     }
 
     public void wakeupAll() {
-        synchronized (this) {
-            boolean needNotify = false;
-
-            for (Map.Entry<Long,Boolean> entry : this.waitingThreadTable.entrySet()) {
-                needNotify = needNotify || !entry.getValue();
-                entry.setValue(true);
+        boolean needNotify = false;
+        for (Map.Entry<Long,AtomicBoolean> entry : this.waitingThreadTable.entrySet()) {
+            if (entry.getValue().compareAndSet(false, true)) {
+                needNotify = true;
             }
-
-            if (needNotify) {
+        }
+        if (needNotify) {
+            synchronized (this) {
                 this.notifyAll();
             }
         }
@@ -79,20 +81,22 @@ public class WaitNotifyObject {
 
     public void allWaitForRunning(long interval) {
         long currentThreadId = Thread.currentThread().getId();
+        AtomicBoolean notified = this.waitingThreadTable.computeIfAbsent(currentThreadId, k -> new AtomicBoolean(false));
+        if (notified.compareAndSet(true, false)) {
+            this.onWaitEnd();
+            return;
+        }
         synchronized (this) {
-            Boolean notified = this.waitingThreadTable.get(currentThreadId);
-            if (notified != null && notified) {
-                this.waitingThreadTable.put(currentThreadId, false);
-                this.onWaitEnd();
-                return;
-            }
-
             try {
+                if (notified.compareAndSet(true, false)) {
+                    this.onWaitEnd();
+                    return;
+                }
                 this.wait(interval);
             } catch (InterruptedException e) {
                 log.error("Interrupted", e);
             } finally {
-                this.waitingThreadTable.put(currentThreadId, false);
+                notified.set(false);
                 this.onWaitEnd();
             }
         }

store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java

@@ -30,6 +30,8 @@ import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.store.CommitLog.MessageExtEncoder;
+import org.apache.rocketmq.store.CommitLog.PutMessageContext;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.After;
 import org.junit.Before;
@@ -42,7 +44,7 @@ public class AppendCallbackTest {
 
     AppendMessageCallback callback;
 
-    CommitLog.MessageExtBatchEncoder batchEncoder = new CommitLog.MessageExtBatchEncoder(10 * 1024 * 1024);
+    MessageExtEncoder batchEncoder = new MessageExtEncoder(10 * 1024 * 1024);
 
     @Before
     public void init() throws Exception {
@@ -84,10 +86,12 @@ public class AppendCallbackTest {
         messageExtBatch.setStoreHost(new InetSocketAddress("127.0.0.1", 124));
         messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
 
-        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
+        PutMessageContext putMessageContext = new PutMessageContext(topic + "-" + queue);
+        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));
         ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
         //encounter end of file when append half of the data
-        AppendMessageResult result = callback.doAppend(0, buff, 1000, messageExtBatch);
+        AppendMessageResult result =
+                callback.doAppend(0, buff, 1000, messageExtBatch, putMessageContext);
         assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus());
         assertEquals(0, result.getWroteOffset());
         assertEquals(0, result.getLogicsOffset());
@@ -121,10 +125,12 @@ public class AppendCallbackTest {
         messageExtBatch.setStoreHost(new InetSocketAddress("::1", 124));
         messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
 
-        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
+        PutMessageContext putMessageContext = new PutMessageContext(topic + "-" + queue);
+        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));
         ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
         //encounter end of file when append half of the data
-        AppendMessageResult result = callback.doAppend(0, buff, 1000, messageExtBatch);
+        AppendMessageResult result =
+                callback.doAppend(0, buff, 1000, messageExtBatch, putMessageContext);
         assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus());
         assertEquals(0, result.getWroteOffset());
         assertEquals(0, result.getLogicsOffset());
@@ -154,9 +160,11 @@ public class AppendCallbackTest {
         messageExtBatch.setStoreHost(new InetSocketAddress("127.0.0.1", 124));
         messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
 
-        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
+        PutMessageContext putMessageContext = new PutMessageContext(topic + "-" + queue);
+        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));
         ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
-        AppendMessageResult allresult = callback.doAppend(0, buff, 1024 * 10, messageExtBatch);
+        AppendMessageResult allresult =
+                callback.doAppend(0, buff, 1024 * 10, messageExtBatch, putMessageContext);
 
         assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());
         assertEquals(0, allresult.getWroteOffset());
@@ -214,9 +222,11 @@ public class AppendCallbackTest {
         messageExtBatch.setStoreHost(new InetSocketAddress("::1", 124));
         messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
 
-        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
+        PutMessageContext putMessageContext = new PutMessageContext(topic + "-" + queue);
+        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));
         ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
-        AppendMessageResult allresult = callback.doAppend(0, buff, 1024 * 10, messageExtBatch);
+        AppendMessageResult allresult =
+                callback.doAppend(0, buff, 1024 * 10, messageExtBatch, putMessageContext);
 
         assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());
         assertEquals(0, allresult.getWroteOffset());

store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java

@@ -23,7 +23,6 @@ import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
-import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.store.config.FlushDiskType;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
@@ -39,6 +38,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.rocketmq.common.message.MessageDecoder.messageProperties2String;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -230,22 +230,6 @@ public class BatchPutMessageTest {
         return msgLen;
     }
 
-    public String messageProperties2String(Map<String, String> properties) {
-        StringBuilder sb = new StringBuilder();
-        if (properties != null) {
-            for (final Map.Entry<String, String> entry : properties.entrySet()) {
-                final String name = entry.getKey();
-                final String value = entry.getValue();
-
-                sb.append(name);
-                sb.append(NAME_VALUE_SEPARATOR);
-                sb.append(value);
-                sb.append(PROPERTY_SEPARATOR);
-            }
-        }
-        return sb.toString();
-    }
-
     private class MyMessageArrivingListener implements MessageArrivingListener {
         @Override
         public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,

test/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

tools/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.1-SNAPSHOT</version>
+        <version>4.9.2-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java

@@ -44,10 +44,10 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
     public Options buildCommandlineOptions(Options options) {
         OptionGroup optionGroup = new OptionGroup();
 
-        Option opt = new Option("b", "brokerAddr", true, "delete acl config account to which broker");
+        Option opt = new Option("b", "brokerAddr", true, "delete acl config account from which broker");
         optionGroup.addOption(opt);
 
-        opt = new Option("c", "clusterName", true, "delete cl config account to which cluster");
+        opt = new Option("c", "clusterName", true, "delete acl config account from which cluster");
         optionGroup.addOption(opt);
 
         optionGroup.setRequired(true);
@@ -76,7 +76,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
                 defaultMQAdminExt.start();
                 defaultMQAdminExt.deletePlainAccessConfig(addr, accessKey);
 
-                System.out.printf("delete plain access config account to %s success.%n", addr);
+                System.out.printf("delete plain access config account from %s success.%n", addr);
                 System.out.printf("account's accesskey is:%s", accessKey);
                 return;
 
@@ -89,7 +89,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
                     CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
                 for (String addr : masterSet) {
                     defaultMQAdminExt.deletePlainAccessConfig(addr, accessKey);
-                    System.out.printf("delete plain access config account to %s success.%n", addr);
+                    System.out.printf("delete plain access config account from %s success.%n", addr);
                 }
 
                 System.out.printf("account's accesskey is:%s", accessKey);

tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java

@@ -50,7 +50,7 @@ public class UpdateAccessConfigSubCommand implements SubCommand {
         Option opt = new Option("b", "brokerAddr", true, "update acl config file to which broker");
         optionGroup.addOption(opt);
 
-        opt = new Option("c", "clusterName", true, "update cl config file to which cluster");
+        opt = new Option("c", "clusterName", true, "update acl config file to which cluster");
         optionGroup.addOption(opt);
 
         optionGroup.setRequired(true);
@@ -164,9 +164,9 @@ public class UpdateAccessConfigSubCommand implements SubCommand {
                 String clusterName = commandLine.getOptionValue('c').trim();
 
                 defaultMQAdminExt.start();
-                Set<String> masterSet =
-                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
-                for (String addr : masterSet) {
+                Set<String> brokerAddrSet =
+                    CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String addr : brokerAddrSet) {
                     defaultMQAdminExt.createAndUpdatePlainAccessConfig(addr, accessConfig);
                     System.out.printf("create or update plain access config to %s success.%n", addr);
                 }

tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java

@@ -68,14 +68,18 @@ public class SendMessageCommand implements SubCommand {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
-    private DefaultMQProducer createProducer(RPCHook rpcHook) {
+    private DefaultMQProducer createProducer(RPCHook rpcHook, boolean msgTraceEnable) {
         if (this.producer != null) {
             return producer;
         } else {
-            producer = new DefaultMQProducer(rpcHook);
+            producer = new DefaultMQProducer(null, rpcHook, msgTraceEnable, null);
             producer.setProducerGroup(Long.toString(System.currentTimeMillis()));
             return producer;
         }
@@ -112,8 +116,11 @@ public class SendMessageCommand implements SubCommand {
         } catch (Exception e) {
             throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e);
         }
-
-        DefaultMQProducer producer = this.createProducer(rpcHook);
+        boolean msgTraceEnable = false;
+        if (commandLine.hasOption('m')) {
+            msgTraceEnable = Boolean.parseBoolean(commandLine.getOptionValue('m').trim());
+        }
+        DefaultMQProducer producer = this.createProducer(rpcHook, msgTraceEnable);
         SendResult result;
         try {
             producer.start();
@@ -153,4 +160,4 @@ public class SendMessageCommand implements SubCommand {
             );
         }
     }
-}
+}