Merge remote-tracking branch 'upstream/snode' into snode

.gitignore

@@ -10,4 +10,6 @@ devenv
 *.versionsBackup
 !NOTICE-BIN
 !LICENSE-BIN
-.DS_Store
+.DS_Store
+localbin
+nohup.out

NOTICE

@@ -1,5 +1,5 @@
 Apache RocketMQ
-Copyright 2016-2017 The Apache Software Foundation
+Copyright 2016-2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -21,6 +21,8 @@ It offers a variety of features:
 * Various message filter mechanics such as SQL and Tag
 * Docker images for isolated testing and cloud isolated clusters
 * Feature-rich administrative dashboard for configuration, metrics and monitoring
+* Access control list
+* Message trace
 
 
 ----------
@@ -37,8 +39,7 @@ It offers a variety of features:
 ----------
 
 ## Apache RocketMQ Community
-* [RocketMQ Community Incubator Projects](https://github.com/apache/rocketmq-externals)
-
+* [RocketMQ Community Projects](https://github.com/apache/rocketmq-externals)
 ----------
 
 ## Contributing
@@ -47,3 +48,5 @@ We always welcome new contributions, whether for trivial cleanups, big new featu
 ----------
 ## License
 [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) Apache Software Foundation
+
+

acl/pom.xml

@@ -9,14 +9,13 @@
 	OF ANY KIND, either express or implied. See the License for the specific 
 	language governing permissions and limitations under the License. -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
-
-    <modelVersion>4.0.0</modelVersion>
-    <packaging>jar</packaging>
     <artifactId>rocketmq-acl</artifactId>
     <name>rocketmq-acl ${project.version}</name>
 

acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java

@@ -59,7 +59,6 @@ public class PlainAccessValidatorTest {
         messageRequestHeader.setTopic("topicA");
         RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
         aclClient.doBeforeRequest("", remotingCommand);
-
         ByteBuffer buf = CodecHelper.encode(remotingCommand);
         buf.getInt();
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
@@ -77,7 +76,6 @@ public class PlainAccessValidatorTest {
         messageRequestHeader.setTopic("topicB");
         RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
         aclClient.doBeforeRequest("", remotingCommand);
-
         ByteBuffer buf = CodecHelper.encode(remotingCommand);
         buf.getInt();
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
@@ -93,7 +91,6 @@ public class PlainAccessValidatorTest {
         messageRequestHeader.setTopic("topicB");
         RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
         aclClient.doBeforeRequest("", remotingCommand);
-
         ByteBuffer buf = CodecHelper.encodeHeader(remotingCommand);
         buf.getInt();
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
@@ -108,7 +105,6 @@ public class PlainAccessValidatorTest {
         messageRequestHeader.setTopic("topicC");
         RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
         aclClient.doBeforeRequest("", remotingCommand);
-
         ByteBuffer buf = CodecHelper.encodeHeader(remotingCommand);
         buf.getInt();
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
@@ -201,6 +197,7 @@ public class PlainAccessValidatorTest {
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
         buf.position(0);
         PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(CodecHelper.decode(buf), "192.168.0.1:9876");
+
         plainAccessValidator.validate(accessResource);
     }
 
@@ -242,7 +239,6 @@ public class PlainAccessValidatorTest {
         messageRequestHeader.setTopic("topicB");
         RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
         aclClientRPCHook.doBeforeRequest("", remotingCommand);
-
         ByteBuffer buf = CodecHelper.encodeHeader(remotingCommand);
         buf.getInt();
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
@@ -261,7 +257,6 @@ public class PlainAccessValidatorTest {
         messageRequestHeader.setTopic("topicB");
         RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
         aclClientRPCHook.doBeforeRequest("", remotingCommand);
-
         ByteBuffer buf = CodecHelper.encodeHeader(remotingCommand);
         buf.getInt();
         buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);

acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionLoaderTest.java

@@ -116,7 +116,7 @@ public class PlainPermissionLoaderTest {
         Assert.assertEquals(resourcePermMap.size(), 3);
 
         Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupA")).byteValue(), Permission.DENY);
-        Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB")).byteValue(), Permission.PUB|Permission.SUB);
+        Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB")).byteValue(), Permission.PUB | Permission.SUB);
         Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupC")).byteValue(), Permission.PUB);
 
         List<String> topics = new ArrayList<String>();
@@ -129,7 +129,7 @@ public class PlainPermissionLoaderTest {
         Assert.assertEquals(resourcePermMap.size(), 6);
 
         Assert.assertEquals(resourcePermMap.get("topicA").byteValue(), Permission.DENY);
-        Assert.assertEquals(resourcePermMap.get("topicB").byteValue(), Permission.PUB|Permission.SUB);
+        Assert.assertEquals(resourcePermMap.get("topicB").byteValue(), Permission.PUB | Permission.SUB);
         Assert.assertEquals(resourcePermMap.get("topicC").byteValue(), Permission.PUB);
     }
 
@@ -156,6 +156,7 @@ public class PlainPermissionLoaderTest {
         plainPermissionLoader.checkPerm(plainAccessResource, ANYPlainAccessResource);
 
     }
+
     @Test(expected = AclException.class)
     public void checkErrorPerm() {
 
@@ -163,6 +164,7 @@ public class PlainPermissionLoaderTest {
         plainAccessResource.addResourceAndPerm("topicF", Permission.SUB);
         plainPermissionLoader.checkPerm(plainAccessResource, SUBPlainAccessResource);
     }
+
     @Test(expected = AclException.class)
     public void accountNullTest() {
         plainAccessConfig.setAccessKey(null);
@@ -212,12 +214,11 @@ public class PlainPermissionLoaderTest {
         Assert.assertTrue(plainPermissionLoader.isWatchStart());
     }
 
-
     @Test
-    public void testWatch() throws IOException, IllegalAccessException ,InterruptedException{
+    public void testWatch() throws IOException, IllegalAccessException, InterruptedException {
         System.setProperty("rocketmq.home.dir", "src/test/resources");
         System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl-test.yml");
-        String fileName =System.getProperty("rocketmq.home.dir", "src/test/resources")+System.getProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml");
+        String fileName = System.getProperty("rocketmq.home.dir", "src/test/resources") + System.getProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml");
         File transport = new File(fileName);
         transport.delete();
         transport.createNewFile();

broker/pom.xml

@@ -1,25 +1,19 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	You under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -52,6 +46,10 @@
             <groupId>${project.groupId}</groupId>
             <artifactId>rocketmq-filter</artifactId>
         </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>rocketmq-acl</artifactId>
+        </dependency>
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>

broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java

@@ -27,10 +27,12 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.acl.AccessValidator;
 import org.apache.rocketmq.broker.client.ClientHousekeepingService;
 import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
 import org.apache.rocketmq.broker.client.ConsumerManager;
@@ -38,6 +40,7 @@ import org.apache.rocketmq.broker.client.DefaultConsumerIdsChangeListener;
 import org.apache.rocketmq.broker.client.ProducerManager;
 import org.apache.rocketmq.broker.client.net.Broker2Client;
 import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;
+import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler;
 import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;
 import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
 import org.apache.rocketmq.broker.filtersrv.FilterServerManager;
@@ -99,6 +102,7 @@ import org.apache.rocketmq.store.MessageArrivingListener;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
 import org.apache.rocketmq.store.stats.BrokerStats;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
@@ -159,6 +163,7 @@ public class BrokerController {
     private TransactionalMessageCheckService transactionalMessageCheckService;
     private TransactionalMessageService transactionalMessageService;
     private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener;
+    private Future<?> slaveSyncFuture;
 
     public BrokerController(
         final BrokerConfig brokerConfig,
@@ -234,6 +239,10 @@ public class BrokerController {
                 this.messageStore =
                     new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener,
                         this.brokerConfig);
+                if (messageStoreConfig.isEnableDLegerCommitLog()) {
+                    DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore);
+                    ((DLedgerCommitLog) ((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler);
+                }
                 this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
                 //load plugin
                 MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
@@ -249,10 +258,8 @@ public class BrokerController {
 
         if (result) {
             this.remotingServer = RemotingServerFactory.getInstance().createRemotingServer().init(this.nettyServerConfig, this.clientHousekeepingService);
-//            this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
             ServerConfig fastConfig = (ServerConfig) this.nettyServerConfig.clone();
             fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2);
-//            this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
             this.fastRemotingServer = RemotingServerFactory.getInstance().createRemotingServer().init(fastConfig, this.clientHousekeepingService);
 
             this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(
@@ -399,37 +406,26 @@ public class BrokerController {
                 }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
             }
 
-            if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
-                if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
-                    this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
-                    this.updateMasterHAServerAddrPeriodically = false;
+            if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+                if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
+                    if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
+                        this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
+                        this.updateMasterHAServerAddrPeriodically = false;
+                    } else {
+                        this.updateMasterHAServerAddrPeriodically = true;
+                    }
                 } else {
-                    this.updateMasterHAServerAddrPeriodically = true;
+                    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+                        @Override
+                        public void run() {
+                            try {
+                                BrokerController.this.printMasterAndSlaveDiff();
+                            } catch (Throwable e) {
+                                log.error("schedule printMasterAndSlaveDiff error.", e);
+                            }
+                        }
+                    }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
                 }
-
-                this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
-
-                    @Override
-                    public void run() {
-                        try {
-                            BrokerController.this.slaveSynchronize.syncAll();
-                        } catch (Throwable e) {
-                            log.error("ScheduledTask syncAll slave exception", e);
-                        }
-                    }
-                }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
-            } else {
-                this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
-
-                    @Override
-                    public void run() {
-                        try {
-                            BrokerController.this.printMasterAndSlaveDiff();
-                        } catch (Throwable e) {
-                            log.error("schedule printMasterAndSlaveDiff error.", e);
-                        }
-                    }
-                }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
             }
 
             if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) {
@@ -473,6 +469,8 @@ public class BrokerController {
                 }
             }
             initialTransaction();
+            initialAcl();
+//            initialRpcHooks();
         }
         return result;
     }
@@ -492,6 +490,57 @@ public class BrokerController {
         this.transactionalMessageCheckService = new TransactionalMessageCheckService(this);
     }
 
+    private void initialAcl() {
+        if (!this.brokerConfig.isAclEnable()) {
+            log.info("The broker dose not enable acl");
+            return;
+        }
+
+        List<AccessValidator> accessValidators = ServiceProvider.loadServiceList(ServiceProvider.ACL_VALIDATOR_ID, AccessValidator.class);
+        if (accessValidators == null || accessValidators.isEmpty()) {
+            log.info("The broker dose not load the AccessValidator");
+            return;
+        }
+
+//        for (AccessValidator accessValidator : accessValidators) {
+//            final AccessValidator validator = accessValidator;
+//            this.registerServerRPCHook(new Interceptor() {
+//
+//                @Override public String interceptorName() {
+//                    return "aclInterceptor";
+//                }
+//
+//                @Override public void beforeRequest(RequestContext requestContext) {
+//
+//                    validator.validate(validator.parse(requestContext.getRequest(), requestContext.getRemotingChannel().remoteAddress().toString()));
+//
+//                }
+//
+//                @Override public void afterRequest(ResponseContext responseContext) {
+//
+//                }
+//
+//                @Override public void onException(ExceptionContext exceptionContext) {
+//
+//                }
+//
+//            });
+//        }
+    }
+
+    private void initialRpcHooks() {
+
+        List<RPCHook> rpcHooks = ServiceProvider.loadServiceList(ServiceProvider.RPC_HOOK_ID, RPCHook.class);
+        if (rpcHooks == null || rpcHooks.isEmpty()) {
+            return;
+        }
+        for (RPCHook rpcHook : rpcHooks) {
+            this.remotingServer.registerServerRPCHook(rpcHook);
+        }
+    }
+
+//    registerInterceoptorGroup()
+
     public void registerProcessor() {
         /**
          * SendMessageProcessor
@@ -822,6 +871,11 @@ public class BrokerController {
             this.filterServerManager.start();
         }
 
+        if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+            startProcessorByHa(messageStoreConfig.getBrokerRole());
+            handleSlaveSynchronize(messageStoreConfig.getBrokerRole());
+        }
+
         this.registerBrokerAll(true, false, true);
 
         this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@@ -844,12 +898,6 @@ public class BrokerController {
             this.brokerFastFailure.start();
         }
 
-        if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
-            if (this.transactionalMessageCheckService != null) {
-                log.info("Start transaction service!");
-                this.transactionalMessageCheckService.start();
-            }
-        }
     }
 
     public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) {
@@ -1000,9 +1048,10 @@ public class BrokerController {
         log.info("register ConsumeMessageHook Hook, {}", hook.hookName());
     }
 
-    public void registerServerRPCHook(RPCHook rpcHook) {
-//        getRemotingServer().registerRPCHook(rpcHook);
-    }
+//    public void registerServerRPCHook(RPCHook rpcHook) {
+////        getRemotingServer().registerRPCHook(rpcHook);
+////        this.fastRemotingServer.registerRPCHook(rpcHook);
+////    }
 
     public RemotingServer getRemotingServer() {
         return remotingServer;
@@ -1064,5 +1113,113 @@ public class BrokerController {
 
     public BlockingQueue<Runnable> getEndTransactionThreadPoolQueue() {
         return endTransactionThreadPoolQueue;
+
     }
+
+    private void handleSlaveSynchronize(BrokerRole role) {
+        if (role == BrokerRole.SLAVE) {
+            if (null != slaveSyncFuture) {
+                slaveSyncFuture.cancel(false);
+            }
+            this.slaveSynchronize.setMasterAddr(null);
+            slaveSyncFuture = this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        BrokerController.this.slaveSynchronize.syncAll();
+                    } catch (Throwable e) {
+                        log.error("ScheduledTask SlaveSynchronize syncAll error.", e);
+                    }
+                }
+            }, 1000 * 3, 1000 * 10, TimeUnit.MILLISECONDS);
+        } else {
+            //handle the slave synchronise
+            if (null != slaveSyncFuture) {
+                slaveSyncFuture.cancel(false);
+            }
+            this.slaveSynchronize.setMasterAddr(null);
+        }
+    }
+
+    public void changeToSlave(int brokerId) {
+        log.info("Begin to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId);
+
+        //change the role
+        brokerConfig.setBrokerId(brokerId == 0 ? 1 : brokerId); //TO DO check
+        messageStoreConfig.setBrokerRole(BrokerRole.SLAVE);
+
+        //handle the scheduled service
+        try {
+            this.messageStore.handleScheduleMessageService(BrokerRole.SLAVE);
+        } catch (Throwable t) {
+            log.error("[MONITOR] handleScheduleMessageService failed when changing to slave", t);
+        }
+
+        //handle the transactional service
+        try {
+            this.shutdownProcessorByHa();
+        } catch (Throwable t) {
+            log.error("[MONITOR] shutdownProcessorByHa failed when changing to slave", t);
+        }
+
+        //handle the slave synchronise
+        handleSlaveSynchronize(BrokerRole.SLAVE);
+
+        try {
+            this.registerBrokerAll(true, true, brokerConfig.isForceRegister());
+        } catch (Throwable ignored) {
+
+        }
+        log.info("Finish to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId);
+    }
+
+    public void changeToMaster(BrokerRole role) {
+        if (role == BrokerRole.SLAVE) {
+            return;
+        }
+        log.info("Begin to change to master brokerName={}", brokerConfig.getBrokerName());
+
+        //handle the slave synchronise
+        handleSlaveSynchronize(role);
+
+        //handle the scheduled service
+        try {
+            this.messageStore.handleScheduleMessageService(role);
+        } catch (Throwable t) {
+            log.error("[MONITOR] handleScheduleMessageService failed when changing to master", t);
+        }
+
+        //handle the transactional service
+        try {
+            this.startProcessorByHa(BrokerRole.SYNC_MASTER);
+        } catch (Throwable t) {
+            log.error("[MONITOR] startProcessorByHa failed when changing to master", t);
+        }
+
+        //if the operations above are totally successful, we change to master
+        brokerConfig.setBrokerId(0); //TO DO check
+        messageStoreConfig.setBrokerRole(role);
+
+        try {
+            this.registerBrokerAll(true, true, brokerConfig.isForceRegister());
+        } catch (Throwable ignored) {
+
+        }
+        log.info("Finish to change to master brokerName={}", brokerConfig.getBrokerName());
+    }
+
+    private void startProcessorByHa(BrokerRole role) {
+        if (BrokerRole.SLAVE != role) {
+            if (this.transactionalMessageCheckService != null) {
+                this.transactionalMessageCheckService.start();
+            }
+        }
+    }
+
+    private void shutdownProcessorByHa() {
+        if (this.transactionalMessageCheckService != null) {
+            this.transactionalMessageCheckService.shutdown(true);
+        }
+    }
+
 }

broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.dledger;
+
+import io.openmessaging.storage.dledger.DLedgerLeaderElector;
+import io.openmessaging.storage.dledger.DLedgerServer;
+import io.openmessaging.storage.dledger.MemberState;
+import io.openmessaging.storage.dledger.utils.DLedgerUtils;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
+
+public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler {
+
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLegerRoleChangeHandler_"));
+    private BrokerController brokerController;
+    private DefaultMessageStore messageStore;
+    private DLedgerCommitLog dLedgerCommitLog;
+    private DLedgerServer dLegerServer;
+    public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) {
+        this.brokerController = brokerController;
+        this.messageStore = messageStore;
+        this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
+        this.dLegerServer = dLedgerCommitLog.getdLedgerServer();
+    }
+
+    @Override public void handle(long term, MemberState.Role role) {
+        Runnable runnable = new Runnable() {
+            @Override public void run() {
+                long start = System.currentTimeMillis();
+                try {
+                    boolean succ = true;
+                    log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole());
+                    switch (role) {
+                        case CANDIDATE:
+                            if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {
+                                brokerController.changeToSlave(dLedgerCommitLog.getId());
+                            }
+                            break;
+                        case FOLLOWER:
+                            brokerController.changeToSlave(dLedgerCommitLog.getId());
+                            break;
+                        case LEADER:
+                            while (true) {
+                                if (!dLegerServer.getMemberState().isLeader()) {
+                                    succ = false;
+                                    break;
+                                }
+                                if (dLegerServer.getdLedgerStore().getLedgerEndIndex() == -1) {
+                                    break;
+                                }
+                                if (dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex()
+                                    && messageStore.dispatchBehindBytes() == 0) {
+                                    break;
+                                }
+                                Thread.sleep(100);
+                            }
+                            if (succ) {
+                                messageStore.recoverTopicQueueTable();
+                                brokerController.changeToMaster(BrokerRole.SYNC_MASTER);
+                            }
+                            break;
+                        default:
+                            break;
+                    }
+                    log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start));
+                } catch (Throwable t) {
+                    log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start), t);
+                }
+            }
+        };
+        executorService.submit(runnable);
+    }
+
+    @Override public void startup() {
+
+    }
+
+    @Override public void shutdown() {
+        executorService.shutdown();
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java

@@ -152,7 +152,7 @@ public class BrokerOuterAPI {
                                 registerBrokerResultList.add(result);
                             }
 
-                            log.info("register broker to name server {} OK", namesrvAddr);
+                            log.info("register broker[{}]to name server {} OK", brokerId, namesrvAddr);
                         } catch (Exception e) {
                             log.warn("registerBroker Exception, {}", namesrvAddr, e);
                         } finally {

broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java

@@ -54,7 +54,7 @@ public class SlaveSynchronize {
 
     private void syncTopicConfig() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 TopicConfigSerializeWrapper topicWrapper =
                     this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak);
@@ -78,7 +78,7 @@ public class SlaveSynchronize {
 
     private void syncConsumerOffset() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 ConsumerOffsetSerializeWrapper offsetWrapper =
                     this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak);
@@ -94,7 +94,7 @@ public class SlaveSynchronize {
 
     private void syncDelayOffset() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 String delayOffset =
                     this.brokerController.getBrokerOuterAPI().getAllDelayOffset(masterAddrBak);
@@ -118,7 +118,7 @@ public class SlaveSynchronize {
 
     private void syncSubscriptionGroupConfig() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null  && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 SubscriptionGroupWrapper subscriptionWrapper =
                     this.brokerController.getBrokerOuterAPI()

broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java

@@ -124,6 +124,16 @@ public class TopicConfigManager extends ConfigManager {
             topicConfig.setWriteQueueNums(1);
             this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
         }
+        {
+            if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) {
+                String topic = this.brokerController.getBrokerConfig().getMsgTraceTopicName();
+                TopicConfig topicConfig = new TopicConfig(topic);
+                this.systemTopicList.add(topic);
+                topicConfig.setReadQueueNums(1);
+                topicConfig.setWriteQueueNums(1);
+                this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+            }
+        }
     }
 
     public boolean isSystemTopic(final String topic) {

broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java

@@ -22,36 +22,15 @@ import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 
-import java.util.concurrent.atomic.AtomicBoolean;
-
 public class TransactionalMessageCheckService extends ServiceThread {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
 
     private BrokerController brokerController;
 
-    private final AtomicBoolean started = new AtomicBoolean(false);
-
     public TransactionalMessageCheckService(BrokerController brokerController) {
         this.brokerController = brokerController;
     }
 
-    @Override
-    public void start() {
-        if (started.compareAndSet(false, true)) {
-            super.start();
-            this.brokerController.getTransactionalMessageService().open();
-        }
-    }
-
-    @Override
-    public void shutdown(boolean interrupt) {
-        if (started.compareAndSet(true, false)) {
-            super.shutdown(interrupt);
-            this.brokerController.getTransactionalMessageService().close();
-            this.brokerController.getTransactionalMessageCheckListener().shutDown();
-        }
-    }
-
     @Override
     public String getServiceName() {
         return TransactionalMessageCheckService.class.getSimpleName();

broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java

@@ -198,7 +198,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ
                         if (null != checkImmunityTimeStr) {
                             checkImmunityTime = getImmunityTime(checkImmunityTimeStr, transactionTimeout);
                             if (valueOfCurrentMinusBorn < checkImmunityTime) {
-                                if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt, checkImmunityTime)) {
+                                if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt)) {
                                     newOffset = i + 1;
                                     i++;
                                     continue;
@@ -315,33 +315,26 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ
      * @param removeMap Op message map to determine whether a half message was responded by producer.
      * @param doneOpOffset Op Message which has been checked.
      * @param msgExt Half message
-     * @param checkImmunityTime User defined time to avoid being detected early.
      * @return Return true if put success, otherwise return false.
      */
-    private boolean checkPrepareQueueOffset(HashMap<Long, Long> removeMap, List<Long> doneOpOffset, MessageExt msgExt,
-        long checkImmunityTime) {
-        if (System.currentTimeMillis() - msgExt.getBornTimestamp() < checkImmunityTime) {
-            String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);
-            if (null == prepareQueueOffsetStr) {
-                return putImmunityMsgBackToHalfQueue(msgExt);
-            } else {
-                long prepareQueueOffset = getLong(prepareQueueOffsetStr);
-                if (-1 == prepareQueueOffset) {
-                    return false;
-                } else {
-                    if (removeMap.containsKey(prepareQueueOffset)) {
-                        long tmpOpOffset = removeMap.remove(prepareQueueOffset);
-                        doneOpOffset.add(tmpOpOffset);
-                        return true;
-                    } else {
-                        return putImmunityMsgBackToHalfQueue(msgExt);
-                    }
-                }
-
-            }
-
+    private boolean checkPrepareQueueOffset(HashMap<Long, Long> removeMap, List<Long> doneOpOffset,
+        MessageExt msgExt) {
+        String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);
+        if (null == prepareQueueOffsetStr) {
+            return putImmunityMsgBackToHalfQueue(msgExt);
         } else {
-            return true;
+            long prepareQueueOffset = getLong(prepareQueueOffsetStr);
+            if (-1 == prepareQueueOffset) {
+                return false;
+            } else {
+                if (removeMap.containsKey(prepareQueueOffset)) {
+                    long tmpOpOffset = removeMap.remove(prepareQueueOffset);
+                    doneOpOffset.add(tmpOpOffset);
+                    return true;
+                } else {
+                    return putImmunityMsgBackToHalfQueue(msgExt);
+                }
+            }
         }
     }
 

broker/src/main/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator

@@ -0,0 +1 @@
+org.apache.rocketmq.acl.plain.PlainAccessValidator

broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java

@@ -24,6 +24,7 @@ import org.apache.rocketmq.remoting.ClientConfig;
 import org.apache.rocketmq.remoting.ServerConfig;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.After;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;

broker/src/test/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransferTest.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.pagecache;
+
+import java.nio.ByteBuffer;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ManyMessageTransferTest {
+
+    @Test
+    public void ManyMessageTransferBuilderTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        GetMessageResult getMessageResult = new GetMessageResult();
+        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);
+    }
+
+    @Test
+    public void ManyMessageTransferPosTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        GetMessageResult getMessageResult = new GetMessageResult();
+        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);
+        Assert.assertEquals(manyMessageTransfer.position(),4);
+    }
+
+    @Test
+    public void ManyMessageTransferCountTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        GetMessageResult getMessageResult = new GetMessageResult();
+        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);
+
+        Assert.assertEquals(manyMessageTransfer.count(),20);
+
+    }
+
+    @Test
+    public void ManyMessageTransferCloseTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        GetMessageResult getMessageResult = new GetMessageResult();
+        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);
+        manyMessageTransfer.close();
+        manyMessageTransfer.deallocate();
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/pagecache/OneMessageTransferTest.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.pagecache;
+
+import java.nio.ByteBuffer;
+import org.apache.rocketmq.store.MappedFile;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class OneMessageTransferTest {
+
+    @Test
+    public void OneMessageTransferTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile());
+        OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);
+    }
+
+    @Test
+    public void OneMessageTransferCountTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile());
+        OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);
+        Assert.assertEquals(manyMessageTransfer.count(),40);
+    }
+
+    @Test
+    public void OneMessageTransferPosTest(){
+        ByteBuffer byteBuffer = ByteBuffer.allocate(20);
+        byteBuffer.putInt(20);
+        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile());
+        OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);
+        Assert.assertEquals(manyMessageTransfer.position(),8);
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java

@@ -17,6 +17,8 @@
 
 package org.apache.rocketmq.broker.util;
 
+import java.util.List;
+import org.apache.rocketmq.acl.AccessValidator;
 import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
 import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
 import org.apache.rocketmq.remoting.util.ServiceProvider;
@@ -39,4 +41,10 @@ public class ServiceProviderTest {
             AbstractTransactionalMessageCheckListener.class);
         assertThat(listener).isNotNull();
     }
+    
+    @Test
+    public void loadAccessValidatorTest() {
+    	 List<AccessValidator> accessValidators = ServiceProvider.loadServiceList(ServiceProvider.ACL_VALIDATOR_ID, AccessValidator.class);
+    	 assertThat(accessValidators).isNotNull();
+    }
 }

broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator

@@ -0,0 +1 @@
+org.apache.rocketmq.acl.plain.PlainAccessValidator

client/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java

@@ -257,6 +257,18 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume
         return this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, timeout);
     }
 
+    @Override
+    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums)
+        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+        return this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums);
+    }
+
+    @Override
+    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout)
+        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+        return this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums, timeout);
+    }
+
     @Override
     public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback)
         throws MQClientException, RemotingException, InterruptedException {
@@ -270,6 +282,20 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume
         this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, pullCallback, timeout);
     }
 
+    @Override
+    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,
+        PullCallback pullCallback)
+        throws MQClientException, RemotingException, InterruptedException {
+        this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums, pullCallback);
+    }
+
+    @Override
+    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,
+        PullCallback pullCallback, long timeout)
+        throws MQClientException, RemotingException, InterruptedException {
+        this.defaultMQPullConsumerImpl.pull(mq, messageSelector, offset, maxNums, pullCallback, timeout);
+    }
+
     @Override
     public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums)
         throws MQClientException, RemotingException, MQBrokerException, InterruptedException {

client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java

@@ -29,6 +29,10 @@ import org.apache.rocketmq.client.consumer.store.OffsetStore;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.client.trace.AsyncTraceDispatcher;
+import org.apache.rocketmq.client.trace.TraceDispatcher;
+import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@ -36,6 +40,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 
@@ -56,6 +61,8 @@ import org.apache.rocketmq.remoting.exception.RemotingException;
  */
 public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsumer {
 
+    private final InternalLogger log = ClientLogger.getLog();
+
     /**
      * Internal implementation. Most of the functions herein are delegated to it.
      */
@@ -246,6 +253,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
      */
     private long consumeTimeout = 15;
 
+    /**
+     * Interface of asynchronous transfer data
+     */
+    private TraceDispatcher traceDispatcher = null;
+
     /**
      * Default constructor.
      */
@@ -258,7 +270,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
      *
      * @param consumerGroup Consume queue.
      * @param rpcHook RPC hook to execute before each remoting command.
-     * @param allocateMessageQueueStrategy message queue allocating algorithm.
+     * @param allocateMessageQueueStrategy Message queue allocating algorithm.
      */
     public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook,
         AllocateMessageQueueStrategy allocateMessageQueueStrategy) {
@@ -267,6 +279,33 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
         defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
     }
 
+    /**
+     * Constructor specifying consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name.
+     *
+     * @param consumerGroup Consume queue.
+     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param allocateMessageQueueStrategy message queue allocating algorithm.
+     * @param enableMsgTrace Switch flag instance for message trace.
+     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
+     */
+    public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook,
+        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, final String customizedTraceTopic) {
+        this.consumerGroup = consumerGroup;
+        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
+        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
+        if (enableMsgTrace) {
+            try {
+                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook);
+                dispatcher.setHostConsumer(this.getDefaultMQPushConsumerImpl());
+                traceDispatcher = dispatcher;
+                this.getDefaultMQPushConsumerImpl().registerConsumeMessageHook(
+                    new ConsumeMessageTraceHookImpl(traceDispatcher));
+            } catch (Throwable e) {
+                log.error("system mqtrace hook init failed ,maybe can't send msg trace data");
+            }
+        }
+    }
+
     /**
      * Constructor specifying RPC hook.
      *
@@ -276,6 +315,28 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
         this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook, new AllocateMessageQueueAveragely());
     }
 
+
+    /**
+     * Constructor specifying consumer group and enabled msg trace flag.
+     *
+     * @param consumerGroup Consumer group.
+     * @param enableMsgTrace Switch flag instance for message trace.
+     */
+    public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace) {
+        this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, null);
+    }
+
+    /**
+     * Constructor specifying consumer group, enabled msg trace flag and customized trace topic name.
+     *
+     * @param consumerGroup Consumer group.
+     * @param enableMsgTrace Switch flag instance for message trace.
+     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
+     */
+    public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, final String customizedTraceTopic) {
+        this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, customizedTraceTopic);
+    }
+
     /**
      * Constructor specifying consumer group.
      *
@@ -518,6 +579,13 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     @Override
     public void start() throws MQClientException {
         this.defaultMQPushConsumerImpl.start();
+        if (null != traceDispatcher) {
+            try {
+                traceDispatcher.start(this.getNamesrvAddr());
+            } catch (MQClientException e) {
+                log.warn("trace dispatcher start failed ", e);
+            }
+        }
     }
 
     /**
@@ -526,6 +594,9 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     @Override
     public void shutdown() {
         this.defaultMQPushConsumerImpl.shutdown();
+        if (null != traceDispatcher) {
+            traceDispatcher.shutdown();
+        }
     }
 
     @Override
@@ -694,4 +765,8 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     public void setConsumeTimeout(final long consumeTimeout) {
         this.consumeTimeout = consumeTimeout;
     }
+
+    public TraceDispatcher getTraceDispatcher() {
+        return traceDispatcher;
+    }
 }

client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java

@@ -66,6 +66,39 @@ public interface MQPullConsumer extends MQConsumer {
         final int maxNums, final long timeout) throws MQClientException, RemotingException,
         MQBrokerException, InterruptedException;
 
+    /**
+     * Pulling the messages, not blocking
+     * <p>
+     * support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}
+     * </p>
+     *
+     * @param mq from which message queue
+     * @param selector message selector({@link MessageSelector}), can be null.
+     * @param offset from where to pull
+     * @param maxNums max pulling numbers
+     * @return The resulting {@code PullRequest}
+     */
+    PullResult pull(final MessageQueue mq, final MessageSelector selector, final long offset,
+        final int maxNums) throws MQClientException, RemotingException, MQBrokerException,
+        InterruptedException;
+
+    /**
+     * Pulling the messages in the specified timeout
+     * <p>
+     * support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}
+     * </p>
+     *
+     * @param mq from which message queue
+     * @param selector message selector({@link MessageSelector}), can be null.
+     * @param offset from where to pull
+     * @param maxNums max pulling numbers
+     * @param timeout Pulling the messages in the specified timeout
+     * @return The resulting {@code PullRequest}
+     */
+    PullResult pull(final MessageQueue mq, final MessageSelector selector, final long offset,
+        final int maxNums, final long timeout) throws MQClientException, RemotingException, MQBrokerException,
+        InterruptedException;
+
     /**
      * Pulling the messages in a async. way
      */
@@ -80,6 +113,20 @@ public interface MQPullConsumer extends MQConsumer {
         final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException,
         InterruptedException;
 
+    /**
+     * Pulling the messages in a async. way. Support message selection
+     */
+    void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums,
+        final PullCallback pullCallback) throws MQClientException, RemotingException,
+        InterruptedException;
+
+    /**
+     * Pulling the messages in a async. way. Support message selection
+     */
+    void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums,
+        final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException,
+        InterruptedException;
+
     /**
      * Pulling the messages,if no message arrival,blocking some time
      *

client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java

@@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentMap;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.Validators;
 import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.MessageSelector;
 import org.apache.rocketmq.client.consumer.PullCallback;
 import org.apache.rocketmq.client.consumer.PullResult;
 import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
@@ -46,6 +47,7 @@ import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.ServiceState;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.filter.ExpressionType;
 import org.apache.rocketmq.common.filter.FilterAPI;
 import org.apache.rocketmq.common.help.FAQUrl;
 import org.apache.rocketmq.logging.InternalLogger;
@@ -158,17 +160,58 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
 
     public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout)
         throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
-        return this.pullSyncImpl(mq, subExpression, offset, maxNums, false, timeout);
+        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);
+        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout);
     }
 
-    private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block,
+    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums)
+        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+        return pull(mq, messageSelector, offset, maxNums, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());
+    }
+
+    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout)
+        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);
+        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout);
+    }
+
+    private SubscriptionData getSubscriptionData(MessageQueue mq, String subExpression)
+        throws MQClientException {
+
+        if (null == mq) {
+            throw new MQClientException("mq is null", null);
+        }
+
+        try {
+            return FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),
+                mq.getTopic(), subExpression);
+        } catch (Exception e) {
+            throw new MQClientException("parse subscription error", e);
+        }
+    }
+
+    private SubscriptionData getSubscriptionData(MessageQueue mq, MessageSelector messageSelector)
+        throws MQClientException {
+
+        if (null == mq) {
+            throw new MQClientException("mq is null", null);
+        }
+
+        try {
+            return FilterAPI.build(mq.getTopic(),
+                messageSelector.getExpression(), messageSelector.getExpressionType());
+        } catch (Exception e) {
+            throw new MQClientException("parse subscription error", e);
+        }
+    }
+
+    private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, boolean block,
         long timeout)
         throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
         this.makeSureStateOK();
 
         if (null == mq) {
             throw new MQClientException("mq is null", null);
-
         }
 
         if (offset < 0) {
@@ -183,20 +226,14 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
 
         int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);
 
-        SubscriptionData subscriptionData;
-        try {
-            subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),
-                mq.getTopic(), subExpression);
-        } catch (Exception e) {
-            throw new MQClientException("parse subscription error", e);
-        }
-
         long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;
 
+        boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType());
         PullResult pullResult = this.pullAPIWrapper.pullKernelImpl(
             mq,
             subscriptionData.getSubString(),
-            0L,
+            subscriptionData.getExpressionType(),
+            isTagType ? 0L : subscriptionData.getSubVersion(),
             offset,
             maxNums,
             sysFlag,
@@ -369,12 +406,27 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
     public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback,
         long timeout)
         throws MQClientException, RemotingException, InterruptedException {
-        this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, false, timeout);
+        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);
+        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, false, timeout);
+    }
+
+    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,
+        PullCallback pullCallback)
+        throws MQClientException, RemotingException, InterruptedException {
+        pull(mq, messageSelector, offset, maxNums, pullCallback, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());
+    }
+
+    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,
+        PullCallback pullCallback,
+        long timeout)
+        throws MQClientException, RemotingException, InterruptedException {
+        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);
+        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, false, timeout);
     }
 
     private void pullAsyncImpl(
         final MessageQueue mq,
-        final String subExpression,
+        final SubscriptionData subscriptionData,
         final long offset,
         final int maxNums,
         final PullCallback pullCallback,
@@ -403,20 +455,14 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
         try {
             int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);
 
-            final SubscriptionData subscriptionData;
-            try {
-                subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),
-                    mq.getTopic(), subExpression);
-            } catch (Exception e) {
-                throw new MQClientException("parse subscription error", e);
-            }
-
             long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;
 
+            boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType());
             this.pullAPIWrapper.pullKernelImpl(
                 mq,
                 subscriptionData.getSubString(),
-                0L,
+                subscriptionData.getExpressionType(),
+                isTagType ? 0L : subscriptionData.getSubVersion(),
                 offset,
                 maxNums,
                 sysFlag,
@@ -444,7 +490,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
 
     public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums)
         throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
-        return this.pullSyncImpl(mq, subExpression, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
+        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);
+        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
     }
 
     public DefaultMQPullConsumer getDefaultMQPullConsumer() {
@@ -454,7 +501,8 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
     public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums,
         PullCallback pullCallback)
         throws MQClientException, RemotingException, InterruptedException {
-        this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, true,
+        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);
+        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, true,
             this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
     }
 

client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java

@@ -212,34 +212,6 @@ public class PullAPIWrapper {
         throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
     }
 
-    public PullResult pullKernelImpl(
-        final MessageQueue mq,
-        final String subExpression,
-        final long subVersion,
-        final long offset,
-        final int maxNums,
-        final int sysFlag,
-        final long commitOffset,
-        final long brokerSuspendMaxTimeMillis,
-        final long timeoutMillis,
-        final CommunicationMode communicationMode,
-        final PullCallback pullCallback
-    ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
-        return pullKernelImpl(
-            mq,
-            subExpression,
-            ExpressionType.TAG,
-            subVersion, offset,
-            maxNums,
-            sysFlag,
-            commitOffset,
-            brokerSuspendMaxTimeMillis,
-            timeoutMillis,
-            communicationMode,
-            pullCallback
-        );
-    }
-
     public long recalculatePullFromWhichNode(final MessageQueue mq) {
         if (this.isConnectBrokerByUser()) {
             return this.defaultBrokerId;

client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java

@@ -678,10 +678,10 @@ public class MQClientInstance {
                             } catch (Exception e) {
                                 log.error("send heart beat error:{}", e);
                                 if (this.isBrokerInNameServer(addr)) {
-                                    log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr);
+                                    log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr, e);
                                 } else {
                                     log.info("send heart beat to broker[{} {} {}] exception, because the broker not up, forget it", brokerName,
-                                        id, addr);
+                                        id, addr, e);
                                 }
                             }
                         }
@@ -1186,6 +1186,7 @@ public class MQClientInstance {
                 return this.brokerVersionTable.get(brokerName).get(brokerAddr);
             }
         }
+        //To do need to fresh the version
         return 0;
     }
 

client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java

@@ -30,8 +30,10 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.Validators;
 import org.apache.rocketmq.client.common.ClientErrorCode;
@@ -101,6 +103,10 @@ public class DefaultMQProducerImpl implements MQProducerInner {
 
     private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy();
 
+    private final BlockingQueue<Runnable> asyncSenderThreadPoolQueue;
+    private final ExecutorService defaultAsyncSenderExecutor;
+    private ExecutorService asyncSenderExecutor;
+
     public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) {
         this(defaultMQProducer, null);
     }
@@ -108,6 +114,22 @@ public class DefaultMQProducerImpl implements MQProducerInner {
     public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) {
         this.defaultMQProducer = defaultMQProducer;
         this.rpcHook = rpcHook;
+
+        this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue<Runnable>(50000);
+        this.defaultAsyncSenderExecutor = new ThreadPoolExecutor(
+            Runtime.getRuntime().availableProcessors(),
+            Runtime.getRuntime().availableProcessors(),
+            1000 * 60,
+            TimeUnit.MILLISECONDS,
+            this.asyncSenderThreadPoolQueue,
+            new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, "AsyncSenderExecutor_" + this.threadIndex.incrementAndGet());
+                }
+            });
     }
 
     public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) {
@@ -462,7 +484,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
     public void send(final Message msg, final SendCallback sendCallback, final long timeout)
         throws MQClientException, RemotingException, InterruptedException {
         final long beginStartTime = System.currentTimeMillis();
-        ExecutorService executor = this.getCallbackExecutor();
+        ExecutorService executor = this.getAsyncSenderExecutor();
         try {
             executor.submit(new Runnable() {
                 @Override
@@ -656,6 +678,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
             return topicPublishInfo;
         }
     }
+
     private void tryToFindSnodePublishInfo() {
         this.mQClientFactory.updateSnodeInfoFromNameServer();
     }
@@ -967,7 +990,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
     public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout)
         throws MQClientException, RemotingException, InterruptedException {
         final long beginStartTime = System.currentTimeMillis();
-        ExecutorService executor = this.getCallbackExecutor();
+        ExecutorService executor = this.getAsyncSenderExecutor();
         try {
             executor.submit(new Runnable() {
                 @Override
@@ -1091,7 +1114,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
         final SendCallback sendCallback, final long timeout)
         throws MQClientException, RemotingException, InterruptedException {
         final long beginStartTime = System.currentTimeMillis();
-        ExecutorService executor = this.getCallbackExecutor();
+        ExecutorService executor = this.getAsyncSenderExecutor();
         try {
             executor.submit(new Runnable() {
                 @Override
@@ -1258,7 +1281,14 @@ public class DefaultMQProducerImpl implements MQProducerInner {
 
     public ExecutorService getCallbackExecutor() {
         return this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().getCallbackExecutor();
+    }
 
+    public ExecutorService getAsyncSenderExecutor() {
+        return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor;
+    }
+
+    public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) {
+        this.asyncSenderExecutor = asyncSenderExecutor;
     }
 
     public SendResult send(Message msg,

client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java

@@ -25,6 +25,10 @@ import org.apache.rocketmq.client.Validators;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.client.trace.AsyncTraceDispatcher;
+import org.apache.rocketmq.client.trace.TraceDispatcher;
+import org.apache.rocketmq.client.trace.hook.SendMessageTraceHookImpl;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageBatch;
@@ -33,6 +37,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageId;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 
@@ -55,6 +60,8 @@ import org.apache.rocketmq.remoting.exception.RemotingException;
  */
 public class DefaultMQProducer extends ClientConfig implements MQProducer {
 
+    private final InternalLogger log = ClientLogger.getLog();
+
     /**
      * Wrapping internal implementations for virtually all methods presented in this class.
      */
@@ -118,6 +125,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
      */
     private int maxMessageSize = 1024 * 1024 * 4; // 4M
 
+    /**
+     * Interface of asynchronous transfer data
+     */
+    private TraceDispatcher traceDispatcher = null;
+
     /**
      * Default constructor.
      */
@@ -136,6 +148,31 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);
     }
 
+    /**
+     * Constructor specifying producer group, RPC hook, enabled msgTrace flag and customized trace topic name.
+     *
+     * @param producerGroup Producer group, see the name-sake field.
+     * @param rpcHook RPC hook to execute per each remoting command execution.
+     * @param enableMsgTrace Switch flag instance for message trace.
+     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
+     */
+    public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic) {
+        this.producerGroup = producerGroup;
+        defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);
+        //if client open the message trace feature
+        if (enableMsgTrace) {
+            try {
+                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook);
+                dispatcher.setHostProducer(this.getDefaultMQProducerImpl());
+                traceDispatcher = dispatcher;
+                this.getDefaultMQProducerImpl().registerSendMessageHook(
+                    new SendMessageTraceHookImpl(traceDispatcher));
+            } catch (Throwable e) {
+                log.error("system mqtrace hook init failed ,maybe can't send msg trace data");
+            }
+        }
+    }
+
     /**
      * Constructor specifying producer group.
      *
@@ -146,8 +183,30 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
     }
 
     /**
-     * Constructor specifying the RPC hook.
+     * Constructor specifying producer group and enabled msg trace flag.
      *
+     * @param producerGroup Producer group, see the name-sake field.
+     * @param enableMsgTrace Switch flag instance for message trace.
+     */
+    public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace) {
+        this(producerGroup, null, enableMsgTrace, null);
+    }
+
+
+    /**
+     * Constructor specifying producer group, enabled msgTrace flag and customized trace topic name.
+     *
+     * @param producerGroup Producer group, see the name-sake field.
+     * @param enableMsgTrace Switch flag instance for message trace.
+     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
+     */
+    public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic) {
+        this(producerGroup, null, enableMsgTrace, customizedTraceTopic);
+    }
+
+    /**
+     * Constructor specifying the RPC hook.
+     * 
      * @param rpcHook RPC hook to execute per each remoting command execution.
      */
     public DefaultMQProducer(RPCHook rpcHook) {
@@ -169,6 +228,13 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
     @Override
     public void start() throws MQClientException {
         this.defaultMQProducerImpl.start();
+        if (null != traceDispatcher) {
+            try {
+                traceDispatcher.start(this.getNamesrvAddr());
+            } catch (MQClientException e) {
+                log.warn("trace dispatcher start failed ", e);
+            }
+        }
     }
 
     /**
@@ -177,6 +243,9 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
     @Override
     public void shutdown() {
         this.defaultMQProducerImpl.shutdown();
+        if (null != traceDispatcher) {
+            traceDispatcher.shutdown();
+        }
     }
 
     /**
@@ -654,6 +723,16 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         this.defaultMQProducerImpl.setCallbackExecutor(callbackExecutor);
     }
 
+    /**
+     * Sets an Executor to be used for executing asynchronous send. If the Executor is not set, {@link
+     * DefaultMQProducerImpl#defaultAsyncSenderExecutor} will be used.
+     *
+     * @param asyncSenderExecutor the instance of Executor
+     */
+    public void setAsyncSenderExecutor(final ExecutorService asyncSenderExecutor) {
+        this.defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor);
+    }
+
     private MessageBatch batch(Collection<Message> msgs) throws MQClientException {
         MessageBatch msgBatch;
         try {
@@ -776,4 +855,9 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
     public void setRetryTimesWhenSendAsyncFailed(final int retryTimesWhenSendAsyncFailed) {
         this.retryTimesWhenSendAsyncFailed = retryTimesWhenSendAsyncFailed;
     }
+
+    public TraceDispatcher getTraceDispatcher() {
+        return traceDispatcher;
+    }
+
 }

client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java

@@ -0,0 +1,386 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+import org.apache.rocketmq.client.common.ThreadLocalIndex;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.MessageQueueSelector;
+import org.apache.rocketmq.client.producer.SendCallback;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.remoting.RPCHook;
+
+import java.io.IOException;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.UUID;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+
+
+import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAME;
+
+public class AsyncTraceDispatcher implements TraceDispatcher {
+
+    private final static InternalLogger log = ClientLogger.getLog();
+    private final int queueSize;
+    private final int batchSize;
+    private final int maxMsgSize;
+    private final DefaultMQProducer traceProducer;
+    private final ThreadPoolExecutor traceExecuter;
+    // The last discard number of log
+    private AtomicLong discardCount;
+    private Thread worker;
+    private ArrayBlockingQueue<TraceContext> traceContextQueue;
+    private ArrayBlockingQueue<Runnable> appenderQueue;
+    private volatile Thread shutDownHook;
+    private volatile boolean stopped = false;
+    private DefaultMQProducerImpl hostProducer;
+    private DefaultMQPushConsumerImpl hostConsumer;
+    private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex();
+    private String dispatcherId = UUID.randomUUID().toString();
+    private String traceTopicName;
+    private AtomicBoolean isStarted = new AtomicBoolean(false);
+
+
+    public AsyncTraceDispatcher(String traceTopicName, RPCHook rpcHook) throws MQClientException {
+        // queueSize is greater than or equal to the n power of 2 of value
+        this.queueSize = 2048;
+        this.batchSize = 100;
+        this.maxMsgSize = 128000;
+        this.discardCount = new AtomicLong(0L);
+        this.traceContextQueue = new ArrayBlockingQueue<TraceContext>(1024);
+        this.appenderQueue = new ArrayBlockingQueue<Runnable>(queueSize);
+        if (!UtilAll.isBlank(traceTopicName)) {
+            this.traceTopicName = traceTopicName;
+        } else {
+            this.traceTopicName = MixAll.RMQ_SYS_TRACE_TOPIC;
+        }
+        this.traceExecuter = new ThreadPoolExecutor(//
+                10, //
+                20, //
+                1000 * 60, //
+                TimeUnit.MILLISECONDS, //
+                this.appenderQueue, //
+                new ThreadFactoryImpl("MQTraceSendThread_"));
+        traceProducer = getAndCreateTraceProducer(rpcHook);
+    }
+
+    public String getTraceTopicName() {
+        return traceTopicName;
+    }
+
+    public void setTraceTopicName(String traceTopicName) {
+        this.traceTopicName = traceTopicName;
+    }
+
+    public DefaultMQProducer getTraceProducer() {
+        return traceProducer;
+    }
+
+    public DefaultMQProducerImpl getHostProducer() {
+        return hostProducer;
+    }
+
+    public void setHostProducer(DefaultMQProducerImpl hostProducer) {
+        this.hostProducer = hostProducer;
+    }
+
+    public DefaultMQPushConsumerImpl getHostConsumer() {
+        return hostConsumer;
+    }
+
+    public void setHostConsumer(DefaultMQPushConsumerImpl hostConsumer) {
+        this.hostConsumer = hostConsumer;
+    }
+
+    public void start(String nameSrvAddr) throws MQClientException {
+        if (isStarted.compareAndSet(false, true)) {
+            traceProducer.setNamesrvAddr(nameSrvAddr);
+            traceProducer.setInstanceName(TRACE_INSTANCE_NAME + "_" + nameSrvAddr);
+            traceProducer.start();
+        }
+        this.worker = new Thread(new AsyncRunnable(), "MQ-AsyncTraceDispatcher-Thread-" + dispatcherId);
+        this.worker.setDaemon(true);
+        this.worker.start();
+        this.registerShutDownHook();
+    }
+
+    private DefaultMQProducer getAndCreateTraceProducer(RPCHook rpcHook) {
+        DefaultMQProducer traceProducerInstance = this.traceProducer;
+        if (traceProducerInstance == null) {
+            traceProducerInstance = new DefaultMQProducer(rpcHook);
+            traceProducerInstance.setProducerGroup(TraceConstants.GROUP_NAME);
+            traceProducerInstance.setSendMsgTimeout(5000);
+            traceProducerInstance.setVipChannelEnabled(false);
+            // The max size of message is 128K
+            traceProducerInstance.setMaxMessageSize(maxMsgSize - 10 * 1000);
+        }
+        return traceProducerInstance;
+    }
+
+    @Override
+    public boolean append(final Object ctx) {
+        boolean result = traceContextQueue.offer((TraceContext) ctx);
+        if (!result) {
+            log.info("buffer full" + discardCount.incrementAndGet() + " ,context is " + ctx);
+        }
+        return result;
+    }
+
+    @Override
+    public void flush() throws IOException {
+        // The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return.
+        long end = System.currentTimeMillis() + 500;
+        while (traceContextQueue.size() > 0 || appenderQueue.size() > 0 && System.currentTimeMillis() <= end) {
+            try {
+                Thread.sleep(1);
+            } catch (InterruptedException e) {
+                break;
+            }
+        }
+        log.info("------end trace send " + traceContextQueue.size() + "   " + appenderQueue.size());
+    }
+
+    @Override
+    public void shutdown() {
+        this.stopped = true;
+        this.traceExecuter.shutdown();
+        if (isStarted.get()) {
+            traceProducer.shutdown();
+        }
+        this.removeShutdownHook();
+    }
+
+    public void registerShutDownHook() {
+        if (shutDownHook == null) {
+            shutDownHook = new Thread(new Runnable() {
+                private volatile boolean hasShutdown = false;
+
+                @Override
+                public void run() {
+                    synchronized (this) {
+                        if (!this.hasShutdown) {
+                            try {
+                                flush();
+                            } catch (IOException e) {
+                                log.error("system MQTrace hook shutdown failed ,maybe loss some trace data");
+                            }
+                        }
+                    }
+                }
+            }, "ShutdownHookMQTrace");
+            Runtime.getRuntime().addShutdownHook(shutDownHook);
+        }
+    }
+
+    public void removeShutdownHook() {
+        if (shutDownHook != null) {
+            Runtime.getRuntime().removeShutdownHook(shutDownHook);
+        }
+    }
+
+    class AsyncRunnable implements Runnable {
+        private boolean stopped;
+
+        @Override
+        public void run() {
+            while (!stopped) {
+                List<TraceContext> contexts = new ArrayList<TraceContext>(batchSize);
+                for (int i = 0; i < batchSize; i++) {
+                    TraceContext context = null;
+                    try {
+                        //get trace data element from blocking Queue — traceContextQueue
+                        context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS);
+                    } catch (InterruptedException e) {
+                    }
+                    if (context != null) {
+                        contexts.add(context);
+                    } else {
+                        break;
+                    }
+                }
+                if (contexts.size() > 0) {
+                    AsyncAppenderRequest request = new AsyncAppenderRequest(contexts);
+                    traceExecuter.submit(request);
+                } else if (AsyncTraceDispatcher.this.stopped) {
+                    this.stopped = true;
+                }
+            }
+
+        }
+    }
+
+    class AsyncAppenderRequest implements Runnable {
+        List<TraceContext> contextList;
+
+        public AsyncAppenderRequest(final List<TraceContext> contextList) {
+            if (contextList != null) {
+                this.contextList = contextList;
+            } else {
+                this.contextList = new ArrayList<TraceContext>(1);
+            }
+        }
+
+        @Override
+        public void run() {
+            sendTraceData(contextList);
+        }
+
+        public void sendTraceData(List<TraceContext> contextList) {
+            Map<String, List<TraceTransferBean>> transBeanMap = new HashMap<String, List<TraceTransferBean>>();
+            for (TraceContext context : contextList) {
+                if (context.getTraceBeans().isEmpty()) {
+                    continue;
+                }
+                // Topic value corresponding to original message entity content
+                String topic = context.getTraceBeans().get(0).getTopic();
+                // Use  original message entity's topic as key
+                String key = topic;
+                List<TraceTransferBean> transBeanList = transBeanMap.get(key);
+                if (transBeanList == null) {
+                    transBeanList = new ArrayList<TraceTransferBean>();
+                    transBeanMap.put(key, transBeanList);
+                }
+                TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context);
+                transBeanList.add(traceData);
+            }
+            for (Map.Entry<String, List<TraceTransferBean>> entry : transBeanMap.entrySet()) {
+                flushData(entry.getValue());
+            }
+        }
+
+        /**
+         * Batch sending data actually
+         */
+        private void flushData(List<TraceTransferBean> transBeanList) {
+            if (transBeanList.size() == 0) {
+                return;
+            }
+            // Temporary buffer
+            StringBuilder buffer = new StringBuilder(1024);
+            int count = 0;
+            Set<String> keySet = new HashSet<String>();
+
+            for (TraceTransferBean bean : transBeanList) {
+                // Keyset of message trace includes msgId of or original message
+                keySet.addAll(bean.getTransKey());
+                buffer.append(bean.getTransData());
+                count++;
+                // Ensure that the size of the package should not exceed the upper limit.
+                if (buffer.length() >= traceProducer.getMaxMessageSize()) {
+                    sendTraceDataByMQ(keySet, buffer.toString());
+                    // Clear temporary buffer after finishing
+                    buffer.delete(0, buffer.length());
+                    keySet.clear();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                sendTraceDataByMQ(keySet, buffer.toString());
+            }
+            transBeanList.clear();
+        }
+
+        /**
+         * Send message trace data
+         *
+         * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId)
+         * @param data   the message trace data in this batch
+         */
+        private void sendTraceDataByMQ(Set<String> keySet, final String data) {
+            String topic = traceTopicName;
+            final Message message = new Message(topic, data.getBytes());
+
+            // Keyset of message trace includes msgId of or original message
+            message.setKeys(keySet);
+            try {
+                Set<String> traceBrokerSet = tryGetMessageQueueBrokerSet(traceProducer.getDefaultMQProducerImpl(), topic);
+                SendCallback callback = new SendCallback() {
+                    @Override
+                    public void onSuccess(SendResult sendResult) {
+                    }
+
+                    @Override
+                    public void onException(Throwable e) {
+                        log.info("send trace data ,the traceData is " + data);
+                    }
+                };
+                if (traceBrokerSet.isEmpty()) {
+                    // No cross set
+                    traceProducer.send(message, callback, 5000);
+                } else {
+                    traceProducer.send(message, new MessageQueueSelector() {
+                        @Override
+                        public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
+                            Set<String> brokerSet = (Set<String>) arg;
+                            List<MessageQueue> filterMqs = new ArrayList<MessageQueue>();
+                            for (MessageQueue queue : mqs) {
+                                if (brokerSet.contains(queue.getBrokerName())) {
+                                    filterMqs.add(queue);
+                                }
+                            }
+                            int index = sendWhichQueue.getAndIncrement();
+                            int pos = Math.abs(index) % filterMqs.size();
+                            if (pos < 0) {
+                                pos = 0;
+                            }
+                            return filterMqs.get(pos);
+                        }
+                    }, traceBrokerSet, callback);
+                }
+
+            } catch (Exception e) {
+                log.info("send trace data,the traceData is" + data);
+            }
+        }
+
+        private Set<String> tryGetMessageQueueBrokerSet(DefaultMQProducerImpl producer, String topic) {
+            Set<String> brokerSet = new HashSet<String>();
+            TopicPublishInfo topicPublishInfo = producer.getTopicPublishInfoTable().get(topic);
+            if (null == topicPublishInfo || !topicPublishInfo.ok()) {
+                producer.getTopicPublishInfoTable().putIfAbsent(topic, new TopicPublishInfo());
+                producer.getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);
+                topicPublishInfo = producer.getTopicPublishInfoTable().get(topic);
+            }
+            if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) {
+                for (MessageQueue queue : topicPublishInfo.getMessageQueueList()) {
+                    brokerSet.add(queue.getBrokerName());
+                }
+            }
+            return brokerSet;
+        }
+    }
+
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java

@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.MessageType;
+
+public class TraceBean {
+    private static final String LOCAL_ADDRESS = UtilAll.ipToIPv4Str(UtilAll.getIP());
+    private String topic = "";
+    private String msgId = "";
+    private String offsetMsgId = "";
+    private String tags = "";
+    private String keys = "";
+    private String storeHost = LOCAL_ADDRESS;
+    private String clientHost = LOCAL_ADDRESS;
+    private long storeTime;
+    private int retryTimes;
+    private int bodyLength;
+    private MessageType msgType;
+
+
+    public MessageType getMsgType() {
+        return msgType;
+    }
+
+
+    public void setMsgType(final MessageType msgType) {
+        this.msgType = msgType;
+    }
+
+
+    public String getOffsetMsgId() {
+        return offsetMsgId;
+    }
+
+
+    public void setOffsetMsgId(final String offsetMsgId) {
+        this.offsetMsgId = offsetMsgId;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+
+    public String getMsgId() {
+        return msgId;
+    }
+
+
+    public void setMsgId(String msgId) {
+        this.msgId = msgId;
+    }
+
+
+    public String getTags() {
+        return tags;
+    }
+
+
+    public void setTags(String tags) {
+        this.tags = tags;
+    }
+
+
+    public String getKeys() {
+        return keys;
+    }
+
+
+    public void setKeys(String keys) {
+        this.keys = keys;
+    }
+
+
+    public String getStoreHost() {
+        return storeHost;
+    }
+
+
+    public void setStoreHost(String storeHost) {
+        this.storeHost = storeHost;
+    }
+
+
+    public String getClientHost() {
+        return clientHost;
+    }
+
+
+    public void setClientHost(String clientHost) {
+        this.clientHost = clientHost;
+    }
+
+
+    public long getStoreTime() {
+        return storeTime;
+    }
+
+
+    public void setStoreTime(long storeTime) {
+        this.storeTime = storeTime;
+    }
+
+
+    public int getRetryTimes() {
+        return retryTimes;
+    }
+
+
+    public void setRetryTimes(int retryTimes) {
+        this.retryTimes = retryTimes;
+    }
+
+
+    public int getBodyLength() {
+        return bodyLength;
+    }
+
+
+    public void setBodyLength(int bodyLength) {
+        this.bodyLength = bodyLength;
+    }
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+public class TraceConstants {
+
+    public static final String GROUP_NAME = "_INNER_TRACE_PRODUCER";
+    public static final char CONTENT_SPLITOR = (char) 1;
+    public static final char FIELD_SPLITOR = (char) 2;
+    public static final String TRACE_INSTANCE_NAME = "PID_CLIENT_INNER_TRACE_PRODUCER";
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceContext.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
+
+import java.util.List;
+
+/**
+ * The context of Trace
+ */
+public class TraceContext implements Comparable<TraceContext> {
+
+    private TraceType traceType;
+    private long timeStamp = System.currentTimeMillis();
+    private String regionId = "";
+    private String regionName = "";
+    private String groupName = "";
+    private int costTime = 0;
+    private boolean isSuccess = true;
+    private String requestId = MessageClientIDSetter.createUniqID();
+    private int contextCode = 0;
+    private List<TraceBean> traceBeans;
+
+    public int getContextCode() {
+        return contextCode;
+    }
+
+    public void setContextCode(final int contextCode) {
+        this.contextCode = contextCode;
+    }
+
+    public List<TraceBean> getTraceBeans() {
+        return traceBeans;
+    }
+
+    public void setTraceBeans(List<TraceBean> traceBeans) {
+        this.traceBeans = traceBeans;
+    }
+
+    public String getRegionId() {
+        return regionId;
+    }
+
+    public void setRegionId(String regionId) {
+        this.regionId = regionId;
+    }
+
+    public TraceType getTraceType() {
+        return traceType;
+    }
+
+    public void setTraceType(TraceType traceType) {
+        this.traceType = traceType;
+    }
+
+    public long getTimeStamp() {
+        return timeStamp;
+    }
+
+    public void setTimeStamp(long timeStamp) {
+        this.timeStamp = timeStamp;
+    }
+
+    public String getGroupName() {
+        return groupName;
+    }
+
+    public void setGroupName(String groupName) {
+        this.groupName = groupName;
+    }
+
+    public int getCostTime() {
+        return costTime;
+    }
+
+    public void setCostTime(int costTime) {
+        this.costTime = costTime;
+    }
+
+    public boolean isSuccess() {
+        return isSuccess;
+    }
+
+    public void setSuccess(boolean success) {
+        isSuccess = success;
+    }
+
+    public String getRequestId() {
+        return requestId;
+    }
+
+    public void setRequestId(String requestId) {
+        this.requestId = requestId;
+    }
+
+    public String getRegionName() {
+        return regionName;
+    }
+
+    public void setRegionName(String regionName) {
+        this.regionName = regionName;
+    }
+
+    @Override
+    public int compareTo(TraceContext o) {
+        return (int) (this.timeStamp - o.getTimeStamp());
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder(1024);
+        sb.append(traceType).append("_").append(groupName)
+            .append("_").append(regionId).append("_").append(isSuccess).append("_");
+        if (traceBeans != null && traceBeans.size() > 0) {
+            for (TraceBean bean : traceBeans) {
+                sb.append(bean.getMsgId() + "_" + bean.getTopic() + "_");
+            }
+        }
+        return "TraceContext{" + sb.toString() + '}';
+    }
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java

@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+import org.apache.rocketmq.common.message.MessageType;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encode/decode for Trace Data
+ */
+public class TraceDataEncoder {
+
+    /**
+     * Resolving traceContext list From trace data String
+     *
+     * @param traceData
+     * @return
+     */
+    public static List<TraceContext> decoderFromTraceDataString(String traceData) {
+        List<TraceContext> resList = new ArrayList<TraceContext>();
+        if (traceData == null || traceData.length() <= 0) {
+            return resList;
+        }
+        String[] contextList = traceData.split(String.valueOf(TraceConstants.FIELD_SPLITOR));
+        for (String context : contextList) {
+            String[] line = context.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+            if (line[0].equals(TraceType.Pub.name())) {
+                TraceContext pubContext = new TraceContext();
+                pubContext.setTraceType(TraceType.Pub);
+                pubContext.setTimeStamp(Long.parseLong(line[1]));
+                pubContext.setRegionId(line[2]);
+                pubContext.setGroupName(line[3]);
+                TraceBean bean = new TraceBean();
+                bean.setTopic(line[4]);
+                bean.setMsgId(line[5]);
+                bean.setTags(line[6]);
+                bean.setKeys(line[7]);
+                bean.setStoreHost(line[8]);
+                bean.setBodyLength(Integer.parseInt(line[9]));
+                pubContext.setCostTime(Integer.parseInt(line[10]));
+                bean.setMsgType(MessageType.values()[Integer.parseInt(line[11])]);
+
+                if (line.length == 13) {
+                    pubContext.setSuccess(Boolean.parseBoolean(line[12]));
+                } else if (line.length == 14) {
+                    bean.setOffsetMsgId(line[12]);
+                    pubContext.setSuccess(Boolean.parseBoolean(line[13]));
+                }
+                pubContext.setTraceBeans(new ArrayList<TraceBean>(1));
+                pubContext.getTraceBeans().add(bean);
+                resList.add(pubContext);
+            } else if (line[0].equals(TraceType.SubBefore.name())) {
+                TraceContext subBeforeContext = new TraceContext();
+                subBeforeContext.setTraceType(TraceType.SubBefore);
+                subBeforeContext.setTimeStamp(Long.parseLong(line[1]));
+                subBeforeContext.setRegionId(line[2]);
+                subBeforeContext.setGroupName(line[3]);
+                subBeforeContext.setRequestId(line[4]);
+                TraceBean bean = new TraceBean();
+                bean.setMsgId(line[5]);
+                bean.setRetryTimes(Integer.parseInt(line[6]));
+                bean.setKeys(line[7]);
+                subBeforeContext.setTraceBeans(new ArrayList<TraceBean>(1));
+                subBeforeContext.getTraceBeans().add(bean);
+                resList.add(subBeforeContext);
+            } else if (line[0].equals(TraceType.SubAfter.name())) {
+                TraceContext subAfterContext = new TraceContext();
+                subAfterContext.setTraceType(TraceType.SubAfter);
+                subAfterContext.setRequestId(line[1]);
+                TraceBean bean = new TraceBean();
+                bean.setMsgId(line[2]);
+                bean.setKeys(line[5]);
+                subAfterContext.setTraceBeans(new ArrayList<TraceBean>(1));
+                subAfterContext.getTraceBeans().add(bean);
+                subAfterContext.setCostTime(Integer.parseInt(line[3]));
+                subAfterContext.setSuccess(Boolean.parseBoolean(line[4]));
+                if (line.length >= 7) {
+                    // add the context type
+                    subAfterContext.setContextCode(Integer.parseInt(line[6]));
+                }
+                resList.add(subAfterContext);
+            }
+        }
+        return resList;
+    }
+
+    /**
+     * Encoding the trace context into data strings and keyset sets
+     *
+     * @param ctx
+     * @return
+     */
+    public static TraceTransferBean encoderFromContextBean(TraceContext ctx) {
+        if (ctx == null) {
+            return null;
+        }
+        //build message trace of the transfering entity content bean
+        TraceTransferBean transferBean = new TraceTransferBean();
+        StringBuilder sb = new StringBuilder(256);
+        switch (ctx.getTraceType()) {
+            case Pub: {
+                TraceBean bean = ctx.getTraceBeans().get(0);
+                //append the content of context and traceBean to transferBean's TransData
+                sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getTags()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getBodyLength()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(bean.getOffsetMsgId()).append(TraceConstants.CONTENT_SPLITOR)//
+                    .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);
+            }
+            break;
+            case SubBefore: {
+                for (TraceBean bean : ctx.getTraceBeans()) {
+                    sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(bean.getRetryTimes()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(bean.getKeys()).append(TraceConstants.FIELD_SPLITOR);//
+                }
+            }
+            break;
+            case SubAfter: {
+                for (TraceBean bean : ctx.getTraceBeans()) {
+                    sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//
+                        .append(ctx.getContextCode()).append(TraceConstants.FIELD_SPLITOR);
+                }
+            }
+            break;
+            default:
+        }
+        transferBean.setTransData(sb.toString());
+        for (TraceBean bean : ctx.getTraceBeans()) {
+
+            transferBean.getTransKey().add(bean.getMsgId());
+            if (bean.getKeys() != null && bean.getKeys().length() > 0) {
+                transferBean.getTransKey().add(bean.getKeys());
+            }
+        }
+        return transferBean;
+    }
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+import org.apache.rocketmq.client.exception.MQClientException;
+import java.io.IOException;
+
+/**
+ * Interface of asynchronous transfer data
+ */
+public interface TraceDispatcher {
+
+    /**
+     * Initialize asynchronous transfer data module
+     */
+    void start(String nameSrvAddr) throws MQClientException;
+
+    /**
+     * Append the transfering data
+     * @param ctx data infomation
+     * @return
+     */
+    boolean append(Object ctx);
+
+    /**
+     * Write flush action
+     *
+     * @throws IOException
+     */
+    void flush() throws IOException;
+
+    /**
+     * Close the trace Hook
+     */
+    void shutdown();
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcherType.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+public enum TraceDispatcherType {
+    PRODUCER,
+    CONSUMER
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceTransferBean.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Trace transfering bean
+ */
+public class TraceTransferBean {
+    private String transData;
+    private Set<String> transKey = new HashSet<String>();
+
+    public String getTransData() {
+        return transData;
+    }
+
+    public void setTransData(String transData) {
+        this.transData = transData;
+    }
+
+    public Set<String> getTransKey() {
+        return transKey;
+    }
+
+    public void setTransKey(Set<String> transKey) {
+        this.transKey = transKey;
+    }
+}

client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace;
+
+public enum TraceType {
+    Pub,
+    SubBefore,
+    SubAfter,
+}

client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace.hook;
+
+import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType;
+import org.apache.rocketmq.client.hook.ConsumeMessageContext;
+import org.apache.rocketmq.client.hook.ConsumeMessageHook;
+import org.apache.rocketmq.client.trace.TraceContext;
+import org.apache.rocketmq.client.trace.TraceDispatcher;
+import org.apache.rocketmq.client.trace.TraceBean;
+import org.apache.rocketmq.client.trace.TraceType;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook {
+
+    private TraceDispatcher localDispatcher;
+
+    public ConsumeMessageTraceHookImpl(TraceDispatcher localDispatcher) {
+        this.localDispatcher = localDispatcher;
+    }
+
+    @Override
+    public String hookName() {
+        return "ConsumeMessageTraceHook";
+    }
+
+    @Override
+    public void consumeMessageBefore(ConsumeMessageContext context) {
+        if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) {
+            return;
+        }
+        TraceContext traceContext = new TraceContext();
+        context.setMqTraceContext(traceContext);
+        traceContext.setTraceType(TraceType.SubBefore);//
+        traceContext.setGroupName(context.getConsumerGroup());//
+        List<TraceBean> beans = new ArrayList<TraceBean>();
+        for (MessageExt msg : context.getMsgList()) {
+            if (msg == null) {
+                continue;
+            }
+            String regionId = msg.getProperty(MessageConst.PROPERTY_MSG_REGION);
+            String traceOn = msg.getProperty(MessageConst.PROPERTY_TRACE_SWITCH);
+
+            if (traceOn != null && traceOn.equals("false")) {
+                // If trace switch is false ,skip it
+                continue;
+            }
+            TraceBean traceBean = new TraceBean();
+            traceBean.setTopic(msg.getTopic());//
+            traceBean.setMsgId(msg.getMsgId());//
+            traceBean.setTags(msg.getTags());//
+            traceBean.setKeys(msg.getKeys());//
+            traceBean.setStoreTime(msg.getStoreTimestamp());//
+            traceBean.setBodyLength(msg.getStoreSize());//
+            traceBean.setRetryTimes(msg.getReconsumeTimes());//
+            traceContext.setRegionId(regionId);//
+            beans.add(traceBean);
+        }
+        if (beans.size() > 0) {
+            traceContext.setTraceBeans(beans);
+            traceContext.setTimeStamp(System.currentTimeMillis());
+            localDispatcher.append(traceContext);
+        }
+    }
+
+    @Override
+    public void consumeMessageAfter(ConsumeMessageContext context) {
+        if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) {
+            return;
+        }
+        TraceContext subBeforeContext = (TraceContext) context.getMqTraceContext();
+
+        if (subBeforeContext.getTraceBeans() == null || subBeforeContext.getTraceBeans().size() < 1) {
+            // If subbefore bean is null ,skip it
+            return;
+        }
+        TraceContext subAfterContext = new TraceContext();
+        subAfterContext.setTraceType(TraceType.SubAfter);//
+        subAfterContext.setRegionId(subBeforeContext.getRegionId());//
+        subAfterContext.setGroupName(subBeforeContext.getGroupName());//
+        subAfterContext.setRequestId(subBeforeContext.getRequestId());//
+        subAfterContext.setSuccess(context.isSuccess());//
+
+        // Caculate the cost time for processing messages
+        int costTime = (int) ((System.currentTimeMillis() - subBeforeContext.getTimeStamp()) / context.getMsgList().size());
+        subAfterContext.setCostTime(costTime);//
+        subAfterContext.setTraceBeans(subBeforeContext.getTraceBeans());
+        String contextType = context.getProps().get(MixAll.CONSUME_CONTEXT_TYPE);
+        if (contextType != null) {
+            subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal());
+        }
+        localDispatcher.append(subAfterContext);
+    }
+}

client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.trace.hook;
+
+import org.apache.rocketmq.client.hook.SendMessageContext;
+import org.apache.rocketmq.client.hook.SendMessageHook;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.client.trace.AsyncTraceDispatcher;
+import org.apache.rocketmq.client.trace.TraceContext;
+import org.apache.rocketmq.client.trace.TraceDispatcher;
+import org.apache.rocketmq.client.trace.TraceBean;
+import org.apache.rocketmq.client.trace.TraceType;
+import java.util.ArrayList;
+
+public class SendMessageTraceHookImpl implements SendMessageHook {
+
+    private TraceDispatcher localDispatcher;
+
+    public SendMessageTraceHookImpl(TraceDispatcher localDispatcher) {
+        this.localDispatcher = localDispatcher;
+    }
+
+    @Override
+    public String hookName() {
+        return "SendMessageTraceHook";
+    }
+
+    @Override
+    public void sendMessageBefore(SendMessageContext context) {
+        //if it is message trace data,then it doesn't recorded
+        if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())) {
+            return;
+        }
+        //build the context content of TuxeTraceContext
+        TraceContext tuxeContext = new TraceContext();
+        tuxeContext.setTraceBeans(new ArrayList<TraceBean>(1));
+        context.setMqTraceContext(tuxeContext);
+        tuxeContext.setTraceType(TraceType.Pub);
+        tuxeContext.setGroupName(context.getProducerGroup());
+        //build the data bean object of message trace
+        TraceBean traceBean = new TraceBean();
+        traceBean.setTopic(context.getMessage().getTopic());
+        traceBean.setTags(context.getMessage().getTags());
+        traceBean.setKeys(context.getMessage().getKeys());
+        traceBean.setStoreHost(context.getBrokerAddr());
+        traceBean.setBodyLength(context.getMessage().getBody().length);
+        traceBean.setMsgType(context.getMsgType());
+        tuxeContext.getTraceBeans().add(traceBean);
+    }
+
+    @Override
+    public void sendMessageAfter(SendMessageContext context) {
+        //if it is message trace data,then it doesn't recorded
+        if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())
+            || context.getMqTraceContext() == null) {
+            return;
+        }
+        if (context.getSendResult() == null) {
+            return;
+        }
+
+        if (context.getSendResult().getRegionId() == null
+            || !context.getSendResult().isTraceOn()) {
+            // if switch is false,skip it
+            return;
+        }
+
+        TraceContext tuxeContext = (TraceContext) context.getMqTraceContext();
+        TraceBean traceBean = tuxeContext.getTraceBeans().get(0);
+        int costTime = (int) ((System.currentTimeMillis() - tuxeContext.getTimeStamp()) / tuxeContext.getTraceBeans().size());
+        tuxeContext.setCostTime(costTime);
+        if (context.getSendResult().getSendStatus().equals(SendStatus.SEND_OK)) {
+            tuxeContext.setSuccess(true);
+        } else {
+            tuxeContext.setSuccess(false);
+        }
+        tuxeContext.setRegionId(context.getSendResult().getRegionId());
+        traceBean.setMsgId(context.getSendResult().getMsgId());
+        traceBean.setOffsetMsgId(context.getSendResult().getOffsetMsgId());
+        traceBean.setStoreTime(tuxeContext.getTimeStamp() + costTime / 2);
+        localDispatcher.append(tuxeContext);
+    }
+}

client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java

@@ -169,10 +169,7 @@ public class DefaultMQProducerTest {
 
     @Test
     public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
-        ExecutorService callbackExecutor = Executors.newSingleThreadExecutor();
         final CountDownLatch countDownLatch = new CountDownLatch(1);
-        when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient));
-        when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor);
         producer.send(message, new SendCallback() {
             @Override
             public void onSuccess(SendResult sendResult) {
@@ -188,16 +185,12 @@ public class DefaultMQProducerTest {
             }
         });
         countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
-        callbackExecutor.shutdown();
     }
 
     @Test
     public void testSendMessageAsync() throws RemotingException, MQClientException, InterruptedException {
         final AtomicInteger cc = new AtomicInteger(0);
         final CountDownLatch countDownLatch = new CountDownLatch(6);
-        ExecutorService callbackExecutor = Executors.newSingleThreadExecutor();
-        when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient));
-        when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor);
 
         SendCallback sendCallback = new SendCallback() {
             @Override
@@ -229,16 +222,13 @@ public class DefaultMQProducerTest {
         producer.send(message, messageQueueSelector, null, sendCallback, 1000);
 
         countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
-        callbackExecutor.shutdown();
         assertThat(cc.get()).isEqualTo(6);
     }
 
     @Test
     public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
-        ExecutorService callbackExecutor = Executors.newSingleThreadExecutor();
+
         final CountDownLatch countDownLatch = new CountDownLatch(1);
-        when(mQClientAPIImpl.getRemotingClient()).thenReturn((nettyRemotingClient));
-        when(nettyRemotingClient.getCallbackExecutor()).thenReturn(callbackExecutor);
         producer.send(bigMessage, new SendCallback() {
             @Override
             public void onSuccess(SendResult sendResult) {
@@ -254,7 +244,6 @@ public class DefaultMQProducerTest {
             }
         });
         countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
-        callbackExecutor.shutdown();
     }
 
     @Test

client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java

@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.client.trace;
+
+import java.io.ByteArrayOutputStream;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.PullCallback;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.CommunicationMode;
+import org.apache.rocketmq.client.impl.FindBrokerResult;
+import org.apache.rocketmq.client.impl.MQClientAPIImpl;
+import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService;
+import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;
+import org.apache.rocketmq.client.impl.consumer.ProcessQueue;
+import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;
+import org.apache.rocketmq.client.impl.consumer.PullMessageService;
+import org.apache.rocketmq.client.impl.consumer.PullRequest;
+import org.apache.rocketmq.client.impl.consumer.PullResultExt;
+import org.apache.rocketmq.client.impl.consumer.RebalancePushImpl;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageClientExt;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.QueueData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DefaultMQConsumerWithTraceTest {
+    private String consumerGroup;
+    private String consumerGroupNormal;
+    private String producerGroupTraceTemp = MixAll.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();
+
+    private String topic = "FooBar";
+    private String brokerName = "BrokerA";
+    private MQClientInstance mQClientFactory;
+
+    @Mock
+    private MQClientAPIImpl mQClientAPIImpl;
+    private PullAPIWrapper pullAPIWrapper;
+    private RebalancePushImpl rebalancePushImpl;
+    private DefaultMQPushConsumer pushConsumer;
+    private DefaultMQPushConsumer normalPushConsumer;
+    private DefaultMQPushConsumer customTraceTopicpushConsumer;
+
+
+    private AsyncTraceDispatcher asyncTraceDispatcher;
+    private MQClientInstance mQClientTraceFactory;
+    @Mock
+    private MQClientAPIImpl mQClientTraceAPIImpl;
+    private DefaultMQProducer traceProducer;
+    private String customerTraceTopic = "rmq_trace_topic_12345";
+
+    @Before
+    public void init() throws Exception {
+        consumerGroup = "FooBarGroup" + System.currentTimeMillis();
+        pushConsumer = new DefaultMQPushConsumer(consumerGroup,true,"");
+        consumerGroupNormal = "FooBarGroup" + System.currentTimeMillis();
+        normalPushConsumer = new DefaultMQPushConsumer(consumerGroupNormal,false,"");
+        customTraceTopicpushConsumer = new DefaultMQPushConsumer(consumerGroup,true,customerTraceTopic);
+        pushConsumer.setNamesrvAddr("127.0.0.1:9876");
+        pushConsumer.setPullInterval(60 * 1000);
+
+        asyncTraceDispatcher = (AsyncTraceDispatcher)pushConsumer.getTraceDispatcher();
+        traceProducer = asyncTraceDispatcher.getTraceProducer();
+
+
+        pushConsumer.registerMessageListener(new MessageListenerConcurrently() {
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
+                ConsumeConcurrentlyContext context) {
+                return null;
+            }
+        });
+
+        DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();
+        rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl()));
+        Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl");
+        field.setAccessible(true);
+        field.set(pushConsumerImpl, rebalancePushImpl);
+        pushConsumer.subscribe(topic, "*");
+        pushConsumer.start();
+
+        mQClientFactory = spy(pushConsumerImpl.getmQClientFactory());
+        mQClientTraceFactory = spy(pushConsumerImpl.getmQClientFactory());
+
+        field = DefaultMQPushConsumerImpl.class.getDeclaredField("mQClientFactory");
+        field.setAccessible(true);
+        field.set(pushConsumerImpl, mQClientFactory);
+
+        field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
+        field.setAccessible(true);
+        field.set(mQClientFactory, mQClientAPIImpl);
+
+        Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory");
+        fieldTrace.setAccessible(true);
+        fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientTraceFactory);
+
+        fieldTrace = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
+        fieldTrace.setAccessible(true);
+        fieldTrace.set(mQClientTraceFactory, mQClientTraceAPIImpl);
+
+        pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false));
+        field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper");
+        field.setAccessible(true);
+        field.set(pushConsumerImpl, pullAPIWrapper);
+
+        pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory);
+        mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl);
+
+        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),
+            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))
+            .thenAnswer(new Answer<Object>() {
+                @Override
+                public Object answer(InvocationOnMock mock) throws Throwable {
+                    PullMessageRequestHeader requestHeader = mock.getArgument(1);
+                    MessageClientExt messageClientExt = new MessageClientExt();
+                    messageClientExt.setTopic(topic);
+                    messageClientExt.setQueueId(0);
+                    messageClientExt.setMsgId("123");
+                    messageClientExt.setBody(new byte[] {'a'});
+                    messageClientExt.setOffsetMsgId("234");
+                    messageClientExt.setBornHost(new InetSocketAddress(8080));
+                    messageClientExt.setStoreHost(new InetSocketAddress(8080));
+                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));
+                    ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);
+                    return pullResult;
+                }
+            });
+
+        doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());
+        Set<MessageQueue> messageQueueSet = new HashSet<MessageQueue>();
+        messageQueueSet.add(createPullRequest().getMessageQueue());
+        pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet);
+    }
+
+    @After
+    public void terminate() {
+        pushConsumer.shutdown();
+    }
+
+//    @Test
+    public void testPullMessage_WithTrace_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException {
+        traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());
+
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        final MessageExt[] messageExts = new MessageExt[1];
+        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
+                ConsumeConcurrentlyContext context) {
+                messageExts[0] = msgs.get(0);
+                countDownLatch.countDown();
+                return null;
+            }
+        }));
+
+        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();
+        pullMessageService.executePullRequestImmediately(createPullRequest());
+        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        assertThat(messageExts[0].getTopic()).isEqualTo(topic);
+        assertThat(messageExts[0].getBody()).isEqualTo(new byte[] {'a'});
+    }
+
+    private PullRequest createPullRequest() {
+        PullRequest pullRequest = new PullRequest();
+        pullRequest.setConsumerGroup(consumerGroup);
+        pullRequest.setNextOffset(1024);
+
+        MessageQueue messageQueue = new MessageQueue();
+        messageQueue.setBrokerName(brokerName);
+        messageQueue.setQueueId(0);
+        messageQueue.setTopic(topic);
+        pullRequest.setMessageQueue(messageQueue);
+        ProcessQueue processQueue = new ProcessQueue();
+        processQueue.setLocked(true);
+        processQueue.setLastLockTimestamp(System.currentTimeMillis());
+        pullRequest.setProcessQueue(processQueue);
+
+        return pullRequest;
+    }
+
+    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,
+        List<MessageExt> messageExtList) throws Exception {
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        for (MessageExt messageExt : messageExtList) {
+            outputStream.write(MessageDecoder.encode(messageExt, false));
+        }
+        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());
+    }
+
+    public static TopicRouteData createTopicRoute() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+
+        topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
+        List<BrokerData> brokerDataList = new ArrayList<BrokerData>();
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName("BrokerA");
+        brokerData.setCluster("DefaultCluster");
+        HashMap<Long, String> brokerAddrs = new HashMap<Long, String>();
+        brokerAddrs.put(0L, "127.0.0.1:10911");
+        brokerData.setBrokerAddrs(brokerAddrs);
+        brokerDataList.add(brokerData);
+        topicRouteData.setBrokerDatas(brokerDataList);
+
+        List<QueueData> queueDataList = new ArrayList<QueueData>();
+        QueueData queueData = new QueueData();
+        queueData.setBrokerName("BrokerA");
+        queueData.setPerm(6);
+        queueData.setReadQueueNums(3);
+        queueData.setWriteQueueNums(4);
+        queueData.setTopicSynFlag(0);
+        queueDataList.add(queueData);
+        topicRouteData.setQueueDatas(queueDataList);
+        return topicRouteData;
+    }
+
+    private SendResult createSendResult(SendStatus sendStatus) {
+        SendResult sendResult = new SendResult();
+        sendResult.setMsgId("123");
+        sendResult.setOffsetMsgId("123");
+        sendResult.setQueueOffset(456);
+        sendResult.setSendStatus(sendStatus);
+        sendResult.setRegionId("HZ");
+        return sendResult;
+    }
+
+    public static TopicRouteData createTraceTopicRoute() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+
+        topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
+        List<BrokerData> brokerDataList = new ArrayList<BrokerData>();
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName("broker-trace");
+        brokerData.setCluster("DefaultCluster");
+        HashMap<Long, String> brokerAddrs = new HashMap<Long, String>();
+        brokerAddrs.put(0L, "127.0.0.1:10912");
+        brokerData.setBrokerAddrs(brokerAddrs);
+        brokerDataList.add(brokerData);
+        topicRouteData.setBrokerDatas(brokerDataList);
+
+        List<QueueData> queueDataList = new ArrayList<QueueData>();
+        QueueData queueData = new QueueData();
+        queueData.setBrokerName("broker-trace");
+        queueData.setPerm(6);
+        queueData.setReadQueueNums(1);
+        queueData.setWriteQueueNums(1);
+        queueData.setTopicSynFlag(1);
+        queueDataList.add(queueData);
+        topicRouteData.setQueueDatas(queueDataList);
+        return topicRouteData;
+    }
+}

client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java

@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.client.trace;
+
+import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.hook.SendMessageContext;
+import org.apache.rocketmq.client.impl.CommunicationMode;
+import org.apache.rocketmq.client.impl.MQClientAPIImpl;
+import org.apache.rocketmq.client.impl.MQClientManager;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendCallback;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.QueueData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DefaultMQProducerWithTraceTest {
+
+    @Spy
+    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig());
+    @Mock
+    private MQClientAPIImpl mQClientAPIImpl;
+
+    private AsyncTraceDispatcher asyncTraceDispatcher;
+
+    private DefaultMQProducer producer;
+    private DefaultMQProducer customTraceTopicproducer;
+    private DefaultMQProducer traceProducer;
+    private DefaultMQProducer normalProducer;
+
+    private Message message;
+    private String topic = "FooBar";
+    private String producerGroupPrefix = "FooBar_PID";
+    private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();
+    private String producerGroupTraceTemp = MixAll.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();
+    private String customerTraceTopic = "rmq_trace_topic_12345";
+
+    @Before
+    public void init() throws Exception {
+
+        customTraceTopicproducer = new DefaultMQProducer(producerGroupTemp, false, customerTraceTopic);
+        normalProducer = new DefaultMQProducer(producerGroupTemp, false, "");
+        producer = new DefaultMQProducer(producerGroupTemp, true, "");
+        producer.setNamesrvAddr("127.0.0.1:9876");
+        normalProducer.setNamesrvAddr("127.0.0.1:9877");
+        customTraceTopicproducer.setNamesrvAddr("127.0.0.1:9878");
+        message = new Message(topic, new byte[]{'a', 'b', 'c'});
+        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
+        asyncTraceDispatcher.setTraceTopicName(customerTraceTopic);
+        asyncTraceDispatcher.getHostProducer();
+        asyncTraceDispatcher.getHostConsumer();
+        traceProducer = asyncTraceDispatcher.getTraceProducer();
+
+        producer.start();
+
+        Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory");
+        field.setAccessible(true);
+        field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);
+
+        Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory");
+        fieldTrace.setAccessible(true);
+        fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory);
+
+        field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
+        field.setAccessible(true);
+        field.set(mQClientFactory, mQClientAPIImpl);
+
+
+        producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());
+
+        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),
+                nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();
+        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),
+                nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
+                .thenReturn(createSendResult(SendStatus.SEND_OK));
+
+    }
+
+    @Test
+    public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
+        traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());
+        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        try {
+            producer.send(message);
+        } catch (MQClientException e) {
+        }
+        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+
+    }
+
+    @Test
+    public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
+        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        try {
+            producer.send(message);
+        } catch (MQClientException e) {
+        }
+        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+
+    }
+
+    @After
+    public void terminate() {
+        producer.shutdown();
+    }
+
+    public static TopicRouteData createTopicRoute() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+
+        topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
+        List<BrokerData> brokerDataList = new ArrayList<BrokerData>();
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName("BrokerA");
+        brokerData.setCluster("DefaultCluster");
+        HashMap<Long, String> brokerAddrs = new HashMap<Long, String>();
+        brokerAddrs.put(0L, "127.0.0.1:10911");
+        brokerData.setBrokerAddrs(brokerAddrs);
+        brokerDataList.add(brokerData);
+        topicRouteData.setBrokerDatas(brokerDataList);
+
+        List<QueueData> queueDataList = new ArrayList<QueueData>();
+        QueueData queueData = new QueueData();
+        queueData.setBrokerName("BrokerA");
+        queueData.setPerm(6);
+        queueData.setReadQueueNums(3);
+        queueData.setWriteQueueNums(4);
+        queueData.setTopicSynFlag(0);
+        queueDataList.add(queueData);
+        topicRouteData.setQueueDatas(queueDataList);
+        return topicRouteData;
+    }
+
+    private SendResult createSendResult(SendStatus sendStatus) {
+        SendResult sendResult = new SendResult();
+        sendResult.setMsgId("123");
+        sendResult.setOffsetMsgId("123");
+        sendResult.setQueueOffset(456);
+        sendResult.setSendStatus(sendStatus);
+        sendResult.setRegionId("HZ");
+        return sendResult;
+    }
+
+    public static TopicRouteData createTraceTopicRoute() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+
+        topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
+        List<BrokerData> brokerDataList = new ArrayList<BrokerData>();
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName("broker-trace");
+        brokerData.setCluster("DefaultCluster");
+        HashMap<Long, String> brokerAddrs = new HashMap<Long, String>();
+        brokerAddrs.put(0L, "127.0.0.1:10912");
+        brokerData.setBrokerAddrs(brokerAddrs);
+        brokerDataList.add(brokerData);
+        topicRouteData.setBrokerDatas(brokerDataList);
+
+        List<QueueData> queueDataList = new ArrayList<QueueData>();
+        QueueData queueData = new QueueData();
+        queueData.setBrokerName("broker-trace");
+        queueData.setPerm(6);
+        queueData.setReadQueueNums(1);
+        queueData.setWriteQueueNums(1);
+        queueData.setTopicSynFlag(1);
+        queueDataList.add(queueData);
+        topicRouteData.setQueueDatas(queueDataList);
+        return topicRouteData;
+    }
+}

common/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java

@@ -25,6 +25,8 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingUtil;
 
+import static org.apache.rocketmq.common.SnodeConfig.localHostName;
+
 public class BrokerConfig {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
 
@@ -51,7 +53,10 @@ public class BrokerConfig {
     @ImportantField
     private boolean autoCreateSubscriptionGroup = true;
     private String messageStorePlugIn = "";
-
+    @ImportantField
+    private String msgTraceTopicName = MixAll.RMQ_SYS_TRACE_TOPIC;
+    @ImportantField
+    private boolean traceTopicEnable = false;
     /**
      * thread numbers for send message thread pool, since spin lock will be used by default since 4.0.x, the default
      * value is 1.
@@ -171,7 +176,6 @@ public class BrokerConfig {
     @ImportantField
     private long transactionCheckInterval = 60 * 1000;
 
-
     @ImportantField
     private boolean transactionEnable = true;
 
@@ -182,6 +186,21 @@ public class BrokerConfig {
     public void setTransactionEnable(boolean transactionEnable) {
         this.transactionEnable = transactionEnable;
     }
+    /**
+     * Acl feature switch
+     */
+    @ImportantField
+    private boolean aclEnable = false;
+
+    public static String localHostName() {
+        try {
+            return InetAddress.getLocalHost().getHostName();
+        } catch (UnknownHostException e) {
+            log.error("Failed to obtain the host name", e);
+        }
+
+        return "DEFAULT_BROKER";
+    }
 
     public boolean isTraceOn() {
         return traceOn;
@@ -247,16 +266,6 @@ public class BrokerConfig {
         this.slaveReadEnable = slaveReadEnable;
     }
 
-    public static String localHostName() {
-        try {
-            return InetAddress.getLocalHost().getHostName();
-        } catch (UnknownHostException e) {
-            log.error("Failed to obtain the host name", e);
-        }
-
-        return "DEFAULT_BROKER";
-    }
-
     public int getRegisterBrokerTimeoutMills() {
         return registerBrokerTimeoutMills;
     }
@@ -744,4 +753,28 @@ public class BrokerConfig {
     public void setWaitTimeMillsInTransactionQueue(long waitTimeMillsInTransactionQueue) {
         this.waitTimeMillsInTransactionQueue = waitTimeMillsInTransactionQueue;
     }
+
+    public String getMsgTraceTopicName() {
+        return msgTraceTopicName;
+    }
+
+    public void setMsgTraceTopicName(String msgTraceTopicName) {
+        this.msgTraceTopicName = msgTraceTopicName;
+    }
+    
+    public boolean isTraceTopicEnable() {
+        return traceTopicEnable;
+    }
+
+    public void setTraceTopicEnable(boolean traceTopicEnable) {
+        this.traceTopicEnable = traceTopicEnable;
+    }
+
+    public boolean isAclEnable() {
+        return aclEnable;
+    }
+
+    public void setAclEnable(boolean aclEnable) {
+        this.aclEnable = aclEnable;
+    }
 }

common/src/main/java/org/apache/rocketmq/common/MQVersion.java

@@ -18,7 +18,7 @@ package org.apache.rocketmq.common;
 
 public class MQVersion {
 
-    public static final int CURRENT_VERSION = Version.V4_3_1.ordinal();
+    public static final int CURRENT_VERSION = Version.V4_4_0.ordinal();
 
     public static String getVersionDesc(int value) {
         int length = Version.values().length;

common/src/main/java/org/apache/rocketmq/common/MixAll.java

@@ -90,6 +90,7 @@ public class MixAll {
     public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType";
 
     public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC";
+    public static final String RMQ_SYS_TRACE_TOPIC = "RMQ_SYS_TRACE_TOPIC";
     public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC";
     public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS";
 

common/src/main/java/org/apache/rocketmq/common/ServiceThread.java

@@ -27,18 +27,29 @@ public abstract class ServiceThread implements Runnable {
 
     private static final long JOIN_TIME = 90 * 1000;
 
-    protected final Thread thread;
+    private Thread thread;
     protected final CountDownLatch2 waitPoint = new CountDownLatch2(1);
     protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false);
     protected volatile boolean stopped = false;
+    protected boolean isDaemon = false;
+
+    //Make it able to restart the thread
+    private final AtomicBoolean started = new AtomicBoolean(false);
 
     public ServiceThread() {
-        this.thread = new Thread(this, this.getServiceName());
+
     }
 
     public abstract String getServiceName();
 
     public void start() {
+        log.info("Try to start service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread);
+        if (!started.compareAndSet(false, true)) {
+            return;
+        }
+        stopped = false;
+        this.thread = new Thread(this, getServiceName());
+        this.thread.setDaemon(isDaemon);
         this.thread.start();
     }
 
@@ -47,6 +58,10 @@ public abstract class ServiceThread implements Runnable {
     }
 
     public void shutdown(final boolean interrupt) {
+        log.info("Try to shutdown service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread);
+        if (!started.compareAndSet(true, false)) {
+            return;
+        }
         this.stopped = true;
         log.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt);
 
@@ -75,11 +90,16 @@ public abstract class ServiceThread implements Runnable {
         return JOIN_TIME;
     }
 
+    @Deprecated
     public void stop() {
         this.stop(false);
     }
 
+    @Deprecated
     public void stop(final boolean interrupt) {
+        if (!started.get()) {
+            return;
+        }
         this.stopped = true;
         log.info("stop thread " + this.getServiceName() + " interrupt " + interrupt);
 
@@ -93,6 +113,9 @@ public abstract class ServiceThread implements Runnable {
     }
 
     public void makeStop() {
+        if (!started.get()) {
+            return;
+        }
         this.stopped = true;
         log.info("makestop thread " + this.getServiceName());
     }
@@ -128,4 +151,12 @@ public abstract class ServiceThread implements Runnable {
     public boolean isStopped() {
         return stopped;
     }
+
+    public boolean isDaemon() {
+        return isDaemon;
+    }
+
+    public void setDaemon(boolean daemon) {
+        isDaemon = daemon;
+    }
 }

common/src/main/java/org/apache/rocketmq/common/UtilAll.java

@@ -61,6 +61,18 @@ public class UtilAll {
         }
     }
 
+    public static void sleep(long sleepMs) {
+        if (sleepMs < 0) {
+            return;
+        }
+        try {
+            Thread.sleep(sleepMs);
+        } catch (Throwable ignored) {
+
+        }
+
+    }
+
     public static String currentStackTrace() {
         StringBuilder sb = new StringBuilder();
         StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();

common/src/main/java/org/apache/rocketmq/common/constant/DBMsgConstants.java

@@ -18,5 +18,5 @@
 package org.apache.rocketmq.common.constant;
 
 public class DBMsgConstants {
-    public static final int MAX_BODY_SIZE = 64 * 1024 * 1204; //64KB
+    public static final int MAX_BODY_SIZE = 64 * 1024 * 1024; //64KB
 }

common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java

@@ -41,6 +41,7 @@ public class MessageDecoder {
     public final static int MESSAGE_MAGIC_CODE = -626843481;
     public static final char NAME_VALUE_SEPARATOR = 1;
     public static final char PROPERTY_SEPARATOR = 2;
+    public static final int PHY_POS_POSITION =  4 + 4 + 4 + 4 + 4 + 8;
     public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE
         + 4 // 2 MAGICCODE
         + 4 // 3 BODYCRC

common/src/test/java/org/apache/rocketmq/common/BrokerConfigTest.java

@@ -27,4 +27,21 @@ public class BrokerConfigTest {
         long expect = 1024L * 1024 * 1024 * 16;
         assertThat(new BrokerConfig().getConsumerFallbehindThreshold()).isEqualTo(expect);
     }
+
+    @Test
+    public void testBrokerConfigAttribute() {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setNamesrvAddr("127.0.0.1:9876");
+        brokerConfig.setAutoCreateTopicEnable(false);
+        brokerConfig.setBrokerName("broker-a");
+        brokerConfig.setBrokerId(0);
+        brokerConfig.setBrokerClusterName("DefaultCluster");
+        brokerConfig.setMsgTraceTopicName("RMQ_SYS_TRACE_TOPIC4");
+        assertThat(brokerConfig.getBrokerClusterName()).isEqualTo("DefaultCluster");
+        assertThat(brokerConfig.getNamesrvAddr()).isEqualTo("127.0.0.1:9876");
+        assertThat(brokerConfig.getMsgTraceTopicName()).isEqualTo("RMQ_SYS_TRACE_TOPIC4");
+        assertThat(brokerConfig.getBrokerId()).isEqualTo(0);
+        assertThat(brokerConfig.getBrokerName()).isEqualTo("broker-a");
+        assertThat(brokerConfig.isAutoCreateTopicEnable()).isEqualTo(false);
+    }
 }

common/src/test/java/org/apache/rocketmq/common/MixAllTest.java

@@ -17,14 +17,13 @@
 
 package org.apache.rocketmq.common;
 
-import org.junit.Test;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.net.InetAddress;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
 

distribution/bin/dledger/fast-try.sh

@@ -0,0 +1,87 @@
+#! /bin/bash 
+
+## Revise the base dir
+CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)"
+RMQ_DIR=$CURRENT_DIR/../..
+cd $RMQ_DIR
+
+function startNameserver() {
+    export JAVA_OPT_EXT=" -Xms512m -Xmx512m  "
+    nohup bin/mqnamesrv &
+}
+
+function startBroker() {
+    export JAVA_OPT_EXT=" -Xms1g -Xmx1g  "
+    conf_name=$1
+    nohup bin/mqbroker -c $conf_name &
+}
+
+function stopNameserver() {
+    PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}')
+    if [ ! -z "$PIDS" ]; then
+        kill -s TERM $PIDS
+    fi
+}
+
+function stopBroker() {
+    conf_name=$1
+    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')
+    i=1
+    while [ ! -z "$PIDS" -a $i -lt 5 ]
+    do
+        echo "Waiting to kill ..."
+        kill -s TERM $PIDS
+        ((i=$i+1))
+        sleep 2
+        PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')
+    done
+    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')
+    if [ ! -z "$PIDS" ]; then
+        kill -9 $PIDS
+    fi
+}
+
+function stopAll() {
+    ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill
+    stopNameserver
+    stopBroker ./conf/dledger/broker-n0.conf
+    stopBroker ./conf/dledger/broker-n1.conf
+    stopBroker ./conf/dledger/broker-n2.conf
+}
+
+function startAll() {
+    startNameserver
+    startBroker ./conf/dledger/broker-n0.conf
+    startBroker ./conf/dledger/broker-n1.conf
+    startBroker ./conf/dledger/broker-n2.conf
+}
+
+function checkConf() {
+    if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then
+        echo "Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists"
+        exit -1
+    fi 
+}
+
+
+
+## Main
+if [ $# -lt 1 ]; then
+    echo "Usage: sh $0 start|stop"
+    exit -1
+fi
+action=$1
+checkConf
+case $action in
+    "start")
+        startAll
+        exit
+        ;;
+    "stop")
+        stopAll
+        ;;
+    *)
+        echo "Usage: sh $0 start|stop"
+        ;;
+esac
+

distribution/bin/runbroker.sh

@@ -37,7 +37,7 @@ export CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH}
 # JVM Configuration
 #===========================================================================================
 JAVA_OPT="${JAVA_OPT} -server -Xms8g -Xmx8g -Xmn4g"
-JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8"
+JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0"
 JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:/dev/shm/mq_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy"
 JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m"
 JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow"

distribution/conf/2m-noslave/broker-trace.properties

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+brokerClusterName=DefaultCluster
+brokerName=broker-trace
+brokerId=0
+deleteWhen=04
+fileReservedTime=48
+brokerRole=ASYNC_MASTER
+flushDiskType=ASYNC_FLUSH

distribution/conf/dledger/broker-n0.conf

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30911
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node00
+storePathCommitLog=/tmp/rmqstore/node00/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n0
+sendMessageThreadPoolNums=16

distribution/conf/dledger/broker-n1.conf

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30921
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node01
+storePathCommitLog=/tmp/rmqstore/node01/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n1
+sendMessageThreadPoolNums=16

distribution/conf/dledger/broker-n2.conf

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30931
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node02
+storePathCommitLog=/tmp/rmqstore/node02/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n2
+sendMessageThreadPoolNums=16

distribution/conf/tools.yml

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+accessKey: rocketmq
+secretKey: 12345678
+    

distribution/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
     <artifactId>rocketmq-distribution</artifactId>
     <name>rocketmq-distribution ${project.version}</name>

distribution/release.xml

@@ -40,7 +40,7 @@
 
         <fileSet>
             <includes>
-                <include>bin/*</include>
+                <include>bin/**</include>
             </includes>
             <fileMode>0755</fileMode>
         </fileSet>

docs/cn/RocketMQ_Example.md

@@ -0,0 +1,956 @@
+1 基本样例
+--------
+
+在基本样例中我们提供如下的功能场景：
+
+* 使用RocketMQ发送三种类型的消息：同步消息，异步消息和单向消息。其中前两种消息是可靠的，因为会有发送是否成功的应答。
+* 使用RocketMQ来消费接收到的消息。
+
+### 1.1 加入依赖：
+
+`maven:`
+```
+<dependency>
+    <groupId>org.apache.rocketmq</groupId>
+    <artifactId>rocketmq-client</artifactId>
+    <version>4.3.0</version>
+</dependency>
+```
+`gradle`
+```
+compile 'org.apache.rocketmq:rocketmq-client:4.3.0'
+```
+### 1.2 消息发送
+
+#### 1、Producer端发送同步消息
+
+这种可靠性同步地发送方式使用的比较广泛，比如：重要的消息通知，短信通知。
+```java
+public class SyncProducer {
+	public static void main(String[] args) throws Exception {
+    	// 实例化消息生产者Producer
+        DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+    	// 设置NameServer的地址
+    	producer.setNamesrvAddr("localhost:9876");
+    	// 启动Producer实例
+        producer.start();
+    	for (int i = 0; i < 100; i++) {
+    	    // 创建消息，并指定Topic，Tag和消息体
+    	    Message msg = new Message("TopicTest" /* Topic */,
+        	"TagA" /* Tag */,
+        	("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
+        	);
+        	// 发送消息到一个Broker
+            SendResult sendResult = producer.send(msg);
+            // 通过sendResult返回消息是否成功送达
+            System.out.printf("%s%n", sendResult);
+    	}
+    	// 如果不再发送消息，关闭Producer实例。
+    	producer.shutdown();
+    }
+}
+```
+#### 2、发送异步消息
+
+异步消息通常用在对响应时间敏感的业务场景，即发送端不能容忍长时间地等待Broker的响应。
+
+```java
+public class AsyncProducer {
+	public static void main(String[] args) throws Exception {
+    	// 实例化消息生产者Producer
+        DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+    	// 设置NameServer的地址
+        producer.setNamesrvAddr("localhost:9876");
+    	// 启动Producer实例
+        producer.start();
+        producer.setRetryTimesWhenSendAsyncFailed(0);
+    	for (int i = 0; i < 100; i++) {
+                final int index = i;
+            	// 创建消息，并指定Topic，Tag和消息体
+                Message msg = new Message("TopicTest",
+                    "TagA",
+                    "OrderID188",
+                    "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+                // SendCallback接收异步返回结果的回调
+                producer.send(msg, new SendCallback() {
+                    @Override
+                    public void onSuccess(SendResult sendResult) {
+                        System.out.printf("%-10d OK %s %n", index,
+                            sendResult.getMsgId());
+                    }
+                    @Override
+                    public void onException(Throwable e) {
+      	              System.out.printf("%-10d Exception %s %n", index, e);
+      	              e.printStackTrace();
+                    }
+            	});
+    	}
+    	// 如果不再发送消息，关闭Producer实例。
+    	producer.shutdown();
+    }
+}
+```
+
+#### 3、单向发送消息
+
+这种方式主要用在不特别关心发送结果的场景，例如日志发送。
+
+```java
+public class OnewayProducer {
+	public static void main(String[] args) throws Exception{
+    	// 实例化消息生产者Producer
+        DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+    	// 设置NameServer的地址
+        producer.setNamesrvAddr("localhost:9876");
+    	// 启动Producer实例
+        producer.start();
+    	for (int i = 0; i < 100; i++) {
+        	// 创建消息，并指定Topic，Tag和消息体
+        	Message msg = new Message("TopicTest" /* Topic */,
+                "TagA" /* Tag */,
+                ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
+        	);
+        	// 发送单向消息，没有任何返回结果
+        	producer.sendOneway(msg);
+
+    	}
+    	// 如果不再发送消息，关闭Producer实例。
+    	producer.shutdown();
+    }
+}
+```
+
+### 1.3 消费消息
+
+```java
+public class Consumer {
+
+	public static void main(String[] args) throws InterruptedException, MQClientException {
+
+    	// 实例化消费者
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name");
+
+    	// 设置NameServer的地址
+        consumer.setNamesrvAddr("localhost:9876");
+
+    	// 订阅一个或者多个Topic，以及Tag来过滤需要消费的消息
+        consumer.subscribe("TopicTest", "*");
+    	// 注册回调实现类来处理从broker拉取回来的消息
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+                // 标记该消息已经被成功消费
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        // 启动消费者实例
+        consumer.start();
+        System.out.printf("Consumer Started.%n");
+	}
+}
+```
+
+2 顺序消息样例
+----------
+
+消息有序指的是可以按照消息的发送顺序来消费(FIFO)。RocketMQ可以严格的保证消息有序，可以分为分区有序或者全局有序。
+
+顺序消费的原理解析，在默认的情况下消息发送会采取Round Robin轮询方式把消息发送到不同的queue(分区队列)；而消费消息的时候从多个queue上拉取消息，这种情况发送和消费是不能保证顺序。但是如果控制发送的顺序消息只依次发送到同一个queue中，消费的时候只从这个queue上依次拉取，则就保证了顺序。当发送和消费参与的queue只有一个，则是全局有序；如果多个queue参与，则为分区有序，即相对每个queue，消息都是有序的。
+
+下面用订单进行分区有序的示例。一个订单的顺序流程是：创建、付款、推送、完成。订单号相同的消息会被先后发送到同一个队列中，消费时，同一个OrderId获取到的肯定是同一个队列。
+
+### 2.1 顺序消息生产
+
+```java
+package org.apache.rocketmq.example.order2;
+
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.MessageQueueSelector;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+/**
+* Producer，发送顺序消息
+*/
+public class Producer {
+
+   public static void main(String[] args) throws Exception {
+       DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+
+       producer.setNamesrvAddr("127.0.0.1:9876");
+
+       producer.start();
+
+       String[] tags = new String[]{"TagA", "TagC", "TagD"};
+
+       // 订单列表
+       List<OrderStep> orderList = new Producer().buildOrders();
+
+       Date date = new Date();
+       SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+       String dateStr = sdf.format(date);
+       for (int i = 0; i < 10; i++) {
+           // 加个时间前缀
+           String body = dateStr + " Hello RocketMQ " + orderList.get(i);
+           Message msg = new Message("TopicTest", tags[i % tags.length], "KEY" + i, body.getBytes());
+
+           SendResult sendResult = producer.send(msg, new MessageQueueSelector() {
+               @Override
+               public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
+                   Long id = (Long) arg;  //根据订单id选择发送queue
+                   long index = id % mqs.size();
+                   return mqs.get((int) index);
+               }
+           }, orderList.get(i).getOrderId());//订单id
+
+           System.out.println(String.format("SendResult status:%s, queueId:%d, body:%s",
+               sendResult.getSendStatus(),
+               sendResult.getMessageQueue().getQueueId(),
+               body));
+       }
+
+       producer.shutdown();
+   }
+
+   /**
+    * 订单的步骤
+    */
+   private static class OrderStep {
+       private long orderId;
+       private String desc;
+
+       public long getOrderId() {
+           return orderId;
+       }
+
+       public void setOrderId(long orderId) {
+           this.orderId = orderId;
+       }
+
+       public String getDesc() {
+           return desc;
+       }
+
+       public void setDesc(String desc) {
+           this.desc = desc;
+       }
+
+       @Override
+       public String toString() {
+           return "OrderStep{" +
+               "orderId=" + orderId +
+               ", desc='" + desc + '\'' +
+               '}';
+       }
+   }
+
+   /**
+    * 生成模拟订单数据
+    */
+   private List<OrderStep> buildOrders() {
+       List<OrderStep> orderList = new ArrayList<OrderStep>();
+
+       OrderStep orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111039L);
+       orderDemo.setDesc("创建");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111065L);
+       orderDemo.setDesc("创建");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111039L);
+       orderDemo.setDesc("付款");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103117235L);
+       orderDemo.setDesc("创建");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111065L);
+       orderDemo.setDesc("付款");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103117235L);
+       orderDemo.setDesc("付款");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111065L);
+       orderDemo.setDesc("完成");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111039L);
+       orderDemo.setDesc("推送");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103117235L);
+       orderDemo.setDesc("完成");
+       orderList.add(orderDemo);
+
+       orderDemo = new OrderStep();
+       orderDemo.setOrderId(15103111039L);
+       orderDemo.setDesc("完成");
+       orderList.add(orderDemo);
+
+       return orderList;
+   }
+}
+```
+
+### 2.2 顺序消费消息
+
+```java
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.message.MessageExt;
+import java.util.List;
+
+package org.apache.rocketmq.example.order2;
+
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.MessageExt;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+* 顺序消息消费，带事务方式（应用可控制Offset什么时候提交）
+*/
+public class ConsumerInOrder {
+
+   public static void main(String[] args) throws Exception {
+       DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_3");
+       consumer.setNamesrvAddr("127.0.0.1:9876");
+       /**
+        * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费<br>
+        * 如果非第一次启动，那么按照上次消费的位置继续消费
+        */
+       consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+
+       consumer.subscribe("TopicTest", "TagA || TagC || TagD");
+
+       consumer.registerMessageListener(new MessageListenerOrderly() {
+
+           Random random = new Random();
+
+           @Override
+           public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {
+               context.setAutoCommit(true);
+               for (MessageExt msg : msgs) {
+                   // 可以看到每个queue有唯一的consume线程来消费, 订单对每个queue(分区)有序
+                   System.out.println("consumeThread=" + Thread.currentThread().getName() + "queueId=" + msg.getQueueId() + ", content:" + new String(msg.getBody()));
+               }
+
+               try {
+                   //模拟业务逻辑处理中...
+                   TimeUnit.SECONDS.sleep(random.nextInt(10));
+               } catch (Exception e) {
+                   e.printStackTrace();
+               }
+               return ConsumeOrderlyStatus.SUCCESS;
+           }
+       });
+
+       consumer.start();
+
+       System.out.println("Consumer Started.");
+   }
+}
+```
+
+3 延时消息样例
+----------
+
+### 3.1 启动消费者等待传入订阅消息
+
+```java
+
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.message.MessageExt;
+import java.util.List;
+
+public class ScheduledMessageConsumer {
+   public static void main(String[] args) throws Exception {
+      // 实例化消费者
+      DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ExampleConsumer");
+      // 订阅Topics
+      consumer.subscribe("TestTopic", "*");
+      // 注册消息监听者
+      consumer.registerMessageListener(new MessageListenerConcurrently() {
+          @Override
+          public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> messages, ConsumeConcurrentlyContext context) {
+              for (MessageExt message : messages) {
+                  // Print approximate delay time period
+                  System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later");
+              }
+              return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+          }
+      });
+      // 启动消费者
+      consumer.start();
+  }
+}
+
+```
+
+### 3.2 发送延时消息
+
+```java
+
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.message.Message;
+
+public class ScheduledMessageProducer {
+   public static void main(String[] args) throws Exception {
+      // 实例化一个生产者来产生延时消息
+      DefaultMQProducer producer = new DefaultMQProducer("ExampleProducerGroup");
+      // 启动生产者
+      producer.start();
+      int totalMessagesToSend = 100;
+      for (int i = 0; i < totalMessagesToSend; i++) {
+          Message message = new Message("TestTopic", ("Hello scheduled message " + i).getBytes());
+          // 设置延时等级3,这个消息将在10s之后发送(现在只支持固定的几个时间,详看delayTimeLevel)
+          message.setDelayTimeLevel(3);
+          // 发送消息
+          producer.send(message);
+      }
+       // 关闭生产者
+      producer.shutdown();
+  }
+}
+```
+
+### 3.3 验证
+
+您将会看到消息的消费比存储时间晚10秒。
+
+### 3.4 延时消息的使用场景
+1. 比如电商里，提交了一个订单就可以发送一个延时消息，1h后去检查这个订单的状态，如果还是未付款就取消订单释放库存。
+
+### 3.5 延时消息的使用限制
+
+```java
+// org/apache/rocketmq/store/config/MessageStoreConfig.java
+
+private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h";
+```
+现在RocketMq并不支持任意时间的延时，需要设置几个固定的延时等级，从1s到2h分别对应着等级1到18
+消息消费失败会进入延时消息队列，消息发送时间与设置的延时等级和重试次数有关，详见代码`SendMessageProcessor.java`
+
+
+4 批量消息样例
+----------
+
+批量发送消息能显著提高传递小消息的性能。限制是这些批量消息应该有相同的topic，相同的waitStoreMsgOK，而且不能是延时消息。此外，这一批消息的总大小不应超过4MB。
+
+### 4.1 发送批量消息
+
+如果您每次只发送不超过4MB的消息，则很容易使用批处理，样例如下：
+
+```java
+String topic = "BatchTest";
+List<Message> messages = new ArrayList<>();
+messages.add(new Message(topic, "TagA", "OrderID001", "Hello world 0".getBytes()));
+messages.add(new Message(topic, "TagA", "OrderID002", "Hello world 1".getBytes()));
+messages.add(new Message(topic, "TagA", "OrderID003", "Hello world 2".getBytes()));
+try {
+   producer.send(messages);
+} catch (Exception e) {
+   e.printStackTrace();
+   //处理error
+}
+
+```
+
+### 4.2 消息列表分割
+
+复杂度只有当你发送大批量时才会增长，你可能不确定它是否超过了大小限制（4MB）。这时候你最好把你的消息列表分割一下：
+
+```java
+
+public class ListSplitter implements Iterator<List<Message>> {
+   private final int SIZE_LIMIT = 1024 * 1024 * 4;
+   private final List<Message> messages;
+   private int currIndex;
+   public ListSplitter(List<Message> messages) {
+           this.messages = messages;
+   }
+   @Override public boolean hasNext() {
+       return currIndex < messages.size();
+   }
+   @Override public List<Message> next() {
+       int nextIndex = currIndex;
+       int totalSize = 0;
+       for (; nextIndex < messages.size(); nextIndex++) {
+           Message message = messages.get(nextIndex);
+           int tmpSize = message.getTopic().length() + message.getBody().length;
+           Map<String, String> properties = message.getProperties();
+           for (Map.Entry<String, String> entry : properties.entrySet()) {
+               tmpSize += entry.getKey().length() + entry.getValue().length();
+           }
+           tmpSize = tmpSize + 20; // 增加日志的开销20字节
+           if (tmpSize > SIZE_LIMIT) {
+               //单个消息超过了最大的限制
+               //忽略,否则会阻塞分裂的进程
+               if (nextIndex - currIndex == 0) {
+                  //假如下一个子列表没有元素,则添加这个子列表然后退出循环,否则只是退出循环
+                  nextIndex++;
+               }
+               break;
+           }
+           if (tmpSize + totalSize > SIZE_LIMIT) {
+               break;
+           } else {
+               totalSize += tmpSize;
+           }
+
+       }
+       List<Message> subList = messages.subList(currIndex, nextIndex);
+       currIndex = nextIndex;
+       return subList;
+   }
+}
+//把大的消息分裂成若干个小的消息
+ListSplitter splitter = new ListSplitter(messages);
+while (splitter.hasNext()) {
+  try {
+      List<Message>  listItem = splitter.next();
+      producer.send(listItem);
+  } catch (Exception e) {
+      e.printStackTrace();
+      //处理error
+  }
+}
+```
+
+5 过滤消息样例
+----------
+
+在大多数情况下，TAG是一个简单而有用的设计，其可以来选择您想要的消息。例如：
+
+```java
+DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_EXAMPLE");
+consumer.subscribe("TOPIC", "TAGA || TAGB || TAGC");
+```
+
+消费者将接收包含TAGA或TAGB或TAGC的消息。但是限制是一个消息只能有一个标签，这对于复杂的场景可能不起作用。在这种情况下，可以使用SQL表达式筛选消息。SQL特性可以通过发送消息时的属性来进行计算。在RocketMQ定义的语法下，可以实现一些简单的逻辑。下面是一个例子：
+```
+------------
+| message  |
+|----------|  a > 5 AND b = 'abc'
+| a = 10   |  --------------------> Gotten
+| b = 'abc'|
+| c = true |
+------------
+------------
+| message  |
+|----------|   a > 5 AND b = 'abc'
+| a = 1    |  --------------------> Missed
+| b = 'abc'|
+| c = true |
+------------
+```
+### 5.1 基本语法
+
+RocketMQ只定义了一些基本语法来支持这个特性。你也可以很容易地扩展它。
+
+- 数值比较，比如：**>，>=，<，<=，BETWEEN，=；**
+- 字符比较，比如：**=，<>，IN；**
+- **IS NULL** 或者 **IS NOT NULL；**
+- 逻辑符号 **AND，OR，NOT；**
+
+常量支持类型为：
+
+- 数值，比如：**123，3.1415；**
+- 字符，比如：**'abc'，必须用单引号包裹起来；**
+- **NULL**，特殊的常量
+- 布尔值，**TRUE** 或 **FALSE**
+
+只有使用push模式的消费者才能用使用SQL92标准的sql语句，接口如下：
+```
+public void subscribe(finalString topic, final MessageSelector messageSelector)
+```
+
+### 5.2 使用样例
+
+#### 1、生产者样例
+
+发送消息时，你能通过`putUserProperty`来设置消息的属性
+
+```java
+DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+producer.start();
+Message msg = new Message("TopicTest",
+   tag,
+   ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)
+);
+// 设置一些属性
+msg.putUserProperty("a", String.valueOf(i));
+SendResult sendResult = producer.send(msg);
+
+producer.shutdown();
+```
+
+#### 2、消费者样例
+
+用MessageSelector.bySql来使用sql筛选消息
+
+```java
+DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4");
+// 只有订阅的消息有这个属性a, a >=0 and a <= 3
+consumer.subscribe("TopicTest", MessageSelector.bySql("a between 0 and 3");
+consumer.registerMessageListener(new MessageListenerConcurrently() {
+   @Override
+   public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+       return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+   }
+});
+consumer.start();
+
+```
+
+6 消息事务样例
+----------
+
+事务消息共有三种状态，提交状态、回滚状态、中间状态：
+
+1. TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。
+2. TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。
+3. TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。
+
+### 6.1 发送事务消息样例
+
+#### 1、创建事务性生产者
+
+使用 `TransactionMQProducer`类创建生产者，并指定唯一的 `ProducerGroup`，就可以设置自定义线程池来处理这些检查请求。执行本地事务后、需要根据执行结果对消息队列进行回复。回传的事务状态在请参考前一节。
+
+```java
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.message.MessageExt;
+import java.util.List;
+public class TransactionProducer {
+   public static void main(String[] args) throws MQClientException, InterruptedException {
+       TransactionListener transactionListener = new TransactionListenerImpl();
+       TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name");
+       ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000), new ThreadFactory() {
+           @Override
+           public Thread newThread(Runnable r) {
+               Thread thread = new Thread(r);
+               thread.setName("client-transaction-msg-check-thread");
+               return thread;
+           }
+       });
+       producer.setExecutorService(executorService);
+       producer.setTransactionListener(transactionListener);
+       producer.start();
+       String[] tags = new String[] {"TagA", "TagB", "TagC", "TagD", "TagE"};
+       for (int i = 0; i < 10; i++) {
+           try {
+               Message msg =
+                   new Message("TopicTest1234", tags[i % tags.length], "KEY" + i,
+                       ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
+               SendResult sendResult = producer.sendMessageInTransaction(msg, null);
+               System.out.printf("%s%n", sendResult);
+               Thread.sleep(10);
+           } catch (MQClientException | UnsupportedEncodingException e) {
+               e.printStackTrace();
+           }
+       }
+       for (int i = 0; i < 100000; i++) {
+           Thread.sleep(1000);
+       }
+       producer.shutdown();
+   }
+}
+
+```
+#### 2、实现事务的监听接口
+
+当发送半消息成功时，我们使用 `executeLocalTransaction` 方法来执行本地事务。它返回前一节中提到的三个事务状态之一。`checkLocalTranscation` 方法用于检查本地事务状态，并回应消息队列的检查请求。它也是返回前一节中提到的三个事务状态之一。
+
+```java
+public class TransactionListenerImpl implements TransactionListener {
+  private AtomicInteger transactionIndex = new AtomicInteger(0);
+  private ConcurrentHashMap<String, Integer> localTrans = new ConcurrentHashMap<>();
+  @Override
+  public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {
+      int value = transactionIndex.getAndIncrement();
+      int status = value % 3;
+      localTrans.put(msg.getTransactionId(), status);
+      return LocalTransactionState.UNKNOW;
+  }
+  @Override
+  public LocalTransactionState checkLocalTransaction(MessageExt msg) {
+      Integer status = localTrans.get(msg.getTransactionId());
+      if (null != status) {
+          switch (status) {
+              case 0:
+                  return LocalTransactionState.UNKNOW;
+              case 1:
+                  return LocalTransactionState.COMMIT_MESSAGE;
+              case 2:
+                  return LocalTransactionState.ROLLBACK_MESSAGE;
+          }
+      }
+      return LocalTransactionState.COMMIT_MESSAGE;
+  }
+}
+
+```
+
+### 6.2 事务消息使用上的限制
+
+1. 事务消息不支持延时消息和批量消息。
+2. 为了避免单个消息被检查太多次而导致半队列消息累积，我们默认将单个消息的检查次数限制为 15 次，但是用户可以通过 Broker 配置文件的 `transactionCheckMax`参数来修改此限制。如果已经检查某条消息超过 N 次的话（ N = `transactionCheckMax` ） 则 Broker 将丢弃此消息，并在默认情况下同时打印错误日志。用户可以通过重写 `AbstractTransactionCheckListener` 类来修改这个行为。
+3. 事务消息将在 Broker 配置文件中的参数 transactionMsgTimeout 这样的特定时间长度之后被检查。当发送事务消息时，用户还可以通过设置用户属性 CHECK_IMMUNITY_TIME_IN_SECONDS 来改变这个限制，该参数优先于 `transactionMsgTimeout` 参数。
+4. 事务性消息可能不止一次被检查或消费。
+5. 提交给用户的目标主题消息可能会失败，目前这依日志的记录而定。它的高可用性通过 RocketMQ 本身的高可用性机制来保证，如果希望确保事务消息不丢失、并且事务完整性得到保证，建议使用同步的双重写入机制。
+6. 事务消息的生产者 ID 不能与其他类型消息的生产者 ID 共享。与其他类型的消息不同，事务消息允许反向查询、MQ服务器能通过它们的生产者 ID 查询到消费者。
+
+7 Logappender样例
+-----------------
+
+RocketMQ日志提供log4j、log4j2和logback日志框架作为业务应用，下面是配置样例
+
+### 7.1 log4j样例
+
+按下面样例使用log4j属性配置
+```
+log4j.appender.mq=org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender
+log4j.appender.mq.Tag=yourTag
+log4j.appender.mq.Topic=yourLogTopic
+log4j.appender.mq.ProducerGroup=yourLogGroup
+log4j.appender.mq.NameServerAddress=yourRocketmqNameserverAddress
+log4j.appender.mq.layout=org.apache.log4j.PatternLayout
+log4j.appender.mq.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-4r [%t] (%F:%L) %-5p - %m%n
+```
+按下面样例使用log4j xml配置来使用异步添加日志
+```
+<appender name="mqAppender1"class="org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender">
+  <param name="Tag" value="yourTag" />
+  <param name="Topic" value="yourLogTopic" />
+  <param name="ProducerGroup" value="yourLogGroup" />
+  <param name="NameServerAddress" value="yourRocketmqNameserverAddress"/>
+  <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss}-%p %t %c - %m%n" />
+  </layout>
+</appender>
+<appender name="mqAsyncAppender1"class="org.apache.log4j.AsyncAppender">
+  <param name="BufferSize" value="1024" />
+  <param name="Blocking" value="false" />
+  <appender-ref ref="mqAppender1"/>
+</appender>
+```
+### 7.2 log4j2样例
+
+用log4j2时，配置如下，如果想要非阻塞，只需要使用异步添加引用即可
+```
+<RocketMQ name="rocketmqAppender" producerGroup="yourLogGroup" nameServerAddress="yourRocketmqNameserverAddress"
+   topic="yourLogTopic" tag="yourTag">
+  <PatternLayout pattern="%d [%p] hahahah %c %m%n"/>
+</RocketMQ>
+```
+### 7.3 logback样例
+```
+<appender name="mqAppender1"class="org.apache.rocketmq.logappender.logback.RocketmqLogbackAppender">
+  <tag>yourTag</tag>
+  <topic>yourLogTopic</topic>
+  <producerGroup>yourLogGroup</producerGroup>
+  <nameServerAddress>yourRocketmqNameserverAddress</nameServerAddress>
+  <layout>
+      <pattern>%date %p %t - %m%n</pattern>
+  </layout>
+</appender>
+<appender name="mqAsyncAppender1"class="ch.qos.logback.classic.AsyncAppender">
+  <queueSize>1024</queueSize>
+  <discardingThreshold>80</discardingThreshold>
+  <maxFlushTime>2000</maxFlushTime>
+  <neverBlock>true</neverBlock>
+  <appender-ref ref="mqAppender1"/>
+</appender>
+```
+
+8 OpenMessaging样例
+---------------
+
+ [OpenMessaging](https://www.google.com/url?q=http://openmessaging.cloud/&sa=D&ust=1546524111089000)旨在建立消息和流处理规范，以为金融、电子商务、物联网和大数据领域提供通用框架及工业级指导方案。在分布式异构环境中，设计原则是面向云、简单、灵活和独立于语言。符合这些规范将帮助企业方便的开发跨平台和操作系统的异构消息传递应用程序。提供了openmessaging-api 0.3.0-alpha的部分实现，下面的示例演示如何基于OpenMessaging访问RocketMQ。
+
+### 8.1 OMSProducer样例
+
+下面的示例演示如何在同步、异步或单向传输中向RocketMQ代理发送消息。
+
+```java
+import io.openmessaging.Future;
+import io.openmessaging.FutureListener;
+import io.openmessaging.Message;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.OMS;
+import io.openmessaging.producer.Producer;
+import io.openmessaging.producer.SendResult;
+import java.nio.charset.Charset;
+import java.util.concurrent.CountDownLatch;
+
+public class SimpleProducer {
+    public static void main(String[] args) {
+       final MessagingAccessPoint messagingAccessPoint =
+           OMS.getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default");
+       final Producer producer = messagingAccessPoint.createProducer();
+       messagingAccessPoint.startup();
+       System.out.printf("MessagingAccessPoint startup OK%n");
+       producer.startup();
+       System.out.printf("Producer startup OK%n");
+       {
+           Message message = producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8")));
+           SendResult sendResult = producer.send(message);
+           //final Void aVoid = result.get(3000L);
+           System.out.printf("Send async message OK, msgId: %s%n", sendResult.messageId());
+       }
+       final CountDownLatch countDownLatch = new CountDownLatch(1);
+       {
+           final Future<SendResult> result = producer.sendAsync(producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))));
+           result.addListener(new FutureListener<SendResult>() {
+               @Override
+               public void operationComplete(Future<SendResult> future) {
+                   if (future.getThrowable() != null) {
+                       System.out.printf("Send async message Failed, error: %s%n", future.getThrowable().getMessage());
+                   } else {
+                       System.out.printf("Send async message OK, msgId: %s%n", future.get().messageId());
+                   }
+                   countDownLatch.countDown();
+               }
+           });
+       }
+       {
+           producer.sendOneway(producer.createBytesMessage("OMS_HELLO_TOPIC", "OMS_HELLO_BODY".getBytes(Charset.forName("UTF-8"))));
+           System.out.printf("Send oneway message OK%n");
+       }
+       try {
+           countDownLatch.await();
+           Thread.sleep(500); // 等一些时间来发送消息
+       } catch (InterruptedException ignore) {
+       }
+       producer.shutdown();
+   }
+}
+```
+
+### 8.2 OMSPullConsumer
+
+用OMS PullConsumer 来从指定的队列中拉取消息
+
+```java
+import io.openmessaging.Message;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.OMS;
+import io.openmessaging.OMSBuiltinKeys;
+import io.openmessaging.consumer.PullConsumer;
+import io.openmessaging.producer.Producer;
+import io.openmessaging.producer.SendResult;
+
+public class SimplePullConsumer {
+    public static void main(String[] args) {
+       final MessagingAccessPoint messagingAccessPoint =
+           OMS.getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default");
+       messagingAccessPoint.startup();
+       final Producer producer = messagingAccessPoint.createProducer();
+       final PullConsumer consumer = messagingAccessPoint.createPullConsumer(
+           OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "OMS_CONSUMER"));
+       messagingAccessPoint.startup();
+       System.out.printf("MessagingAccessPoint startup OK%n");
+       final String queueName = "TopicTest";
+       producer.startup();
+       Message msg = producer.createBytesMessage(queueName, "Hello Open Messaging".getBytes());
+       SendResult sendResult = producer.send(msg);
+       System.out.printf("Send Message OK. MsgId: %s%n", sendResult.messageId());
+       producer.shutdown();
+       consumer.attachQueue(queueName);
+       consumer.startup();
+       System.out.printf("Consumer startup OK%n");
+       // 运行直到发现一个消息被发送了
+       boolean stop = false;
+       while (!stop) {
+           Message message = consumer.receive();
+           if (message != null) {
+               String msgId = message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID);
+               System.out.printf("Received one message: %s%n", msgId);
+               consumer.ack(msgId);
+               if (!stop) {
+                   stop = msgId.equalsIgnoreCase(sendResult.messageId());
+               }
+           } else {
+               System.out.printf("Return without any message%n");
+           }
+       }
+       consumer.shutdown();
+       messagingAccessPoint.shutdown();
+   }
+}
+```
+
+### 8.3 OMSPushConsumer
+
+以下示范如何将 OMS PushConsumer 添加到指定的队列，并通过 MessageListener 消费这些消息。
+
+```java
+import io.openmessaging.Message;
+import io.openmessaging.MessagingAccessPoint;
+import io.openmessaging.OMS;
+import io.openmessaging.OMSBuiltinKeys;
+import io.openmessaging.consumer.MessageListener;
+import io.openmessaging.consumer.PushConsumer;
+
+public class SimplePushConsumer {
+    public static void main(String[] args) {
+       final MessagingAccessPoint messagingAccessPoint = OMS
+           .getMessagingAccessPoint("oms:rocketmq://localhost:9876/default:default");
+       final PushConsumer consumer = messagingAccessPoint.
+           createPushConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, "OMS_CONSUMER"));
+       messagingAccessPoint.startup();
+       System.out.printf("MessagingAccessPoint startup OK%n");
+       Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+           @Override
+           public void run() {
+               consumer.shutdown();
+               messagingAccessPoint.shutdown();
+           }
+       }));
+       consumer.attachQueue("OMS_HELLO_TOPIC", new MessageListener() {
+           @Override
+           public void onReceived(Message message, Context context) {
+               System.out.printf("Received one message: %s%n", message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID));
+               context.ack();
+           }
+       });
+       consumer.startup();
+       System.out.printf("Consumer startup OK%n");
+   }
+}
+```

docs/cn/acl/user_guide.md

@@ -0,0 +1,82 @@
+# 权限控制
+## 前言
+该文档主要介绍如何快速部署和使用支持权限控制特性的RocketMQ 集群。
+
+## 1.权限控制特性介绍
+权限控制（ACL）主要为RocketMQ提供Topic资源级别的用户访问控制。用户在使用RocketMQ权限控制时，可以在Client客户端通过 RPCHook注入AccessKey和SecretKey签名；同时，将对应的权限控制属性（包括Topic访问权限、IP白名单和AccessKey和SecretKey签名等）设置在distribution/conf/plain_acl.yml的配置文件中。Broker端对AccessKey所拥有的权限进行校验，校验不过，抛出异常；
+ACL客户端可以参考：**org.apache.rocketmq.example.simple**包下面的**AclClient**代码。
+
+## 2. 权限控制的定义与属性值
+### 2.1权限定义
+对RocketMQ的Topic资源访问权限控制定义主要如下表所示，分为以下四种
+
+
+| 权限 | 含义 |
+| --- | --- |
+| DENY | 拒绝 |
+| ANY | PUB 或者 SUB 权限 |
+| PUB | 发送权限 |
+| SUB | 订阅权限 |
+
+### 2.2 权限定义的关键属性
+| 字段 | 取值 | 含义 |
+| --- | --- | --- |
+| globalWhiteRemoteAddresses | \*;192.168.\*.\*;192.168.0.1 | 全局IP白名单 |
+| accessKey | 字符串 | Access Key |
+| secretKey | 字符串 | Secret Key |
+| whiteRemoteAddress | \*;192.168.\*.\*;192.168.0.1 | 用户IP白名单 |
+| admin | true;false | 是否管理员账户 |
+| defaultTopicPerm | DENY;PUB;SUB;PUB\|SUB | 默认的Topic权限 |
+| defaultGroupPerm | DENY;PUB;SUB;PUB\|SUB | 默认的ConsumerGroup权限 |
+| topicPerms | topic=权限 | 各个Topic的权限 |
+| groupPerms | group=权限 | 各个ConsumerGroup的权限 |
+
+具体可以参考**distribution/conf/plain_acl.yml**配置文件
+
+## 3. 支持权限控制的集群部署
+在**distribution/conf/plain_acl.yml**配置文件中按照上述说明定义好权限属性后，打开**aclEnable**开关变量即可开启RocketMQ集群的ACL特性。这里贴出Broker端开启ACL特性的properties配置文件内容：
+```
+brokerClusterName=DefaultCluster
+brokerName=broker-a
+brokerId=0
+deleteWhen=04
+fileReservedTime=48
+brokerRole=ASYNC_MASTER
+flushDiskType=ASYNC_FLUSH
+storePathRootDir=/data/rocketmq/rootdir-a-m
+storePathCommitLog=/data/rocketmq/commitlog-a-m
+autoCreateSubscriptionGroup=true
+## if acl is open,the flag will be true
+aclEnable=true
+listenPort=10911
+brokerIP1=XX.XX.XX.XX1
+namesrvAddr=XX.XX.XX.XX:9876
+```
+
+## 4. 权限控制主要流程
+ACL主要流程分为两部分，主要包括权限解析和权限校验。
+
+### 4.1 权限解析
+Broker端对客户端的RequestCommand请求进行解析，拿到需要鉴权的属性字段。
+主要包括：
+（1）AccessKey：类似于用户名，代指用户主体，权限数据与之对应；
+（2）Signature：客户根据 SecretKey 签名得到的串，服务端再用SecretKey进行签名验证；
+
+### 4.2 权限校验
+Broker端对权限的校验逻辑主要分为以下几步：
+（1）检查是否命中全局 IP 白名单；如果是，则认为校验通过；否则走 2；
+（2）检查是否命中用户 IP 白名单；如果是，则认为校验通过；否则走 3；
+（3）校验签名，校验不通过，抛出异常；校验通过，则走 4；
+（4）对用户请求所需的权限 和 用户所拥有的权限进行校验；不通过，抛出异常； 
+用户所需权限的校验需要注意已下内容：
+（1）特殊的请求例如 UPDATE_AND_CREATE_TOPIC 等，只能由 admin 账户进行操作；
+（2）对于某个资源，如果有显性配置权限，则采用配置的权限；如果没有显性配置权限，则采用默认的权限；
+
+## 5. 热加载修改后权限控制定义
+RocketrMQ的权限控制存储的默认实现是基于yml配置文件。用户可以动态修改权限控制定义的属性，而不需重新启动Broker服务节点。
+
+
+
+
+
+

docs/cn/architecture.md

@@ -0,0 +1,46 @@
+# 架构设计
+
+## 技术架构
+![](image/rocketmq_architecture_1.png)
+
+RocketMQ架构上主要分为四部分，如上图所示:
+
+
+- Producer：消息发布的角色，支持分布式集群方式部署。Producer通过MQ的负载均衡模块选择相应的Broker集群队列进行消息投递，投递的过程支持快速失败并且低延迟。
+
+- Consumer：消息消费的角色，支持分布式集群方式部署。支持以push推，pull拉两种模式对消息进行消费。同时也支持集群方式和广播方式的消费，它提供实时消息订阅机制，可以满足大多数用户的需求。
+
+- NameServer：NameServer是一个非常简单的Topic路由注册中心，其角色类似Dubbo中的zookeeper，支持Broker的动态注册与发现。主要包括两个功能：Broker管理，NameServer接受Broker集群的注册信息并且保存下来作为路由信息的基本数据。然后提供心跳检测机制，检查Broker是否还存活；路由信息管理，每个NameServer将保存关于Broker集群的整个路由信息和用于客户端查询的队列信息。然后Producer和Conumser通过NameServer就可以知道整个Broker集群的路由信息，从而进行消息的投递和消费。NameServer通常也是集群的方式部署，各实例间相互不进行信息通讯。Broker是向每一台NameServer注册自己的路由信息，所以每一个NameServer实例上面都保存一份完整的路由信息。当某个NameServer因某种原因下线了，Broker仍然可以向其它NameServer同步其路由信息，Producer,Consumer仍然可以动态感知Broker的路由的信息。 
+
+- BrokerServer：Broker主要负责消息的存储、投递和查询以及服务高可用保证，为了实现这些功能，Broker包含了以下几个重要子模块。
+1. Remoting Module：整个Broker的实体，负责处理来自clients端的请求。
+2. Client Manager：负责管理客户端(Producer/Consumer)和维护Consumer的Topic订阅信息
+3. Store Service：提供方便简单的API接口处理消息存储到物理硬盘和查询功能。
+4. HA Service：高可用服务，提供Master Broker 和 Slave Broker之间的数据同步功能。
+5. Index Service：根据特定的Message key对投递到Broker的消息进行索引服务，以提供消息的快速查询。
+
+![](image/rocketmq_architecture_2.png)
+
+## 部署架构
+
+
+![](image/rocketmq_architecture_3.png)
+
+
+### RocketMQ 网络部署特点
+
+- NameServer是一个几乎无状态节点，可集群部署，节点之间无任何信息同步。
+
+- Broker部署相对复杂，Broker分为Master与Slave，一个Master可以对应多个Slave，但是一个Slave只能对应一个Master，Master与Slave 的对应关系通过指定相同的BrokerName，不同的BrokerId 来定义，BrokerId为0表示Master，非0表示Slave。Master也可以部署多个。每个Broker与NameServer集群中的所有节点建立长连接，定时注册Topic信息到所有NameServer。 注意：当前RocketMQ版本在部署架构上支持一Master多Slave，但只有BrokerId=1的从服务器才会参与消息的读负载。
+
+- Producer与NameServer集群中的其中一个节点（随机选择）建立长连接，定期从NameServer获取Topic路由信息，并向提供Topic 服务的Master建立长连接，且定时向Master发送心跳。Producer完全无状态，可集群部署。
+
+- Consumer与NameServer集群中的其中一个节点（随机选择）建立长连接，定期从NameServer获取Topic路由信息，并向提供Topic服务的Master、Slave建立长连接，且定时向Master、Slave发送心跳。Consumer既可以从Master订阅消息，也可以从Slave订阅消息，消费者在向Master拉取消息时，Master服务器会根据拉取偏移量与最大偏移量的距离（判断是否读老消息，产生读I/O），以及从服务器是否可读等因素建议下一次是从Master还是Slave拉取。
+
+结合部署架构图，描述集群工作流程：
+
+- 启动NameServer，NameServer起来后监听端口，等待Broker、Producer、Consumer连上来，相当于一个路由控制中心。
+- Broker启动，跟所有的NameServer保持长连接，定时发送心跳包。心跳包中包含当前Broker信息(IP+端口等)以及存储所有Topic信息。注册成功后，NameServer集群中就有Topic跟Broker的映射关系。
+- 收发消息前，先创建Topic，创建Topic时需要指定该Topic要存储在哪些Broker上，也可以在发送消息时自动创建Topic。
+- Producer发送消息，启动时先跟NameServer集群中的其中一台建立长连接，并从NameServer中获取当前发送的Topic存在哪些Broker上，轮询从队列列表中选择一个队列，然后与队列所在的Broker建立长连接从而向Broker发消息。
+- Consumer跟Producer类似，跟其中一台NameServer建立长连接，获取当前订阅Topic存在哪些Broker上，然后直接跟Broker建立连接通道，开始消费消息。

docs/cn/best_practice.md

@@ -0,0 +1,369 @@
+#  最佳实践(best practice)
+
+
+## 1   生产者
+
+### 1.1 发送消息注意事项
+
+#### 1  Tags的使用
+
+一个应用尽可能用一个Topic，而消息子类型则可以用tags来标识。tags可以由应用自由设置，只有生产者在发送消息设置了tags，消费方在订阅消息时才可以利用tags通过broker做消息过滤：message.setTags("TagA")。  
+ 
+#### 2 Keys的使用
+
+每个消息在业务层面的唯一标识码要设置到keys字段，方便将来定位消息丢失问题。服务器会为每个消息创建索引（哈希索引），应用可以通过topic、key来查询这条消息内容，以及消息被谁消费。由于是哈希索引，请务必保证key尽可能唯一，这样可以避免潜在的哈希冲突。
+
+
+```java
+   // 订单Id   
+   String orderId = "20034568923546";   
+   message.setKeys(orderId);   
+```
+#### 3 日志的打印
+
+​消息发送成功或者失败要打印消息日志，务必要打印SendResult和key字段。send消息方法只要不抛异常，就代表发送成功。发送成功会有多个状态，在sendResult里定义。以下对每个状态进行说明：     
+
+- **SEND_OK**
+
+消息发送成功。要注意的是消息发送成功也不意味着它是可靠的。要确保不会丢失任何消息，还应启用同步Master服务器或同步刷盘，即SYNC_MASTER或SYNC_FLUSH。
+
+
+- **FLUSH_DISK_TIMEOUT**
+
+消息发送成功但是服务器刷盘超时。此时消息已经进入服务器队列（内存），只有服务器宕机，消息才会丢失。消息存储配置参数中可以设置刷盘方式和同步刷盘时间长度，如果Broker服务器设置了刷盘方式为同步刷盘，即FlushDiskType=SYNC_FLUSH（默认为异步刷盘方式），当Broker服务器未在同步刷盘时间内（默认为5s）完成刷盘，则将返回该状态——刷盘超时。
+
+- **FLUSH_SLAVE_TIMEOUT**
+
+消息发送成功，但是服务器同步到Slave时超时。此时消息已经进入服务器队列，只有服务器宕机，消息才会丢失。如果Broker服务器的角色是同步Master，即SYNC_MASTER（默认是异步Master即ASYNC_MASTER），并且从Broker服务器未在同步刷盘时间（默认为5秒）内完成与主服务器的同步，则将返回该状态——数据同步到Slave服务器超时。
+
+- **SLAVE_NOT_AVAILABLE**
+
+消息发送成功，但是此时Slave不可用。此时消息已经进入Master服务器队列，只有Master服务器宕机，消息才会丢失。如果Broker服务器的角色是同步Master，即SYNC_MASTER（默认是异步Master服务器即ASYNC_MASTER），但没有配置slave Broker服务器，则将返回该状态——无Slave服务器可用。
+
+
+### 1.2 消息发送失败处理方式
+
+Producer的send方法本身支持内部重试，重试逻辑如下：
+
+- 至多重试2次（同步发送为2次，异步发送为0次）。
+- 如果发送失败，则轮转到下一个Broker。这个方法的总耗时时间不超过sendMsgTimeout设置的值，默认10s。
+- 如果本身向broker发送消息产生超时异常，就不会再重试。
+
+以上策略也是在一定程度上保证了消息可以发送成功。如果业务对消息可靠性要求比较高，建议应用增加相应的重试逻辑：比如调用send同步方法发送失败时，则尝试将消息存储到db，然后由后台线程定时重试，确保消息一定到达Broker。
+
+上述db重试方式为什么没有集成到MQ客户端内部做，而是要求应用自己去完成，主要基于以下几点考虑：首先，MQ的客户端设计为无状态模式，方便任意的水平扩展，且对机器资源的消耗仅仅是cpu、内存、网络。其次，如果MQ客户端内部集成一个KV存储模块，那么数据只有同步落盘才能较可靠，而同步落盘本身性能开销较大，所以通常会采用异步落盘，又由于应用关闭过程不受MQ运维人员控制，可能经常会发生 kill -9 这样暴力方式关闭，造成数据没有及时落盘而丢失。第三，Producer所在机器的可靠性较低，一般为虚拟机，不适合存储重要数据。综上，建议重试过程交由应用来控制。
+
+### 1.3选择oneway形式发送
+通常消息的发送是这样一个过程：
+
+- 客户端发送请求到服务器
+- 服务器处理请求
+- 服务器向客户端返回应答
+
+所以，一次消息发送的耗时时间是上述三个步骤的总和，而某些场景要求耗时非常短，但是对可靠性要求并不高，例如日志收集类应用，此类应用可以采用oneway形式调用，oneway形式只发送请求不等待应答，而发送请求在客户端实现层面仅仅是一个操作系统系统调用的开销，即将数据写入客户端的socket缓冲区，此过程耗时通常在微秒级。
+
+
+## 2   消费者
+
+### 2.1 消费过程幂等
+
+RocketMQ无法避免消息重复（Exactly-Once），所以如果业务对消费重复非常敏感，务必要在业务层面进行去重处理。可以借助关系数据库进行去重。首先需要确定消息的唯一键，可以是msgId，也可以是消息内容中的唯一标识字段，例如订单Id等。在消费之前判断唯一键是否在关系数据库中存在。如果不存在则插入，并消费，否则跳过。（实际过程要考虑原子性问题，判断是否存在可以尝试插入，如果报主键冲突，则插入失败，直接跳过）
+
+msgId一定是全局唯一标识符，但是实际使用中，可能会存在相同的消息有两个不同msgId的情况（消费者主动重发、因客户端重投机制导致的重复等），这种情况就需要使业务字段进行重复消费。
+
+### 2.2 消费速度慢的处理方式
+
+#### 1 提高消费并行度
+
+绝大部分消息消费行为都属于 IO 密集型，即可能是操作数据库，或者调用 RPC，这类消费行为的消费速度在于后端数据库或者外系统的吞吐量，通过增加消费并行度，可以提高总的消费吞吐量，但是并行度增加到一定程度，反而会下降。所以，应用必须要设置合理的并行度。 如下有几种修改消费并行度的方法：
+
+- 同一个 ConsumerGroup 下，通过增加 Consumer 实例数量来提高并行度（需要注意的是超过订阅队列数的 Consumer 实例无效）。可以通过加机器，或者在已有机器启动多个进程的方式。
+- 提高单个 Consumer 的消费并行线程，通过修改参数 consumeThreadMin、consumeThreadMax实现。
+
+#### 2   批量方式消费
+
+某些业务流程如果支持批量方式消费，则可以很大程度上提高消费吞吐量，例如订单扣款类应用，一次处理一个订单耗时 1 s，一次处理 10 个订单可能也只耗时 2 s，这样即可大幅度提高消费的吞吐量，通过设置 consumer的 consumeMessageBatchMaxSize 返个参数，默认是 1，即一次只消费一条消息，例如设置为 N，那么每次消费的消息数小于等于 N。
+
+#### 3   跳过非重要消息
+
+发生消息堆积时，如果消费速度一直追不上发送速度，如果业务对数据要求不高的话，可以选择丢弃不重要的消息。例如，当某个队列的消息数堆积到100000条以上，则尝试丢弃部分或全部消息，这样就可以快速追上发送消息的速度。示例代码如下：
+
+```java
+    public ConsumeConcurrentlyStatus consumeMessage(
+            List<MessageExt> msgs,
+            ConsumeConcurrentlyContext context) {
+        long offset = msgs.get(0).getQueueOffset();
+        String maxOffset =
+                msgs.get(0).getProperty(Message.PROPERTY_MAX_OFFSET);
+        long diff = Long.parseLong(maxOffset) - offset;
+        if (diff > 100000) {
+            // TODO 消息堆积情况的特殊处理
+            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+        }
+        // TODO 正常消费过程
+        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+    }    
+```
+
+
+#### 4 优化每条消息消费过程     
+ 
+举例如下，某条消息的消费过程如下：
+
+- 根据消息从 DB 查询【数据 1】
+- 根据消息从 DB 查询【数据 2】
+- 复杂的业务计算
+- 向 DB 插入【数据 3】
+- 向 DB 插入【数据 4】
+
+这条消息的消费过程中有4次与 DB的 交互，如果按照每次 5ms 计算，那么总共耗时 20ms，假设业务计算耗时 5ms，那么总过耗时 25ms，所以如果能把 4 次 DB 交互优化为 2 次，那么总耗时就可以优化到 15ms，即总体性能提高了 40%。所以应用如果对时延敏感的话，可以把DB部署在SSD硬盘，相比于SCSI磁盘，前者的RT会小很多。
+
+### 2.3 消费打印日志
+
+如果消息量较少，建议在消费入口方法打印消息，消费耗时等，方便后续排查问题。
+
+
+```java
+   public ConsumeConcurrentlyStatus consumeMessage(
+            List<MessageExt> msgs,
+            ConsumeConcurrentlyContext context) {
+        log.info("RECEIVE_MSG_BEGIN: " + msgs.toString());
+        // TODO 正常消费过程
+        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+    }   
+```
+
+如果能打印每条消息消费耗时，那么在排查消费慢等线上问题时，会更方便。
+
+### 2.4 其他消费建议
+
+#### 1 关于消费者和订阅
+
+​第一件需要注意的事情是，不同的消费者组可以独立的消费一些 topic，并且每个消费者组都有自己的消费偏移量，请确保同一组内的每个消费者订阅信息保持一致。
+
+#### 2 关于有序消息
+
+消费者将锁定每个消息队列，以确保他们被逐个消费，虽然这将会导致性能下降，但是当你关心消息顺序的时候会很有用。我们不建议抛出异常，你可以返回 ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT 作为替代。
+
+#### 3 关于并发消费
+
+顾名思义，消费者将并发消费这些消息，建议你使用它来获得良好性能，我们不建议抛出异常，你可以返回 ConsumeConcurrentlyStatus.RECONSUME_LATER 作为替代。
+
+#### 4 关于消费状态Consume Status
+
+对于并发的消费监听器，你可以返回 RECONSUME_LATER 来通知消费者现在不能消费这条消息，并且希望可以稍后重新消费它。然后，你可以继续消费其他消息。对于有序的消息监听器，因为你关心它的顺序，所以不能跳过消息，但是你可以返回SUSPEND_CURRENT_QUEUE_A_MOMENT 告诉消费者等待片刻。
+
+#### 5 关于Blocking
+
+不建议阻塞监听器，因为它会阻塞线程池，并最终可能会终止消费进程
+
+#### 6 关于线程数设置     
+
+消费者使用 ThreadPoolExecutor 在内部对消息进行消费，所以你可以通过设置 setConsumeThreadMin 或 setConsumeThreadMax 来改变它。
+
+#### 7 关于消费位点
+
+当建立一个新的消费者组时，需要决定是否需要消费已经存在于 Broker 中的历史消息CONSUME_FROM_LAST_OFFSET 将会忽略历史消息，并消费之后生成的任何消息。CONSUME_FROM_FIRST_OFFSET 将会消费每个存在于 Broker 中的信息。你也可以使用 CONSUME_FROM_TIMESTAMP 来消费在指定时间戳后产生的消息。
+
+
+
+
+
+## 3   Broker
+
+### 3.1 Broker 角色
+​  Broker 角色分为 ASYNC_MASTER（异步主机）、SYNC_MASTER（同步主机）以及SLAVE（从机）。如果对消息的可靠性要求比较严格，可以采用 SYNC_MASTER加SLAVE的部署方式。如果对消息可靠性要求不高，可以采用ASYNC_MASTER加SLAVE的部署方式。如果只是测试方便，则可以选择仅ASYNC_MASTER或仅SYNC_MASTER的部署方式。
+### 3.2 FlushDiskType
+​ SYNC_FLUSH（同步刷新）相比于ASYNC_FLUSH（异步处理）会损失很多性能，但是也更可靠，所以需要根据实际的业务场景做好权衡。
+​ 
+​ 
+​ 
+​ 
+​ 
+​ 
+
+## 4  NameServer
+
+​RocketMQ 中，Name Servers 被设计用来做简单的路由管理。其职责包括：
+
+- Brokers 定期向每个名称服务器注册路由数据。
+- 名称服务器为客户端，包括生产者，消费者和命令行客户端提供最新的路由信息。
+​     
+​
+
+
+
+
+## 5 客户端配置
+
+​ 相对于RocketMQ的Broker集群，生产者和消费者都是客户端。本小节主要描述生产者和消费者公共的行为配置。
+
+### 5.1 客户端寻址方式
+
+RocketMQ可以令客户端找到Name Server, 然后通过Name Server再找到Broker。如下所示有多种配置方式，优先级由高到低，高优先级会覆盖低优先级。
+
+- 代码中指定Name Server地址，多个namesrv地址之间用分号分割   
+
+```java
+producer.setNamesrvAddr("192.168.0.1:9876;192.168.0.2:9876");  
+
+consumer.setNamesrvAddr("192.168.0.1:9876;192.168.0.2:9876");
+```
+- Java启动参数中指定Name Server地址
+
+```text
+-Drocketmq.namesrv.addr=192.168.0.1:9876;192.168.0.2:9876  
+```
+- 环境变量指定Name Server地址
+
+```text
+export   NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876   
+```
+- HTTP静态服务器寻址（默认）
+
+客户端启动后，会定时访问一个静态HTTP服务器，地址如下：<http://jmenv.tbsite.net:8080/rocketmq/nsaddr>，这个URL的返回内容如下：
+
+```text
+192.168.0.1:9876;192.168.0.2:9876   
+```
+客户端默认每隔2分钟访问一次这个HTTP服务器，并更新本地的Name Server地址。URL已经在代码中硬编码，可通过修改/etc/hosts文件来改变要访问的服务器，例如在/etc/hosts增加如下配置：
+```text
+10.232.22.67    jmenv.taobao.net   
+```
+推荐使用HTTP静态服务器寻址方式，好处是客户端部署简单，且Name Server集群可以热升级。
+
+### 5.2 客户端配置
+
+DefaultMQProducer、TransactionMQProducer、DefaultMQPushConsumer、DefaultMQPullConsumer都继承于ClientConfig类，ClientConfig为客户端的公共配置类。客户端的配置都是get、set形式，每个参数都可以用spring来配置，也可以在代码中配置，例如namesrvAddr这个参数可以这样配置，producer.setNamesrvAddr("192.168.0.1:9876")，其他参数同理。
+
+#### 1  客户端的公共配置
+
+| 参数名                        | 默认值  | 说明                                                         |
+| ----------------------------- | ------- | ------------------------------------------------------------ |
+| namesrvAddr                   |         | Name Server地址列表，多个NameServer地址用分号隔开            |
+| clientIP                      | 本机IP  | 客户端本机IP地址，某些机器会发生无法识别客户端IP地址情况，需要应用在代码中强制指定 |
+| instanceName                  | DEFAULT | 客户端实例名称，客户端创建的多个Producer、Consumer实际是共用一个内部实例（这个实例包含网络连接、线程资源等） |
+| clientCallbackExecutorThreads | 4       | 通信层异步回调线程数                                         |
+| pollNameServerInteval         | 30000   | 轮询Name Server间隔时间，单位毫秒                            |
+| heartbeatBrokerInterval       | 30000   | 向Broker发送心跳间隔时间，单位毫秒                           |
+| persistConsumerOffsetInterval | 5000    | 持久化Consumer消费进度间隔时间，单位毫秒                     |
+
+#### 2  Producer配置
+
+| 参数名                           | 默认值           | 说明                                                         |
+| -------------------------------- | ---------------- | ------------------------------------------------------------ |
+| producerGroup                    | DEFAULT_PRODUCER | Producer组名，多个Producer如果属于一个应用，发送同样的消息，则应该将它们归为同一组 |
+| createTopicKey                   | TBW102           | 在发送消息时，自动创建服务器不存在的topic，需要指定Key，该Key可用于配置发送消息所在topic的默认路由。 |
+| defaultTopicQueueNums            | 4                | 在发送消息，自动创建服务器不存在的topic时，默认创建的队列数  |
+| sendMsgTimeout                   | 10000            | 发送消息超时时间，单位毫秒                                   |
+| compressMsgBodyOverHowmuch       | 4096             | 消息Body超过多大开始压缩（Consumer收到消息会自动解压缩），单位字节 |
+| retryAnotherBrokerWhenNotStoreOK | FALSE            | 如果发送消息返回sendResult，但是sendStatus!=SEND_OK，是否重试发送 |
+| retryTimesWhenSendFailed         | 2                | 如果消息发送失败，最大重试次数，该参数只对同步发送模式起作用 |
+| maxMessageSize                   | 4MB              | 客户端限制的消息大小，超过报错，同时服务端也会限制，所以需要跟服务端配合使用。 |
+| transactionCheckListener         |                  | 事务消息回查监听器，如果发送事务消息，必须设置               |
+| checkThreadPoolMinSize           | 1                | Broker回查Producer事务状态时，线程池最小线程数                     |
+| checkThreadPoolMaxSize           | 1                | Broker回查Producer事务状态时，线程池最大线程数                     |
+| checkRequestHoldMax              | 2000             | Broker回查Producer事务状态时，Producer本地缓冲请求队列大小   |
+| RPCHook                          | null             | 该参数是在Producer创建时传入的，包含消息发送前的预处理和消息响应后的处理两个接口，用户可以在第一个接口中做一些安全控制或者其他操作。 |
+
+#### 3  PushConsumer配置
+
+| 参数名                       | 默认值                        | 说明                                                         |
+| ---------------------------- | ----------------------------- | ------------------------------------------------------------ |
+| consumerGroup                | DEFAULT_CONSUMER              | Consumer组名，多个Consumer如果属于一个应用，订阅同样的消息，且消费逻辑一致，则应该将它们归为同一组 |
+| messageModel                 | CLUSTERING                    | 消费模型支持集群消费和广播消费两种                           |
+| consumeFromWhere             | CONSUME_FROM_LAST_OFFSET      | Consumer启动后，默认从上次消费的位置开始消费，这包含两种情况：一种是上次消费的位置未过期，则消费从上次中止的位置进行；一种是上次消费位置已经过期，则从当前队列第一条消息开始消费 |
+| consumeTimestamp             | 半个小时前                    | 只有当consumeFromWhere值为CONSUME_FROM_TIMESTAMP时才起作用。 |
+| allocateMessageQueueStrategy | AllocateMessageQueueAveragely | Rebalance算法实现策略                                        |
+| subscription                 |                               | 订阅关系                                                     |
+| messageListener              |                               | 消息监听器                                                   |
+| offsetStore                  |                               | 消费进度存储                                                 |
+| consumeThreadMin             | 10                            | 消费线程池最小线程数                                               |
+| consumeThreadMax             | 20                            | 消费线程池最大线程数                                               |
+|                              |                               |                                                              |
+| consumeConcurrentlyMaxSpan   | 2000                          | 单队列并行消费允许的最大跨度                                 |
+| pullThresholdForQueue        | 1000                          | 拉消息本地队列缓存消息最大数                                 |
+| pullInterval                 | 0                             | 拉消息间隔，由于是长轮询，所以为0，但是如果应用为了流控，也可以设置大于0的值，单位毫秒 |
+| consumeMessageBatchMaxSize   | 1                             | 批量消费，一次消费多少条消息                                 |
+| pullBatchSize                | 32                            | 批量拉消息，一次最多拉多少条                                 |
+
+#### 4  PullConsumer配置
+
+| 参数名                           | 默认值                        | 说明                                                         |
+| -------------------------------- | ----------------------------- | ------------------------------------------------------------ |
+| consumerGroup                    | DEFAULT_CONSUMER              | Consumer组名，多个Consumer如果属于一个应用，订阅同样的消息，且消费逻辑一致，则应该将它们归为同一组 |
+| brokerSuspendMaxTimeMillis       | 20000                         | 长轮询，Consumer拉消息请求在Broker挂起最长时间，单位毫秒     |
+| consumerTimeoutMillisWhenSuspend | 30000                         | 长轮询，Consumer拉消息请求在Broker挂起超过指定时间，客户端认为超时，单位毫秒 |
+| consumerPullTimeoutMillis        | 10000                         | 非长轮询，拉消息超时时间，单位毫秒                           |
+| messageModel                     | BROADCASTING                  | 消息支持两种模式：集群消费和广播消费           |
+| messageQueueListener             |                               | 监听队列变化                                                 |
+| offsetStore                      |                               | 消费进度存储                                                 |
+| registerTopics                   |                               | 注册的topic集合                                              |
+| allocateMessageQueueStrategy     | AllocateMessageQueueAveragely | Rebalance算法实现策略                                        |
+
+#### 5  Message数据结构
+
+| 字段名         | 默认值 | 说明                                                         |
+| -------------- | ------ | ------------------------------------------------------------ |
+| Topic          | null   | 必填，消息所属topic的名称                                        |
+| Body           | null   | 必填，消息体                                                 |
+| Tags           | null   | 选填，消息标签，方便服务器过滤使用。目前只支持每个消息设置一个tag |
+| Keys           | null   | 选填，代表这条消息的业务关键词，服务器会根据keys创建哈希索引，设置后，可以在Console系统根据Topic、Keys来查询消息，由于是哈希索引，请尽可能保证key唯一，例如订单号，商品Id等。 |
+| Flag           | 0      | 选填，完全由应用来设置，RocketMQ不做干预                     |
+| DelayTimeLevel | 0      | 选填，消息延时级别，0表示不延时，大于0会延时特定的时间才会被消费 |
+| WaitStoreMsgOK | TRUE   | 选填，表示消息是否在服务器落盘后才返回应答。                 |
+
+
+​
+​
+
+## 6  系统配置
+
+本小节主要介绍系统（JVM/OS）相关的配置。
+
+### 6.1 JVM选项
+
+​ 推荐使用最新发布的JDK 1.8版本。通过设置相同的Xms和Xmx值来防止JVM调整堆大小以获得更好的性能。简单的JVM配置如下所示：
+​  
+​```
+​
+​-server -Xms8g -Xmx8g -Xmn4g   
+​ ```
+​ 
+​  
+如果您不关心RocketMQ Broker的启动时间，还有一种更好的选择，就是通过“预触摸”Java堆以确保在JVM初始化期间每个页面都将被分配。那些不关心启动时间的人可以启用它：
+​ -XX:+AlwaysPreTouch   
+禁用偏置锁定可能会减少JVM暂停，
+​ -XX:-UseBiasedLocking   
+至于垃圾回收，建议使用带JDK 1.8的G1收集器。
+
+```text
+-XX:+UseG1GC -XX:G1HeapRegionSize=16m   
+-XX:G1ReservePercent=25 
+-XX:InitiatingHeapOccupancyPercent=30
+```
+
+​ 这些GC选项看起来有点激进，但事实证明它在我们的生产环境中具有良好的性能。另外不要把-XX:MaxGCPauseMillis的值设置太小，否则JVM将使用一个小的年轻代来实现这个目标，这将导致非常频繁的minor GC，所以建议使用rolling GC日志文件：
+
+```text
+-XX:+UseGCLogFileRotation   
+-XX:NumberOfGCLogFiles=5 
+-XX:GCLogFileSize=30m
+```
+
+如果写入GC文件会增加代理的延迟，可以考虑将GC日志文件重定向到内存文件系统：
+
+```text
+-Xloggc:/dev/shm/mq_gc_%p.log123   
+```
+### 6.2 Linux内核参数
+
+​ os.sh脚本在bin文件夹中列出了许多内核参数，可以进行微小的更改然后用于生产用途。下面的参数需要注意，更多细节请参考/proc/sys/vm/*的[文档](https://www.kernel.org/doc/Documentation/sysctl/vm.txt)
+
+- **vm.extra_free_kbytes**，告诉VM在后台回收（kswapd）启动的阈值与直接回收（通过分配进程）的阈值之间保留额外的可用内存。RocketMQ使用此参数来避免内存分配中的长延迟。（与具体内核版本相关）
+- **vm.min_free_kbytes**，如果将其设置为低于1024KB，将会巧妙的将系统破坏，并且系统在高负载下容易出现死锁。
+- **vm.max_map_count**，限制一个进程可能具有的最大内存映射区域数。RocketMQ将使用mmap加载CommitLog和ConsumeQueue，因此建议将为此参数设置较大的值。（agressiveness --> aggressiveness）
+- **vm.swappiness**，定义内核交换内存页面的积极程度。较高的值会增加攻击性，较低的值会减少交换量。建议将值设置为10来避免交换延迟。
+- **File descriptor limits**，RocketMQ需要为文件（CommitLog和ConsumeQueue）和网络连接打开文件描述符。我们建议设置文件描述符的值为655350。
+- [Disk scheduler](https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Performance_Tuning_Guide/ch06s04s02.html)，RocketMQ建议使用I/O截止时间调度器，它试图为请求提供有保证的延迟。
+[]([]())

docs/cn/concept.md

@@ -0,0 +1,39 @@
+# 基本概念
+## 消息模型（Message Model）
+
+<img src="./images/message_model_1.jpg" style="width: 560px"/>
+
+RocketMQ 消息模型如图1所示，主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。
+##  消息生产者（Producer）
+ 负责生产消息，一般由业务系统负责生产消息。一个消息生产者会把业务应用系统里产生的消息发送到broker服务器。RocketMQ提供多种发送方式，同步发送、异步发送、顺序发送、单向发送。同步和异步方式均需要Broker返回确认信息，单向发送不需要。
+## 消息消费者（Consumer）
+ 负责消费消息，一般是后台系统负责异步消费。一个消息消费者会从Broker服务器拉取消息、并将其提供给应用程序。从用户应用的角度而言提供了两种消费形式：拉取式消费、推动式消费。
+## 主题（Topic）
+  表示一类消息的集合，每个主题包含若干条消息，每条消息只能属于一个主题，是RocketMQ进行消息订阅的基本单位。
+##代理服务器（Broker Server）
+消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。
+## 名字服务（Name Server）
+ 名称服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。
+## 拉取式消费（Pull Consumer）
+  Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。
+## 推动式消费（Push Consumer）
+ Consumer消费的一种类型，该模式下Broker收到数据后会主动推送给消费端，该消费模式一般实时性较高。
+## 生产者组（Producer Group）
+  同一类Producer的集合，这类Producer发送同一类消息且发送逻辑一致。如果发送的是事物消息且原始生产者在发送之后崩溃，则Broker服务器会联系同一生产者组的其他生产者实例以提交或回溯消费。
+## 消费者组（Consumer Group）
+  同一类Consumer的集合，这类Consumer通常消费同一类消息且消费逻辑一致。消费者组使得在消息消费方面，实现负载均衡和容错的目标变得非常容易。要注意的是，消费者组的消费者实例必须订阅完全相同的Topic。RocketMQ 支持两种消息模式：集群消费（Clustering）和广播消费（Broadcasting）。
+## 集群消费（Clustering）
+集群消费模式下,相同Consumer Group的每个Consumer实例平均分摊消息。
+## 广播消费（Broadcasting）
+广播消费模式下，相同Consumer Group的每个Consumer实例都接收全量的消息。
+## 普通顺序消息（Normal Ordered Message）
+普通顺序消费模式下，消费者通过同一个消费队列收到的消息是有顺序的，不同消息队列收到的消息则可能是无顺序的。
+## 严格顺序消息（Strictly Ordered Message）
+严格顺序消息模式下，消费者收到的所有消息均是有顺序的。
+## 代理服务器（Broker Server）
+消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。
+## 消息（Message）
+消息系统所传输信息的物理载体，生产和消费数据的最小单位，每条消息必须属于一个主题。RocketMQ中每个消息拥有唯一的Message ID，且可以携带具有业务标识的Key。系统提供了通过Message ID和Key查询消息的功能。
+## 标签（Tag）
+ 为消息设置的标志，用于同一主题下区分不同类型的消息。来自同一业务单元的消息，可以根据不同业务目的在同一主题下设置不同标签。标签能够有效地保持代码的清晰度和连贯性，并优化RocketMQ提供的查询系统。消费者可以根据Tag实现对不同子主题的不同消费逻辑，实现更好的扩展性。
+

docs/cn/dledger/deploy_guide.md

@@ -0,0 +1,76 @@
+## 前言
+该文档主要介绍如何部署自动容灾切换的 RocketMQ-on-DLedger Group。
+
+RocketMQ-on-DLedger Group 是指一组**相同名称**的 Broker，至少需要 3 个节点，通过 Raft 自动选举出一个 Leader，其余节点 作为 Follower，并在 Leader 和 Follower 之间复制数据以保证高可用。  
+RocketMQ-on-DLedger Group 能自动容灾切换，并保证数据一致。  
+RocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多个 RocketMQ-on-DLedger Group 同时对外提供服务。  
+
+## 1. 新集群部署
+
+#### 1.1 编写配置
+每个 RocketMQ-on-DLedger Group 至少准备三台机器（本文假设为 3）。  
+编写 3 个配置文件，建议参考 conf/dledger 目录下的配置文件样例。  
+关键配置介绍：  
+
+| name | 含义 | 举例 |
+| --- | --- | --- |
+| enableDLegerCommitLog | 是否启动 DLedger  | true |
+| dLegerGroup | DLedger Raft Group的名字，建议和 brokerName 保持一致 | RaftNode00 |
+| dLegerPeers | DLedger Group 内各节点的端口信息，同一个 Group 内的各个节点配置必须要保证一致 | n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 |
+| dLegerSelfId | 节点 id, 必须属于 dLegerPeers 中的一个；同 Group 内各个节点要唯一 | n0 |
+| sendMessageThreadPoolNums | 发送线程个数，建议配置成 Cpu 核数 | 16 |
+
+这里贴出 conf/dledger/broker-n0.conf 的配置举例。  
+
+```
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30911
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node00
+storePathCommitLog=/tmp/rmqstore/node00/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n0
+sendMessageThreadPoolNums=16
+```
+
+### 1.2 启动 Broker
+
+与老版本的启动方式一致。
+
+`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & `  
+`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & `  
+`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & `  
+
+
+## 2. 旧集群升级
+
+如果旧集群采用 Master 方式部署，则每个 Master 都需要转换成一个 RocketMQ-on-DLedger Group。  
+如果旧集群采用 Master-Slave 方式部署，则每个 Master-Slave 组都需要转换成一个 RocketMQ-on-DLedger Group。
+
+### 2.1 杀掉旧的 Broker
+
+可以通过 kill 命令来完成，也可以调用 `bin/mqshutdown broker`。
+
+### 2.2 检查旧的 Commitlog
+
+RocketMQ-on-DLedger 组中的每个节点，可以兼容旧的 Commitlog ，但其 Raft 复制过程，只能针对新增加的消息。因此，为了避免出现异常，需要保证 旧的 Commitlog 是一致的。  
+如果旧的集群是采用 Master-Slave 方式部署，有可能在shutdown时，其数据并不是一致的，建议通过md5sum 的方式，检查最近的最少 2 个 Commmitlog 文件，如果发现不一致，则通过拷贝的方式进行对齐。  
+
+虽然 RocketMQ-on-DLedger Group 也可以以 2 节点方式部署，但其会丧失容灾切换能力（2n + 1 原则，至少需要3个节点才能容忍其中 1 个宕机）。  
+所以在对齐了 Master 和 Slave 的 Commitlog 之后，还需要准备第 3 台机器，并把旧的 Commitlog 从 Master 拷贝到 第 3 台机器（记得同时拷贝一下 config 文件夹）。  
+   
+在 3 台机器准备好了之后，旧 Commitlog 文件也保证一致之后，就可以开始走下一步修改配置了。
+
+### 2.3 修改配置
+
+参考新集群部署。
+
+### 2.4 重新启动 Broker 
+
+参考新集群部署。
+
+

docs/cn/dledger/quick_start.md

@@ -0,0 +1,61 @@
+### 前言
+该文档主要介绍如何快速构建和部署基于 DLedger 的可以自动容灾切换的 RocketMQ 集群。
+
+详细的新集群部署和旧集群升级指南请参考 [部署指南](deploy_guide.md)。
+
+### 1. 源码构建
+构建分为两个部分，需要先构建 DLedger，然后 构建 RocketMQ
+
+#### 1.1 构建 DLedger
+
+`git clone https://github.com/openmessaging/openmessaging-storage-dledger.git`
+
+`cd openmessaging-storage-dledger`
+
+`mvn clean install -DskipTests`
+
+#### 1.2 构建 RocketMQ
+
+`git clone https://github.com/apache/rocketmq.git`
+
+`cd rocketmq`
+
+`git checkout -b store_with_dledger origin/store_with_dledger`
+
+`mvn -Prelease-all -DskipTests clean install -U`
+
+### 2. 快速部署
+
+在构建成功后
+
+`cd distribution/target/apache-rocketmq`
+
+`sh bin/dledger/fast-try.sh start`
+
+如果上面的步骤执行成功，可以通过 mqadmin 运维命令查看集群状态。
+
+`sh bin/mqadmin clusterList -n 127.0.0.1:9876`
+
+顺利的话，会看到如下内容：
+
+![ClusterList](https://img.alicdn.com/5476e8b07b923/TB11Z.ZyCzqK1RjSZFLXXcn2XXa)
+
+（BID 为 0 的表示 Master，其余都是 Follower）
+
+启动成功，现在可以向集群收发消息，并进行容灾切换测试了。
+
+关闭快速集群，可以执行：
+
+`sh bin/dledger/fast-try.sh stop`
+
+快速部署，默认配置在 conf/dledger 里面，默认的存储路径在 /tmp/rmqstore。
+
+
+### 3. 容灾切换
+
+部署成功，杀掉 Leader 之后（在上面的例子中，杀掉端口 30931 所在的进程），等待约 10s 左右，用 clusterList 命令查看集群，就会发现 Leader 切换到另一个节点了。
+
+
+
+
+

docs/cn/msg_trace/user_guide.md

@@ -0,0 +1,106 @@
+# 消息轨迹
+## 前言
+该文档主要介绍如何快速部署和使用支持消息轨迹特性的RocketMQ 集群。
+
+## 1. 消息轨迹数据关键属性
+| Producer端| Consumer端 | Broker端 |
+| --- | --- | --- |
+| 生产实例信息 | 消费实例信息 | 消息的Topic |
+| 发送消息时间 | 投递时间,投递轮次  | 消息存储位置 |
+| 消息是否发送成功 | 消息是否消费成功 | 消息的Key值 |
+| 发送耗时 | 消费耗时 | 消息的Tag值 |
+
+## 2. 支持消息轨迹集群部署
+
+### 2.1 Broker端配置文件
+这里贴出Broker端开启消息轨迹特性的properties配置文件内容：
+```
+brokerClusterName=DefaultCluster
+brokerName=broker-a
+brokerId=0
+deleteWhen=04
+fileReservedTime=48
+brokerRole=ASYNC_MASTER
+flushDiskType=ASYNC_FLUSH
+storePathRootDir=/data/rocketmq/rootdir-a-m
+storePathCommitLog=/data/rocketmq/commitlog-a-m
+autoCreateSubscriptionGroup=true
+## if msg tracing is open,the flag will be true
+traceTopicEnable=true
+listenPort=10911
+brokerIP1=XX.XX.XX.XX1
+namesrvAddr=XX.XX.XX.XX:9876
+```
+
+### 2.2 普通模式
+RocketMQ集群中每一个Broker节点均用于存储Client端收集并发送过来的消息轨迹数据。因此，对于RocketMQ集群中的Broker节点数量并无要求和限制。
+
+### 2.3 物理IO隔离模式
+对于消息轨迹数据量较大的场景，可以在RocketMQ集群中选择其中一个Broker节点专用于存储消息轨迹，使得用户普通的消息数据与消息轨迹数据的物理IO完全隔离，互不影响。在该模式下，RockeMQ集群中至少有两个Broker节点，其中一个Broker节点定义为存储消息轨迹数据的服务端。
+
+### 2.4 启动开启消息轨迹的Broker
+`nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &`
+  
+## 3. 保存消息轨迹的Topic定义
+RocketMQ的消息轨迹特性支持两种存储轨迹数据的方式：
+
+### 3.1 系统级的TraceTopic
+在默认情况下，消息轨迹数据是存储于系统级的TraceTopic中(其名称为：**RMQ_SYS_TRACE_TOPIC**)。该Topic在Broker节点启动时，会自动创建出来（如上所叙，需要在Broker端的配置文件中将**traceTopicEnable**的开关变量设置为**true**）。
+
+### 3.2 用户自定义的TraceTopic 
+如果用户不准备将消息轨迹的数据存储于系统级的默认TraceTopic，也可以自己定义并创建用户级的Topic来保存轨迹（即为创建普通的Topic用于保存消息轨迹数据）。下面一节会介绍Client客户端的接口如何支持用户自定义的TraceTopic。
+
+## 4. 支持消息轨迹的Client客户端实践
+为了尽可能地减少用户业务系统使用RocketMQ消息轨迹特性的改造工作量，作者在设计时候采用对原来接口增加一个开关参数(**enableMsgTrace**)来实现消息轨迹是否开启；并新增一个自定义参(**customizedTraceTopic**)数来实现用户存储消息轨迹数据至自己创建的用户级Topic。
+
+### 4.1 发送消息时开启消息轨迹
+```
+        DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName",true);
+        producer.setNamesrvAddr("XX.XX.XX.XX1");
+        producer.start();
+            try {
+                {
+                    Message msg = new Message("TopicTest",
+                        "TagA",
+                        "OrderID188",
+                        "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+                    SendResult sendResult = producer.send(msg);
+                    System.out.printf("%s%n", sendResult);
+                }
+
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+```
+
+### 4.2 订阅消息时开启消息轨迹
+```
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true);
+        consumer.subscribe("TopicTest", "*");
+        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+        consumer.setConsumeTimestamp("20181109221800");
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        consumer.start();
+        System.out.printf("Consumer Started.%n");
+```
+
+### 4.3 支持自定义存储消息轨迹Topic
+在上面的发送和订阅消息时候分别将DefaultMQProducer和DefaultMQPushConsumer实例的初始化修改为如下即可支持自定义存储消息轨迹Topic。
+```
+        ##其中Topic_test11111需要用户自己预先创建，来保存消息轨迹；
+        DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName",true,"Topic_test11111");
+        ......
+
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true,"Topic_test11111");
+        ......
+```
+
+
+
+

example/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -51,7 +51,12 @@
         <dependency>
             <groupId>org.apache.rocketmq</groupId>
             <artifactId>rocketmq-openmessaging</artifactId>
-            <version>4.4.0-SNAPSHOT</version>
+            <version>4.4.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-acl</artifactId>
+            <version>4.4.1-SNAPSHOT</version>
         </dependency>
     </dependencies>
 </project>

example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterProducer.java

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.rocketmq.example.filter;
 
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
@@ -21,8 +22,10 @@ import org.apache.rocketmq.client.producer.SendResult;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 
-public class Producer {
+public class SqlFilterProducer {
+
     public static void main(String[] args) throws Exception {
+
         DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
 
         producer.start();
@@ -30,7 +33,7 @@ public class Producer {
         String[] tags = new String[] {"TagA", "TagB", "TagC"};
 
         for (int i = 0; i < 10; i++) {
-            Message msg = new Message("TagFilterTest",
+            Message msg = new Message("SqlFilterTest",
                 tags[i % tags.length],
                 ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)
             );

example/src/main/java/org/apache/rocketmq/example/filter/TagFilterConsumer.java

@@ -25,7 +25,7 @@ import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.common.message.MessageExt;
 
-public class Consumer {
+public class TagFilterConsumer {
 
     public static void main(String[] args) throws InterruptedException, MQClientException, IOException {
         DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name");

example/src/main/java/org/apache/rocketmq/example/filter/TagFilterProducer.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.filter;
+
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+public class TagFilterProducer {
+
+    public static void main(String[] args) throws Exception {
+
+        DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
+        producer.start();
+
+        String[] tags = new String[] {"TagA", "TagB", "TagC"};
+
+        for (int i = 0; i < 60; i++) {
+            Message msg = new Message("TagFilterTest",
+                tags[i % tags.length],
+                "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+
+            SendResult sendResult = producer.send(msg);
+            System.out.printf("%s%n", sendResult);
+        }
+
+        producer.shutdown();
+    }
+}

example/src/main/java/org/apache/rocketmq/example/simple/AclClient.java

@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.simple;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+
+public class AclClient {
+
+    private static final Map<MessageQueue, Long> OFFSE_TABLE = new HashMap<MessageQueue, Long>();
+
+    private static final String ACL_ACCESS_KEY = "RocketMQ";
+
+    private static final String ACL_SECRET_KEY = "1234567";
+
+    public static void main(String[] args) throws MQClientException, InterruptedException {
+        producer();
+        pushConsumer();
+        pullConsumer();
+    }
+
+    public static void producer() throws MQClientException {
+        DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName", getAclRPCHook());
+        producer.setNamesrvAddr("127.0.0.1:9876");
+        producer.start();
+
+        for (int i = 0; i < 128; i++)
+            try {
+                {
+                    Message msg = new Message("TopicTest",
+                        "TagA",
+                        "OrderID188",
+                        "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+                    SendResult sendResult = producer.send(msg);
+                    System.out.printf("%s%n", sendResult);
+                }
+
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+
+        producer.shutdown();
+    }
+
+    public static void pushConsumer() throws MQClientException {
+
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_5", getAclRPCHook(), new AllocateMessageQueueAveragely());
+        consumer.setNamesrvAddr("127.0.0.1:9876");
+        consumer.subscribe("TopicTest", "*");
+        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+        // Wrong time format 2017_0422_221800
+        consumer.setConsumeTimestamp("20180422221800");
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+                printBody(msgs);
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        consumer.start();
+        System.out.printf("Consumer Started.%n");
+    }
+
+    public static void pullConsumer() throws MQClientException {
+        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_6", getAclRPCHook());
+        consumer.setNamesrvAddr("127.0.0.1:9876");
+        consumer.start();
+
+        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues("TopicTest");
+        for (MessageQueue mq : mqs) {
+            System.out.printf("Consume from the queue: %s%n", mq);
+            SINGLE_MQ:
+            while (true) {
+                try {
+                    PullResult pullResult =
+                        consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32);
+                    System.out.printf("%s%n", pullResult);
+                    putMessageQueueOffset(mq, pullResult.getNextBeginOffset());
+                    printBody(pullResult);
+                    switch (pullResult.getPullStatus()) {
+                        case FOUND:
+                            break;
+                        case NO_MATCHED_MSG:
+                            break;
+                        case NO_NEW_MSG:
+                            break SINGLE_MQ;
+                        case OFFSET_ILLEGAL:
+                            break;
+                        default:
+                            break;
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+
+        consumer.shutdown();
+    }
+
+    private static void printBody(PullResult pullResult) {
+        printBody(pullResult.getMsgFoundList());
+    }
+
+    private static void printBody(List<MessageExt> msg) {
+        if (msg == null || msg.size() == 0)
+            return;
+        for (MessageExt m : msg) {
+            if (m != null) {
+                System.out.printf("msgId : %s  body : %s  \n\r", m.getMsgId(), new String(m.getBody()));
+            }
+        }
+    }
+
+    private static long getMessageQueueOffset(MessageQueue mq) {
+        Long offset = OFFSE_TABLE.get(mq);
+        if (offset != null)
+            return offset;
+
+        return 0;
+    }
+
+    private static void putMessageQueueOffset(MessageQueue mq, long offset) {
+        OFFSE_TABLE.put(mq, offset);
+    }
+
+    static RPCHook getAclRPCHook() {
+        return new AclClientRPCHook(new SessionCredentials(ACL_ACCESS_KEY,ACL_SECRET_KEY));
+    }
+}

example/src/main/java/org/apache/rocketmq/example/simple/Producer.java

@@ -26,7 +26,6 @@ public class Producer {
     public static void main(String[] args) throws MQClientException, InterruptedException {
 
         DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName");
-
         producer.start();
 
         for (int i = 0; i < 128; i++)

example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java

@@ -29,10 +29,10 @@ public class PullConsumer {
 
     public static void main(String[] args) throws MQClientException {
         DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5");
-
+        consumer.setNamesrvAddr("127.0.0.1:9876");
         consumer.start();
 
-        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues("TopicTest1");
+        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues("broker-a");
         for (MessageQueue mq : mqs) {
             System.out.printf("Consume from the queue: %s%n", mq);
             SINGLE_MQ:

example/src/main/java/org/apache/rocketmq/example/simple/PullConsumerTest.java

@@ -24,6 +24,7 @@ import org.apache.rocketmq.common.message.MessageQueue;
 public class PullConsumerTest {
     public static void main(String[] args) throws MQClientException {
         DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5");
+        consumer.setNamesrvAddr("127.0.0.1:9876");
         consumer.start();
 
         try {

example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java

@@ -29,10 +29,10 @@ public class PushConsumer {
 
     public static void main(String[] args) throws InterruptedException, MQClientException {
         DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1");
-        consumer.subscribe("Jodie_topic_1023", "*");
+        consumer.subscribe("TopicTest", "*");
         consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
         //wrong time format 2017_0422_221800
-        consumer.setConsumeTimestamp("20170422221800");
+        consumer.setConsumeTimestamp("20181109221800");
         consumer.registerMessageListener(new MessageListenerConcurrently() {
 
             @Override

example/src/main/java/org/apache/rocketmq/example/tracemessage/TraceProducer.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.example.filter;
+package org.apache.rocketmq.example.tracemessage;
 
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
@@ -23,45 +23,27 @@ import org.apache.rocketmq.client.producer.SendResult;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 
-public class SqlProducer {
+public class TraceProducer {
+    public static void main(String[] args) throws MQClientException, InterruptedException {
 
-    public static void main(String[] args) {
-        DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
-        try {
-            producer.start();
-        } catch (MQClientException e) {
-            e.printStackTrace();
-            return;
-        }
+        DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName", true);
+        producer.start();
 
-        for (int i = 0; i < 10; i++) {
+        for (int i = 0; i < 128; i++)
             try {
-                String tag;
-                int div = i % 3;
-                if (div == 0) {
-                    tag = "TagA";
-                } else if (div == 1) {
-                    tag = "TagB";
-                } else {
-                    tag = "TagC";
+                {
+                    Message msg = new Message("TopicTest",
+                        "TagA",
+                        "OrderID188",
+                        "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
+                    SendResult sendResult = producer.send(msg);
+                    System.out.printf("%s%n", sendResult);
                 }
-                Message msg = new Message("TopicTest",
-                    tag,
-                    ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)
-                );
-                msg.putUserProperty("a", String.valueOf(i));
 
-                SendResult sendResult = producer.send(msg);
-                System.out.printf("%s%n", sendResult);
             } catch (Exception e) {
                 e.printStackTrace();
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e1) {
-                    e1.printStackTrace();
-                }
             }
-        }
+
         producer.shutdown();
     }
 }

example/src/main/java/org/apache/rocketmq/example/tracemessage/TracePushConsumer.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.example.tracemessage;
+
+import java.util.List;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.MessageExt;
+
+public class TracePushConsumer {
+    public static void main(String[] args) throws InterruptedException, MQClientException {
+        // Here,we use the default message track trace topic name
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true);
+        consumer.subscribe("TopicTest", "*");
+        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+        // Wrong time format 2017_0422_221800
+        consumer.setConsumeTimestamp("20181109221800");
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        consumer.start();
+        System.out.printf("Consumer Started.%n");
+    }
+}

filter/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

logappender/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>rocketmq-logappender</artifactId>

logging/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

namesrv/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java

@@ -181,6 +181,17 @@ public class RouteInfoManager {
                     brokerData = new BrokerData(clusterName, brokerName, new HashMap<Long, String>());
                     this.brokerAddrTable.put(brokerName, brokerData);
                 }
+                Map<Long, String> brokerAddrsMap = brokerData.getBrokerAddrs();
+                //Switch slave to master: first remove <1, IP:PORT> in namesrv, then add <0, IP:PORT>
+                //The same IP:PORT must only have one record in brokerAddrTable
+                Iterator<Entry<Long, String>> it = brokerAddrsMap.entrySet().iterator();
+                while (it.hasNext()) {
+                    Entry<Long, String> item = it.next();
+                    if (null != brokerAddr && brokerAddr.equals(item.getValue()) && brokerId != item.getKey()) {
+                        it.remove();
+                    }
+                }
+
                 String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr);
                 registerFirst = registerFirst || (null == oldAddr);
 

openmessaging/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

pom.xml

@@ -30,7 +30,7 @@
     <inceptionYear>2012</inceptionYear>
     <groupId>org.apache.rocketmq</groupId>
     <artifactId>rocketmq-all</artifactId>
-    <version>4.4.0-SNAPSHOT</version>
+    <version>4.4.1-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>Apache RocketMQ ${project.version}</name>
     <url>http://rocketmq.apache.org/</url>
@@ -217,9 +217,6 @@
                     <execution>
                         <id>generate-effective-dependencies-pom</id>
                         <phase>generate-resources</phase>
-                        <goals>
-                            <goal>effective-pom</goal>
-                        </goals>
                         <configuration>
                             <output>${project.build.directory}/effective-pom/effective-dependencies.xml</output>
                         </configuration>
@@ -263,6 +260,7 @@
                         <exclude>src/main/resources/META-INF/service/*</exclude>
                         <exclude>*/target/**</exclude>
                         <exclude>*/*.iml</exclude>
+                        <exclude>docs/**</exclude>
                     </excludes>
                 </configuration>
             </plugin>
@@ -458,7 +456,7 @@
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
-            <version>2.6.3</version>
+            <version>2.24.0</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -600,6 +598,16 @@
                 <artifactId>log4j</artifactId>
                 <version>1.2.17</version>
             </dependency>
+            <dependency>
+                <groupId>org.yaml</groupId>
+                <artifactId>snakeyaml</artifactId>
+                <version>1.19</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-codec</groupId>
+                <artifactId>commons-codec</artifactId>
+                <version>1.9</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.logging.log4j</groupId>
                 <artifactId>log4j-core</artifactId>

remoting/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java

@@ -57,6 +57,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
 import org.apache.rocketmq.remoting.util.ThreadUtils;
 
+
 public abstract class NettyRemotingAbstract {
 
     /**
@@ -108,6 +109,8 @@ public abstract class NettyRemotingAbstract {
      */
     protected volatile SslContext sslContext;
 
+
+
     static {
         NettyLogger.initNettyLogger();
     }
@@ -183,6 +186,7 @@ public abstract class NettyRemotingAbstract {
         }
     }
 
+
     /**
      * Process incoming request command issued by remote peer.
      *
@@ -341,10 +345,12 @@ public abstract class NettyRemotingAbstract {
         }
     }
 
+
+
     /**
      * Custom interceptor hook.
      *
-     * @return RPC hook if specified; null otherwise.
+     * @return Interceptor hooks if specified; null otherwise.
      */
     public abstract InterceptorGroup getInterceptorGroup();
 

remoting/src/main/java/org/apache/rocketmq/remoting/transport/rocketmq/NettyRemotingServer.java

@@ -39,6 +39,9 @@ import java.net.InetSocketAddress;
 import java.security.cert.CertificateException;
 import java.util.NoSuchElementException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.ChannelEventListener;
@@ -75,7 +78,6 @@ public class NettyRemotingServer extends NettyRemotingServerAbstract implements
     private ChannelEventListener channelEventListener;
     private DefaultEventExecutorGroup defaultEventExecutorGroup;
     private Class<? extends ServerSocketChannel> socketChannelClass;
-
     private int port = 0;
     private InterceptorGroup interceptorGroup;
 
@@ -108,20 +110,57 @@ public class NettyRemotingServer extends NettyRemotingServerAbstract implements
         if (publicThreadNums <= 0) {
             publicThreadNums = 4;
         }
-        this.publicExecutor = ThreadUtils.newFixedThreadPool(
-            publicThreadNums,
-            10000, "Remoting-PublicExecutor", true);
+
+        this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactory() {
+            private AtomicInteger threadIndex = new AtomicInteger(0);
+
+            @Override
+            public Thread newThread(Runnable r) {
+                return new Thread(r, "NettyServerPublicExecutor_" + this.threadIndex.incrementAndGet());
+            }
+        });
+
         if (JvmUtils.isUseEpoll() && this.nettyServerConfig.isUseEpollNativeSelector()) {
-            this.eventLoopGroupSelector = new EpollEventLoopGroup(serverConfig.getServerSelectorThreads(), ThreadUtils.newGenericThreadFactory("NettyEpollIoThreads",
-                serverConfig.getServerSelectorThreads()));
-            this.eventLoopGroupBoss = new EpollEventLoopGroup(serverConfig.getServerAcceptorThreads(), ThreadUtils.newGenericThreadFactory("NettyBossThreads",
-                serverConfig.getServerAcceptorThreads()));
+            this.eventLoopGroupBoss = new EpollEventLoopGroup(1, new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyEPOLLBoss_%d", this.threadIndex.incrementAndGet()));
+                }
+            });
+
+            this.eventLoopGroupSelector = new EpollEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+                private int threadTotal = nettyServerConfig.getServerSelectorThreads();
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyServerEPOLLSelector_%d_%d", threadTotal, this.threadIndex.incrementAndGet()));
+                }
+            });
             this.socketChannelClass = EpollServerSocketChannel.class;
+
         } else {
-            this.eventLoopGroupBoss = new NioEventLoopGroup(serverConfig.getServerAcceptorThreads(), ThreadUtils.newGenericThreadFactory("NettyBossThreads",
-                serverConfig.getServerAcceptorThreads()));
-            this.eventLoopGroupSelector = new NioEventLoopGroup(serverConfig.getServerSelectorThreads(), ThreadUtils.newGenericThreadFactory("NettyNioIoThreads",
-                serverConfig.getServerSelectorThreads()));
+            this.eventLoopGroupBoss = new NioEventLoopGroup(1, new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyNIOBoss_%d", this.threadIndex.incrementAndGet()));
+                }
+            });
+
+            this.eventLoopGroupSelector = new NioEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+                private int threadTotal = nettyServerConfig.getServerSelectorThreads();
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyServerNIOSelector_%d_%d", threadTotal, this.threadIndex.incrementAndGet()));
+                }
+            });
+
             this.socketChannelClass = NioServerSocketChannel.class;
         }
         this.port = nettyServerConfig.getListenPort();

remoting/src/main/java/org/apache/rocketmq/remoting/util/ServiceProvider.java

@@ -40,8 +40,12 @@ public class ServiceProvider {
 
     public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener";
 
+    public static final String RPC_HOOK_ID = "META-INF/service/org.apache.rocketmq.remoting.RPCHook";
+
+
     public static final String ACL_VALIDATOR_ID = "META-INF/service/org.apache.rocketmq.acl.AccessValidator";
 
+
     static {
         thisClassLoader = getClassLoader(ServiceProvider.class);
     }

snode/src/main/java/org/apache/rocketmq/snode/client/impl/IOTClientManagerImpl.java

@@ -21,7 +21,7 @@ import org.apache.rocketmq.snode.SnodeController;
 
 public class IOTClientManagerImpl extends ClientManagerImpl {
 
-    public static final String IOTGROUP = "IOTGROUP";
+    public static final String IOT_GROUP = "IOT_GROUP";
     private final SnodeController snodeController;
 
     public IOTClientManagerImpl(SnodeController snodeController) {

srvutil/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.4.0-SNAPSHOT</version>
+        <version>4.4.1-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>